{ "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { "buffer": { "type": "object", "description": "An optional buffer to store messages during transit. Default: map[none:map[]]", "properties": { "memory": { "type": "object", "properties": { "batch_policy": { "type": "object", "description": "Optionally configure a policy to flush buffered messages in batches.", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "enabled": { "type": "boolean", "description": "Whether to batch messages as they are flushed. Default: false" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } } }, "limit": { "type": "integer", "description": "The maximum buffer size (in bytes) to allow before applying backpressure upstream. Default: 524288000" }, "spillover": { "type": "boolean", "description": "Whether to drop incoming messages that will exceed the buffer limit. Default: false" } } }, "none": { "type": "object", "description": " Default: map[]" }, "sqlite": { "type": "object", "properties": { "path": { "type": "string", "description": "The path of the database file, which will be created if it does not already exist." }, "post_processors": { "type": "array", "description": "An optional list of processors to apply to messages after they are consumed from the buffer. These processors are useful for undoing any compression, archiving, etc that may have been done by your `pre_processors`.", "items": { "type": "object" } }, "pre_processors": { "type": "array", "description": "An optional list of processors to apply to messages before they are stored within the buffer. These processors are useful for compressing, archiving or otherwise reducing the data in size before it's stored on disk.", "items": { "type": "object" } } } }, "system_window": { "type": "object", "properties": { "allowed_lateness": { "type": "string", "description": "An optional duration string describing the length of time to wait after a window has ended before flushing it, allowing late arrivals to be included. Since this windowing buffer uses the system clock an allowed lateness can improve the matching of messages when using event time. Default: ", "examples": [ "10s", "1m" ] }, "offset": { "type": "string", "description": "An optional duration string to offset the beginning of each window by, otherwise they are aligned to the zeroth minute and zeroth hour on the UTC clock. The offset cannot be a larger or equal measure to the window size or the slide. Default: ", "examples": [ "-6h", "30m" ] }, "size": { "type": "string", "description": "A duration string describing the size of each window. By default windows are aligned to the zeroth minute and zeroth hour on the UTC clock, meaning windows of 1 hour duration will match the turn of each hour in the day, this can be adjusted with the `offset` field.", "examples": [ "30s", "10m" ] }, "slide": { "type": "string", "description": "An optional duration string describing by how much time the beginning of each window should be offset from the beginning of the previous, and therefore creates sliding windows instead of tumbling. When specified this duration must be smaller than the `size` of the window. Default: ", "examples": [ "30s", "10m" ] }, "timestamp_mapping": { "type": "string", "description": "\nA [Bloblang mapping](/docs/guides/bloblang/about) applied to each message during ingestion that provides the timestamp to use for allocating it a window. By default the function `now()` is used in order to generate a fresh timestamp at the time of ingestion (the processing time), whereas this mapping can instead extract a timestamp from the message itself (the event time).\n\nThe timestamp value assigned to `root` must either be a numerical unix time in seconds (with up to nanosecond precision via decimals), or a string in ISO 8601 format. If the mapping fails or provides an invalid result the message will be dropped (with logging to describe the problem).\n Default: root = now()", "examples": [ "root = this.created_at", "root = metadata(\"kafka_timestamp_unix\").string()" ] } } } } }, "cache_resources": { "type": "array", "description": "A list of cache resources, each must have a unique label. Default: []", "items": { "type": "object", "properties": { "aws_dynamodb": { "type": "object", "properties": { "consistent_read": { "type": "boolean", "description": "Whether to use strongly consistent reads on Get commands. Default: false" }, "credentials": { "type": "object", "description": "Optional manual configuration of AWS credentials to use. More information can be found [in this document](/docs/guides/cloud/aws).", "properties": { "expiry_window": { "type": "string", "description": "Allow the credentials to trigger refreshing prior to the credentials actually expiring. This is beneficial so race conditions with expiring credentials do not cause requests to fail. For example '10s' would refresh credentials ten seconds before expiration. Setting to a duration of `0` disables the expiry window. Default: " }, "from_ec2_role": { "type": "boolean", "description": "Use the credentials of a host EC2 machine configured to assume [an IAM role associated with the instance](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html). Default: false" }, "id": { "type": "string", "description": "The ID of credentials to use. Default: " }, "profile": { "type": "string", "description": "A profile from `~/.aws/credentials` to use. Default: " }, "role": { "type": "string", "description": "A role ARN to assume. Default: " }, "role_external_id": { "type": "string", "description": "An external ID to provide when assuming a role. Default: " }, "secret": { "type": "string", "description": "The secret for the credentials being used. Default: " }, "token": { "type": "string", "description": "The token for the credentials being used, required when using short term credentials. Default: " } } }, "data_key": { "type": "string", "description": "The key of the table column to store item values within." }, "default_ttl": { "type": "string", "description": "An optional default TTL to set for items, calculated from the moment the item is cached. A `ttl_key` must be specified in order to set item TTLs." }, "endpoint": { "type": "string", "description": "Allows you to specify a custom endpoint for the AWS API. Default: " }, "hash_key": { "type": "string", "description": "The key of the table column to store item keys within." }, "region": { "type": "string", "description": "The AWS region to target. Default: " }, "retries": { "type": "object", "description": "Determine time intervals and cut offs for retry attempts.", "properties": { "initial_interval": { "type": "string", "description": "The initial period to wait between retry attempts. Default: 1s", "examples": [ "50ms", "1s" ] }, "max_elapsed_time": { "type": "string", "description": "The maximum overall period of time to spend on retry attempts before the request is aborted. Default: 30s", "examples": [ "1m", "1h" ] }, "max_interval": { "type": "string", "description": "The maximum period to wait between retry attempts Default: 5s", "examples": [ "5s", "1m" ] } } }, "table": { "type": "string", "description": "The table to store items in." }, "ttl_key": { "type": "string", "description": "The column key to place the TTL value within." } } }, "aws_s3": { "type": "object", "properties": { "bucket": { "type": "string", "description": "The S3 bucket to store items in." }, "content_type": { "type": "string", "description": "The content type to set for each item. Default: application/octet-stream" }, "credentials": { "type": "object", "description": "Optional manual configuration of AWS credentials to use. More information can be found [in this document](/docs/guides/cloud/aws).", "properties": { "expiry_window": { "type": "string", "description": "Allow the credentials to trigger refreshing prior to the credentials actually expiring. This is beneficial so race conditions with expiring credentials do not cause requests to fail. For example '10s' would refresh credentials ten seconds before expiration. Setting to a duration of `0` disables the expiry window. Default: " }, "from_ec2_role": { "type": "boolean", "description": "Use the credentials of a host EC2 machine configured to assume [an IAM role associated with the instance](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html). Default: false" }, "id": { "type": "string", "description": "The ID of credentials to use. Default: " }, "profile": { "type": "string", "description": "A profile from `~/.aws/credentials` to use. Default: " }, "role": { "type": "string", "description": "A role ARN to assume. Default: " }, "role_external_id": { "type": "string", "description": "An external ID to provide when assuming a role. Default: " }, "secret": { "type": "string", "description": "The secret for the credentials being used. Default: " }, "token": { "type": "string", "description": "The token for the credentials being used, required when using short term credentials. Default: " } } }, "endpoint": { "type": "string", "description": "Allows you to specify a custom endpoint for the AWS API. Default: " }, "force_path_style_urls": { "type": "boolean", "description": "Forces the client API to use path style URLs, which helps when connecting to custom endpoints. Default: false" }, "region": { "type": "string", "description": "The AWS region to target. Default: " }, "retries": { "type": "object", "description": "Determine time intervals and cut offs for retry attempts.", "properties": { "initial_interval": { "type": "string", "description": "The initial period to wait between retry attempts. Default: 1s", "examples": [ "50ms", "1s" ] }, "max_elapsed_time": { "type": "string", "description": "The maximum overall period of time to spend on retry attempts before the request is aborted. Default: 30s", "examples": [ "1m", "1h" ] }, "max_interval": { "type": "string", "description": "The maximum period to wait between retry attempts Default: 5s", "examples": [ "5s", "1m" ] } } } } }, "couchbase": { "type": "object", "properties": { "bucket": { "type": "string", "description": "Couchbase bucket." }, "collection": { "type": "string", "description": "Bucket collection. Default: _default" }, "default_ttl": { "type": "string", "description": "An optional default TTL to set for items, calculated from the moment the item is cached." }, "password": { "type": "string", "description": "Password to connect to the cluster." }, "timeout": { "type": "string", "description": "Operation timeout. Default: 15s" }, "transcoder": { "type": "string", "description": "Couchbase transcoder to use. Default: legacy" }, "url": { "type": "string", "description": "Couchbase connection string.", "examples": [ "couchbase://localhost:11210" ] }, "username": { "type": "string", "description": "Username to connect to the cluster." } } }, "file": { "type": "object", "properties": { "directory": { "type": "string", "description": "The directory within which to store items." } } }, "gcp_cloud_storage": { "type": "object", "properties": { "bucket": { "type": "string", "description": "The Google Cloud Storage bucket to store items in." }, "content_type": { "type": "string", "description": "Optional field to explicitly set the Content-Type." } } }, "lru": { "type": "object", "properties": { "algorithm": { "type": "string", "description": "the lru cache implementation Default: standard" }, "cap": { "type": "integer", "description": "The cache maximum capacity (number of entries) Default: 1000" }, "init_values": { "type": "object", "description": "A table of key/value pairs that should be present in the cache on initialization. This can be used to create static lookup tables. Default: map[]", "examples": [ { "Nickelback": "1995", "Spice Girls": "1994", "The Human League": "1977" } ] }, "optimistic": { "type": "boolean", "description": "If true, we do not lock on read/write events. The lru package is thread-safe, however the ADD operation is not atomic. Default: false" }, "two_queues_ghost_ratio": { "type": "number", "description": "is the default ratio of ghost entries kept to track entries recently evicted on two_queues cache. Default: 0.5" }, "two_queues_recent_ratio": { "type": "number", "description": "is the ratio of the two_queues cache dedicated to recently added entries that have only been accessed once. Default: 0.25" } } }, "memcached": { "type": "object", "properties": { "addresses": { "type": "array", "description": "A list of addresses of memcached servers to use.", "items": { "type": "string" } }, "default_ttl": { "type": "string", "description": "A default TTL to set for items, calculated from the moment the item is cached. Default: 300s" }, "prefix": { "type": "string", "description": "An optional string to prefix item keys with in order to prevent collisions with similar services." }, "retries": { "type": "object", "description": "Determine time intervals and cut offs for retry attempts.", "properties": { "initial_interval": { "type": "string", "description": "The initial period to wait between retry attempts. Default: 1s", "examples": [ "50ms", "1s" ] }, "max_elapsed_time": { "type": "string", "description": "The maximum overall period of time to spend on retry attempts before the request is aborted. Default: 30s", "examples": [ "1m", "1h" ] }, "max_interval": { "type": "string", "description": "The maximum period to wait between retry attempts Default: 5s", "examples": [ "5s", "1m" ] } } } } }, "memory": { "type": "object", "properties": { "compaction_interval": { "type": "string", "description": "The period of time to wait before each compaction, at which point expired items are removed. This field can be set to an empty string in order to disable compactions/expiry entirely. Default: 60s" }, "default_ttl": { "type": "string", "description": "The default TTL of each item. After this period an item will be eligible for removal during the next compaction. Default: 5m" }, "init_values": { "type": "object", "description": "A table of key/value pairs that should be present in the cache on initialization. This can be used to create static lookup tables. Default: map[]", "examples": [ { "Nickelback": "1995", "Spice Girls": "1994", "The Human League": "1977" } ] }, "shards": { "type": "integer", "description": "A number of logical shards to spread keys across, increasing the shards can have a performance benefit when processing a large number of keys. Default: 1" } } }, "mongodb": { "type": "object", "properties": { "collection": { "type": "string", "description": "The name of the target collection." }, "database": { "type": "string", "description": "The name of the target MongoDB database." }, "key_field": { "type": "string", "description": "The field in the document that is used as the key." }, "password": { "type": "string", "description": "The password to connect to the database. Default: " }, "url": { "type": "string", "description": "The URL of the target MongoDB server.", "examples": [ "mongodb://localhost:27017" ] }, "username": { "type": "string", "description": "The username to connect to the database. Default: " }, "value_field": { "type": "string", "description": "The field in the document that is used as the value." } } }, "multilevel": { "type": "array", "items": { "type": "string" } }, "nats_kv": { "type": "object", "properties": { "auth": { "type": "object", "description": "Optional configuration of NATS authentication parameters.", "properties": { "nkey_file": { "type": "string", "description": "An optional file containing a NKey seed.", "examples": [ "./seed.nk" ] }, "user_credentials_file": { "type": "string", "description": "An optional file containing user credentials which consist of an user JWT and corresponding NKey seed.", "examples": [ "./user.creds" ] }, "user_jwt": { "type": "string", "description": "An optional plain text user JWT (given along with the corresponding user NKey Seed)." }, "user_nkey_seed": { "type": "string", "description": "An optional plain text user NKey Seed (given along with the corresponding user JWT)." } } }, "bucket": { "type": "string", "description": "The name of the KV bucket.", "examples": [ "my_kv_bucket" ] }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "urls": { "type": "array", "description": "A list of URLs to connect to. If an item of the list contains commas it will be expanded into multiple URLs.", "items": { "type": "string" }, "examples": [ [ "nats://127.0.0.1:4222" ], [ "nats://username:password@127.0.0.1:4222" ] ] } } }, "noop": { "type": "object", "description": " Default: map[]" }, "redis": { "type": "object", "properties": { "default_ttl": { "type": "string", "description": "An optional default TTL to set for items, calculated from the moment the item is cached." }, "kind": { "type": "string", "description": "Specifies a simple, cluster-aware, or failover-aware redis client. Default: simple" }, "master": { "type": "string", "description": "Name of the redis master when `kind` is `failover` Default: ", "examples": [ "mymaster" ] }, "prefix": { "type": "string", "description": "An optional string to prefix item keys with in order to prevent collisions with similar services." }, "retries": { "type": "object", "description": "Determine time intervals and cut offs for retry attempts.", "properties": { "initial_interval": { "type": "string", "description": "The initial period to wait between retry attempts. Default: 500ms", "examples": [ "50ms", "1s" ] }, "max_elapsed_time": { "type": "string", "description": "The maximum overall period of time to spend on retry attempts before the request is aborted. Default: 5s", "examples": [ "1m", "1h" ] }, "max_interval": { "type": "string", "description": "The maximum period to wait between retry attempts Default: 1s", "examples": [ "5s", "1m" ] } } }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.\n\n**Troubleshooting**\n\nSome cloud hosted instances of Redis (such as Azure Cache) might need some hand holding in order to establish stable connections. Unfortunately, it is often the case that TLS issues will manifest as generic error messages such as \"i/o timeout\". If you're using TLS and are seeing connectivity problems consider setting `enable_renegotiation` to `true`, and ensuring that the server supports at least TLS version 1.2.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "url": { "type": "string", "description": "The URL of the target Redis server. Database is optional and is supplied as the URL path.", "examples": [ "redis://:6397", "redis://localhost:6379", "redis://foousername:foopassword@redisplace:6379", "redis://:foopassword@redisplace:6379", "redis://localhost:6379/1", "redis://localhost:6379/1,redis://localhost:6380/1" ] } } }, "ristretto": { "type": "object", "properties": { "default_ttl": { "type": "string", "description": "A default TTL to set for items, calculated from the moment the item is cached. Set to an empty string or zero duration to disable TTLs. Default: ", "examples": [ "5m", "60s" ] }, "get_retries": { "type": "object", "description": "Determines how and whether get attempts should be retried if the key is not found. Ristretto is a concurrent cache that does not immediately reflect writes, and so it can sometimes be useful to enable retries at the cost of speed in cases where the key is expected to exist.", "properties": { "enabled": { "type": "boolean", "description": "Whether retries should be enabled. Default: false" }, "initial_interval": { "type": "string", "description": "The initial period to wait between retry attempts. Default: 1s", "examples": [ "50ms", "1s" ] }, "max_elapsed_time": { "type": "string", "description": "The maximum overall period of time to spend on retry attempts before the request is aborted. Default: 30s", "examples": [ "1m", "1h" ] }, "max_interval": { "type": "string", "description": "The maximum period to wait between retry attempts Default: 5s", "examples": [ "5s", "1m" ] } } } } }, "sql": { "type": "object", "properties": { "azure": { "type": "object", "description": "Optional Fields that can be set to use Azure based authentication for Azure Postgres SQL", "properties": { "entra_enabled": { "type": "boolean", "description": "An optional field used to generate an entra token to connect to 'Azure Database for PostgreSQL flexible server', This will create a new connection string with the host, user and database from the DSN field - you may need to URL encode the dsn! The [Default Azure Credential Chain](https://learn.microsoft.com/en-gb/azure/developer/go/sdk/authentication/authentication-overview#defaultazurecredential) is used from the Azure SDK. Default: false" }, "token_request_options": { "type": "object", "properties": { "claims": { "type": "string", "description": "Set additional claims for the token. Default: " }, "enable_cae": { "type": "boolean", "description": "Indicates whether to enable Continuous Access Evaluation (CAE) for the requested token Default: false" }, "scopes": { "type": "array", "description": "Scopes contains the list of permission scopes required for the token. Default: [https://ossrdbms-aad.database.windows.net/.default]", "items": { "type": "string" } }, "tenant_id": { "type": "string", "description": "tenant_id identifies the tenant from which to request the token. azure credentials authenticate in their configured default tenants when this field isn't set. Default: " } } } } }, "conn_max_idle": { "type": "integer", "description": "An optional maximum number of connections in the idle connection pool. If conn_max_open is greater than 0 but less than the new conn_max_idle, then the new conn_max_idle will be reduced to match the conn_max_open limit. If `value \u003c= 0`, no idle connections are retained. The default max idle connections is currently 2. This may change in a future release. Default: 2" }, "conn_max_idle_time": { "type": "string", "description": "An optional maximum amount of time a connection may be idle. Expired connections may be closed lazily before reuse. If `value \u003c= 0`, connections are not closed due to a connections idle time." }, "conn_max_life_time": { "type": "string", "description": "An optional maximum amount of time a connection may be reused. Expired connections may be closed lazily before reuse. If `value \u003c= 0`, connections are not closed due to a connections age." }, "conn_max_open": { "type": "integer", "description": "An optional maximum number of open connections to the database. If conn_max_idle is greater than 0 and the new conn_max_open is less than conn_max_idle, then conn_max_idle will be reduced to match the new conn_max_open limit. If `value \u003c= 0`, then there is no limit on the number of open connections. The default is 0 (unlimited)." }, "credentials": { "type": "object", "description": "Optional manual configuration of AWS credentials to use. More information can be found [in this document](/docs/guides/cloud/aws).", "properties": { "expiry_window": { "type": "string", "description": "Allow the credentials to trigger refreshing prior to the credentials actually expiring. This is beneficial so race conditions with expiring credentials do not cause requests to fail. For example '10s' would refresh credentials ten seconds before expiration. Setting to a duration of `0` disables the expiry window. Default: " }, "from_ec2_role": { "type": "boolean", "description": "Use the credentials of a host EC2 machine configured to assume [an IAM role associated with the instance](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html). Default: false" }, "id": { "type": "string", "description": "The ID of credentials to use. Default: " }, "profile": { "type": "string", "description": "A profile from `~/.aws/credentials` to use. Default: " }, "role": { "type": "string", "description": "A role ARN to assume. Default: " }, "role_external_id": { "type": "string", "description": "An external ID to provide when assuming a role. Default: " }, "secret": { "type": "string", "description": "The secret for the credentials being used. Default: " }, "token": { "type": "string", "description": "The token for the credentials being used, required when using short term credentials. Default: " } } }, "driver": { "type": "string", "description": "A database [driver](#drivers) to use." }, "dsn": { "type": "string", "description": "A Data Source Name to identify the target database.\n\n#### Drivers\n\nThe following is a list of supported drivers, their placeholder style, and their respective DSN formats:\n\n| Driver | Data Source Name Format |\n|---|---|\n| `clickhouse` | [`clickhouse://[username[:password]@][netloc][:port]/dbname[?param1=value1\u0026...\u0026paramN=valueN]`](https://github.com/ClickHouse/clickhouse-go#dsn) |\n| `mysql` | `[username[:password]@][protocol[(address)]]/dbname[?param1=value1\u0026...\u0026paramN=valueN]` |\n| `postgres` | `postgres://[user[:password]@][netloc][:port][/dbname][?param1=value1\u0026...]` |\n| `mssql` | `sqlserver://[user[:password]@][netloc][:port][?database=dbname\u0026param1=value1\u0026...]` |\n| `sqlite` | `file:/path/to/filename.db[?param\u0026=value1\u0026...]` |\n| `oracle` | `oracle://[username[:password]@][netloc][:port]/service_name?server=server2\u0026server=server3` |\n| `snowflake` | `username[:password]@account_identifier/dbname/schemaname[?param1=value\u0026...\u0026paramN=valueN]` |\n| `spanner` | `projects/[project]/instances/[instance]/databases/dbname` |\n| `trino` | [`http[s]://user[:pass]@host[:port][?parameters]`](https://github.com/trinodb/trino-go-client#dsn-data-source-name) |\n| `gocosmos` | [`AccountEndpoint=\u003ccosmosdb-endpoint\u003e;AccountKey=\u003ccosmosdb-account-key\u003e[;TimeoutMs=\u003ctimeout-in-ms\u003e][;Version=\u003ccosmosdb-api-version\u003e][;DefaultDb/Db=\u003cdb-name\u003e][;AutoId=\u003ctrue/false\u003e][;InsecureSkipVerify=\u003ctrue/false\u003e]`](https://pkg.go.dev/github.com/microsoft/gocosmos#readme-example-usage) |\n\nPlease note that the `postgres` driver enforces SSL by default, you can override this with the parameter `sslmode=disable` if required.\n\nThe `snowflake` driver supports multiple DSN formats. Please consult [the docs](https://pkg.go.dev/github.com/snowflakedb/gosnowflake#hdr-Connection_String) for more details. For [key pair authentication](https://docs.snowflake.com/en/user-guide/key-pair-auth.html#configuring-key-pair-authentication), the DSN has the following format: `\u003csnowflake_user\u003e@\u003csnowflake_account\u003e/\u003cdb_name\u003e/\u003cschema_name\u003e?warehouse=\u003cwarehouse\u003e\u0026role=\u003crole\u003e\u0026authenticator=snowflake_jwt\u0026privateKey=\u003cbase64_url_encoded_private_key\u003e`, where the value for the `privateKey` parameter can be constructed from an unencrypted RSA private key file `rsa_key.p8` using `openssl enc -d -base64 -in rsa_key.p8 | basenc --base64url -w0` (you can use `gbasenc` insted of `basenc` on OSX if you install `coreutils` via Homebrew). If you have a password-encrypted private key, you can decrypt it using `openssl pkcs8 -in rsa_key_encrypted.p8 -out rsa_key.p8`. Also, make sure fields such as the username are URL-encoded.\n\nThe [`gocosmos`](https://pkg.go.dev/github.com/microsoft/gocosmos) driver is still experimental, but it has support for [hierarchical partition keys](https://learn.microsoft.com/en-us/azure/cosmos-db/hierarchical-partition-keys) as well as [cross-partition queries](https://learn.microsoft.com/en-us/azure/cosmos-db/nosql/how-to-query-container#cross-partition-query). Please refer to the [SQL notes](https://github.com/microsoft/gocosmos/blob/main/SQL.md) for details.", "examples": [ "clickhouse://username:password@host1:9000,host2:9000/database?dial_timeout=200ms\u0026max_execution_time=60", "foouser:foopassword@tcp(localhost:3306)/foodb", "postgres://foouser:foopass@localhost:5432/foodb?sslmode=disable", "oracle://foouser:foopass@localhost:1521/service_name" ] }, "endpoint": { "type": "string", "description": "Allows you to specify a custom endpoint for the AWS API. Default: " }, "iam_enabled": { "type": "boolean", "description": "An optional field used to generate an IAM authentication token to connect to an Amazon Relational Database (RDS) DB instance. This will overwrite the Password in the DSN with the generated token only if the drivers are `mysql` or `postgres`. Default: false" }, "init_files": { "type": "array", "description": "\nAn optional list of file paths containing SQL statements to execute immediately upon the first connection to the target database. This is a useful way to initialise tables before processing data. Glob patterns are supported, including super globs (double star).\n\nCare should be taken to ensure that the statements are idempotent, and therefore would not cause issues when run multiple times after service restarts. If both `init_statement` and `init_files` are specified the `init_statement` is executed _after_ the `init_files`.\n\nIf a statement fails for any reason a warning log will be emitted but the operation of this component will not be stopped.\n", "items": { "type": "string" }, "examples": [ [ "./init/*.sql" ], [ "./foo.sql", "./bar.sql" ] ] }, "init_statement": { "type": "string", "description": "\nAn optional SQL statement to execute immediately upon the first connection to the target database. This is a useful way to initialise tables before processing data. Care should be taken to ensure that the statement is idempotent, and therefore would not cause issues when run multiple times after service restarts.\n\nIf both `init_statement` and `init_files` are specified the `init_statement` is executed _after_ the `init_files`.\n\nIf the statement fails for any reason a warning log will be emitted but the operation of this component will not be stopped.\n", "examples": [ "\nCREATE TABLE IF NOT EXISTS some_table (\n foo varchar(50) not null,\n bar integer,\n baz varchar(50),\n primary key (foo)\n) WITHOUT ROWID;\n" ] }, "init_verify_conn": { "type": "boolean", "description": "Whether to verify the database connection on startup by performing a simple ping, by default this is disabled. Default: false" }, "key_column": { "type": "string", "description": "The name of a column to be used for storing cache item keys. This column should support strings of arbitrary size.", "examples": [ "foo" ] }, "region": { "type": "string", "description": "The AWS region to target. Default: " }, "secret_name": { "type": "string", "description": "An optional field that can be used to get the Username + Password from AWS Secrets Manager. This will overwrite the Username + Password in the DSN with the values from the Secret only if the driver is set to `postgres`." }, "set_suffix": { "type": "string", "description": "An optional suffix to append to each insert query for a cache `set` operation. This should modify an insert statement into an upsert appropriate for the given SQL engine.", "examples": [ "ON DUPLICATE KEY UPDATE bar=VALUES(bar)", "ON CONFLICT (foo) DO UPDATE SET bar=excluded.bar", "ON CONFLICT (foo) DO NOTHING" ] }, "table": { "type": "string", "description": "The table to insert/read/delete cache items.", "examples": [ "foo" ] }, "value_column": { "type": "string", "description": "The name of a column to be used for storing cache item values. This column should support strings of arbitrary size.", "examples": [ "bar" ] } } }, "ttlru": { "type": "object", "properties": { "cap": { "type": "integer", "description": "The cache maximum capacity (number of entries) Default: 1024" }, "default_ttl": { "type": "string", "description": "The cache ttl of each element Default: 5m0s" }, "init_values": { "type": "object", "description": "A table of key/value pairs that should be present in the cache on initialization. This can be used to create static lookup tables. Default: map[]", "examples": [ { "Nickelback": "1995", "Spice Girls": "1994", "The Human League": "1977" } ] }, "optimistic": { "type": "boolean", "description": "If true, we do not lock on read/write events. The ttlru package is thread-safe, however the ADD operation is not atomic. Default: false" }, "ttl": { "type": "string", "description": "Deprecated. Please use `default_ttl` field" } } } } } }, "error_handling": { "type": "object", "description": "Environment-wide settings for handling errored messages.", "properties": { "log": { "type": "object", "description": "Configuration for global logging message-level errors.", "properties": { "add_payload": { "type": "boolean", "description": "Whether to add a failed message payload to an error log. Default: false" }, "enabled": { "type": "boolean", "description": "Whether to enable message-level error logging. Default: false" }, "sampling_ratio": { "type": "number", "description": "Sets the ratio of errored messages within a batch to sample. Default: 1" } } }, "strategy": { "type": "string", "description": "The error handling strategy. Default: none" } } }, "http": { "type": "object", "description": "Configures the service-wide HTTP server.", "properties": { "address": { "type": "string", "description": "The address to bind to. Default: 0.0.0.0:4195" }, "basic_auth": { "type": "object", "description": "Allows you to enforce and customise basic authentication for requests to the HTTP server.", "properties": { "algorithm": { "type": "string", "description": "Encryption algorithm used to generate `password_hash`. Default: sha256", "examples": [ "md5", "sha256", "bcrypt", "scrypt" ] }, "enabled": { "type": "boolean", "description": "Enable basic authentication Default: false" }, "password_hash": { "type": "string", "description": "Hashed password required to authenticate. (base64 encoded) Default: " }, "realm": { "type": "string", "description": "Custom realm name Default: restricted" }, "salt": { "type": "string", "description": "Salt for scrypt algorithm. (base64 encoded) Default: " }, "username": { "type": "string", "description": "Username required to authenticate. Default: " } } }, "cert_file": { "type": "string", "description": "An optional certificate file for enabling TLS. Default: " }, "cors": { "type": "object", "description": "Adds Cross-Origin Resource Sharing headers.", "properties": { "allowed_headers": { "type": "array", "description": "Appends additional headers to the list of default allowed headers: Accept, Accept-Language, Content-Language \u0026 Origin. These default headers are therefore always allowed. Default: []", "items": { "type": "string" } }, "allowed_methods": { "type": "array", "description": "Used to explicitly set allowed methods in the Access-Control-Allow-Methods header. Default: [GET HEAD POST PUT PATCH DELETE]", "items": { "type": "string" } }, "allowed_origins": { "type": "array", "description": "An explicit list of origins that are allowed for CORS requests. Default: []", "items": { "type": "string" } }, "enabled": { "type": "boolean", "description": "Whether to allow CORS requests. Default: false" } } }, "debug_endpoints": { "type": "boolean", "description": "Whether to register a few extra endpoints that can be useful for debugging performance or behavioral problems. Default: false" }, "enabled": { "type": "boolean", "description": "Whether to enable to HTTP server. Default: true" }, "key_file": { "type": "string", "description": "An optional key file for enabling TLS. Default: " }, "root_path": { "type": "string", "description": "Specifies a general prefix for all endpoints, this can help isolate the service endpoints when using a reverse proxy with other shared services. All endpoints will still be registered at the root as well as behind the prefix, e.g. with a root_path set to `/foo` the endpoint `/version` will be accessible from both `/version` and `/foo/version`. Default: /bento" } } }, "input": { "type": "object", "description": "An input to source messages from. Default: map[stdin:map[]]", "properties": { "amqp_0_9": { "type": "object", "properties": { "auto_ack": { "type": "boolean", "description": "Acknowledge messages automatically as they are consumed rather than waiting for acknowledgments from downstream. This can improve throughput and prevent the pipeline from blocking but at the cost of eliminating delivery guarantees. Default: false" }, "bindings_declare": { "type": "array", "description": "Allows you to passively declare bindings for the target queue.", "items": { "type": "object" }, "examples": [ [ { "exchange": "foo", "key": "bar" } ] ] }, "consumer_tag": { "type": "string", "description": "A consumer tag. Default: " }, "nack_reject_patterns": { "type": "array", "description": "A list of regular expression patterns whereby if a message that has failed to be delivered by Bento has an error that matches it will be dropped (or delivered to a dead-letter queue if one exists). By default failed messages are nacked with requeue enabled. Default: []", "items": { "type": "string" }, "examples": [ [ "^reject me please:.+$" ] ] }, "prefetch_count": { "type": "integer", "description": "The maximum number of pending messages to have consumed at a time. Default: 10" }, "prefetch_size": { "type": "integer", "description": "The maximum amount of pending messages measured in bytes to have consumed at a time. Default: 0" }, "queue": { "type": "string", "description": "An AMQP queue to consume from." }, "queue_declare": { "type": "object", "description": "Allows you to passively declare the target queue. If the queue already exists then the declaration passively verifies that they match the target fields.", "properties": { "auto_delete": { "type": "boolean", "description": "Whether the declared queue will auto-delete. Default: false" }, "durable": { "type": "boolean", "description": "Whether the declared queue is durable. Default: true" }, "enabled": { "type": "boolean", "description": "Whether to enable queue declaration. Default: false" } } }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "urls": { "type": "array", "description": "A list of URLs to connect to. The first URL to successfully establish a connection will be used until the connection is closed. If an item of the list contains commas it will be expanded into multiple URLs.", "items": { "type": "string" }, "examples": [ [ "amqp://guest:guest@127.0.0.1:5672/" ], [ "amqp://127.0.0.1:5672/,amqp://127.0.0.2:5672/" ], [ "amqp://127.0.0.1:5672/", "amqp://127.0.0.2:5672/" ] ] } } }, "amqp_1": { "type": "object", "properties": { "azure_renew_lock": { "type": "boolean", "description": "Experimental: Azure service bus specific option to renew lock if processing takes more then configured lock time Default: false" }, "credit": { "type": "integer", "description": "Specifies the maximum number of unacknowledged messages the sender can transmit. Once this limit is reached, no more messages will arrive until messages are acknowledged and settled. Default: 64" }, "read_header": { "type": "boolean", "description": "Read additional message header fields into `amqp_*` metadata properties. Default: false" }, "sasl": { "type": "object", "description": "Enables SASL authentication.", "properties": { "mechanism": { "type": "string", "description": "The SASL authentication mechanism to use. Default: none" }, "password": { "type": "string", "description": "A SASL plain text password. It is recommended that you use environment variables to populate this field. Default: ", "examples": [ "${PASSWORD}" ] }, "user": { "type": "string", "description": "A SASL plain text username. It is recommended that you use environment variables to populate this field. Default: ", "examples": [ "${USER}" ] } } }, "source_address": { "type": "string", "description": "The source address to consume from.", "examples": [ "/foo", "queue:/bar", "topic:/baz" ] }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "url": { "type": "string", "description": "A URL to connect to.", "examples": [ "amqp://localhost:5672/", "amqps://guest:guest@localhost:5672/" ] }, "urls": { "type": "array", "description": "A list of URLs to connect to. The first URL to successfully establish a connection will be used until the connection is closed. If an item of the list contains commas it will be expanded into multiple URLs.", "items": { "type": "string" }, "examples": [ [ "amqp://guest:guest@127.0.0.1:5672/" ], [ "amqp://127.0.0.1:5672/,amqp://127.0.0.2:5672/" ], [ "amqp://127.0.0.1:5672/", "amqp://127.0.0.2:5672/" ] ] } } }, "aws_kinesis": { "type": "object", "properties": { "auto_replay_nacks": { "type": "boolean", "description": "Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation. Default: true" }, "batching": { "type": "object", "description": "\nAllows you to configure a [batching policy](/docs/configuration/batching).", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] }, "checkpoint_limit": { "type": "integer", "description": "The maximum gap between the in flight sequence versus the latest acknowledged sequence at a given time. Increasing this limit enables parallel processing and batching at the output level to work on individual shards. Any given sequence will not be committed unless all messages under that offset are delivered in order to preserve at least once delivery guarantees. Default: 1024" }, "commit_period": { "type": "string", "description": "The period of time between each update to the checkpoint table. Default: 5s" }, "credentials": { "type": "object", "description": "Optional manual configuration of AWS credentials to use. More information can be found [in this document](/docs/guides/cloud/aws).", "properties": { "expiry_window": { "type": "string", "description": "Allow the credentials to trigger refreshing prior to the credentials actually expiring. This is beneficial so race conditions with expiring credentials do not cause requests to fail. For example '10s' would refresh credentials ten seconds before expiration. Setting to a duration of `0` disables the expiry window. Default: " }, "from_ec2_role": { "type": "boolean", "description": "Use the credentials of a host EC2 machine configured to assume [an IAM role associated with the instance](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html). Default: false" }, "id": { "type": "string", "description": "The ID of credentials to use. Default: " }, "profile": { "type": "string", "description": "A profile from `~/.aws/credentials` to use. Default: " }, "role": { "type": "string", "description": "A role ARN to assume. Default: " }, "role_external_id": { "type": "string", "description": "An external ID to provide when assuming a role. Default: " }, "secret": { "type": "string", "description": "The secret for the credentials being used. Default: " }, "token": { "type": "string", "description": "The token for the credentials being used, required when using short term credentials. Default: " } } }, "dynamodb": { "type": "object", "description": "Determines the table used for storing and accessing the latest consumed sequence for shards, and for coordinating balanced consumers of streams.", "properties": { "billing_mode": { "type": "string", "description": "When creating the table determines the billing mode. Default: PAY_PER_REQUEST" }, "create": { "type": "boolean", "description": "Whether, if the table does not exist, it should be created. Default: false" }, "read_capacity_units": { "type": "integer", "description": "Set the provisioned read capacity when creating the table with a `billing_mode` of `PROVISIONED`. Default: 0" }, "table": { "type": "string", "description": "The name of the table to access. Default: " }, "write_capacity_units": { "type": "integer", "description": "Set the provisioned write capacity when creating the table with a `billing_mode` of `PROVISIONED`. Default: 0" } } }, "endpoint": { "type": "string", "description": "Allows you to specify a custom endpoint for the AWS API. Default: " }, "lease_period": { "type": "string", "description": "The period of time after which a client that has failed to update a shard checkpoint is assumed to be inactive. Default: 30s" }, "rebalance_period": { "type": "string", "description": "The period of time between each attempt to rebalance shards across clients. Default: 30s" }, "region": { "type": "string", "description": "The AWS region to target. Default: " }, "start_from_oldest": { "type": "boolean", "description": "Whether to consume from the oldest message when a sequence does not yet exist for the stream. Default: true" }, "streams": { "type": "array", "description": "One or more Kinesis data streams to consume from. Streams can either be specified by their name or full ARN. Shards of a stream are automatically balanced across consumers by coordinating through the provided DynamoDB table. Multiple comma separated streams can be listed in a single element. Shards are automatically distributed across consumers of a stream by coordinating through the provided DynamoDB table. Alternatively, it's possible to specify an explicit shard to consume from with a colon after the stream name, e.g. `foo:0` would consume the shard `0` of the stream `foo`.", "items": { "type": "string" }, "examples": [ [ "foo", "arn:aws:kinesis:*:111122223333:stream/my-stream" ] ] } } }, "aws_s3": { "type": "object", "properties": { "bucket": { "type": "string", "description": "The bucket to consume from. If the field `sqs.url` is specified this field is optional. Default: " }, "codec": { "type": "string", "description": "The way in which the bytes of a data source should be converted into discrete messages, codecs are useful for specifying how large files or continuous streams of data might be processed in small chunks rather than loading it all in memory. It's possible to consume lines using a custom delimiter with the `delim:x` codec, where x is the character sequence custom delimiter. Codecs can be chained with `/`, for example a gzip compressed CSV file can be consumed with the codec `gzip/csv`.", "examples": [ "lines", "delim:\t", "delim:foobar", "gzip/csv" ] }, "credentials": { "type": "object", "description": "Optional manual configuration of AWS credentials to use. More information can be found [in this document](/docs/guides/cloud/aws).", "properties": { "expiry_window": { "type": "string", "description": "Allow the credentials to trigger refreshing prior to the credentials actually expiring. This is beneficial so race conditions with expiring credentials do not cause requests to fail. For example '10s' would refresh credentials ten seconds before expiration. Setting to a duration of `0` disables the expiry window. Default: " }, "from_ec2_role": { "type": "boolean", "description": "Use the credentials of a host EC2 machine configured to assume [an IAM role associated with the instance](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html). Default: false" }, "id": { "type": "string", "description": "The ID of credentials to use. Default: " }, "profile": { "type": "string", "description": "A profile from `~/.aws/credentials` to use. Default: " }, "role": { "type": "string", "description": "A role ARN to assume. Default: " }, "role_external_id": { "type": "string", "description": "An external ID to provide when assuming a role. Default: " }, "secret": { "type": "string", "description": "The secret for the credentials being used. Default: " }, "token": { "type": "string", "description": "The token for the credentials being used, required when using short term credentials. Default: " } } }, "delete_objects": { "type": "boolean", "description": "Whether to delete downloaded objects from the bucket once they are processed. Default: false" }, "endpoint": { "type": "string", "description": "Allows you to specify a custom endpoint for the AWS API. Default: " }, "force_path_style_urls": { "type": "boolean", "description": "Forces the client API to use path style URLs for downloading keys, which is often required when connecting to custom endpoints. Default: false" }, "max_buffer": { "type": "integer", "description": " Default: 1000000" }, "prefix": { "type": "string", "description": "An optional path prefix, if set only objects with the prefix are consumed when walking a bucket. Default: " }, "region": { "type": "string", "description": "The AWS region to target. Default: " }, "scanner": { "type": "object", "description": "The [scanner](/docs/components/scanners/about) by which the stream of bytes consumed will be broken out into individual messages. Scanners are useful for processing large sources of data without holding the entirety of it within memory. For example, the `csv` scanner allows you to process individual CSV rows without loading the entire CSV file in memory at once. Default: map[to_the_end:map[]]" }, "sqs": { "type": "object", "description": "Consume SQS messages in order to trigger key downloads.", "properties": { "bucket_path": { "type": "string", "description": "A [dot path](/docs/configuration/field_paths) whereby the bucket name can be found in SQS messages. Default: Records.*.s3.bucket.name" }, "delay_period": { "type": "string", "description": "An optional period of time to wait from when a notification was originally sent to when the target key download is attempted. Default: ", "examples": [ "10s", "5m" ] }, "endpoint": { "type": "string", "description": "A custom endpoint to use when connecting to SQS. Default: " }, "envelope_path": { "type": "string", "description": "A [dot path](/docs/configuration/field_paths) of a field to extract an enveloped JSON payload for further extracting the key and bucket from SQS messages. This is specifically useful when subscribing an SQS queue to an SNS topic that receives bucket events. Default: ", "examples": [ "Message" ] }, "key_path": { "type": "string", "description": "A [dot path](/docs/configuration/field_paths) whereby object keys are found in SQS messages. Default: Records.*.s3.object.key" }, "max_messages": { "type": "integer", "description": "The maximum number of SQS messages to consume from each request. Default: 10" }, "url": { "type": "string", "description": "An optional SQS URL to connect to. When specified this queue will control which objects are downloaded. Default: " }, "wait_time_seconds": { "type": "integer", "description": "Whether to set the wait time. Enabling this activates long-polling. Valid values: 0 to 20. Default: 0" } } } } }, "aws_sqs": { "type": "object", "properties": { "credentials": { "type": "object", "description": "Optional manual configuration of AWS credentials to use. More information can be found [in this document](/docs/guides/cloud/aws).", "properties": { "expiry_window": { "type": "string", "description": "Allow the credentials to trigger refreshing prior to the credentials actually expiring. This is beneficial so race conditions with expiring credentials do not cause requests to fail. For example '10s' would refresh credentials ten seconds before expiration. Setting to a duration of `0` disables the expiry window. Default: " }, "from_ec2_role": { "type": "boolean", "description": "Use the credentials of a host EC2 machine configured to assume [an IAM role associated with the instance](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html). Default: false" }, "id": { "type": "string", "description": "The ID of credentials to use. Default: " }, "profile": { "type": "string", "description": "A profile from `~/.aws/credentials` to use. Default: " }, "role": { "type": "string", "description": "A role ARN to assume. Default: " }, "role_external_id": { "type": "string", "description": "An external ID to provide when assuming a role. Default: " }, "secret": { "type": "string", "description": "The secret for the credentials being used. Default: " }, "token": { "type": "string", "description": "The token for the credentials being used, required when using short term credentials. Default: " } } }, "custom_request_headers": { "type": "object", "description": "A map used to send custom HTTP headers alongside each SQS operation to AWS. Default: map[]" }, "delete_message": { "type": "boolean", "description": "Whether to delete the consumed message once it is acked. Disabling allows you to handle the deletion using a different mechanism. Default: true" }, "endpoint": { "type": "string", "description": "Allows you to specify a custom endpoint for the AWS API. Default: " }, "max_number_of_messages": { "type": "integer", "description": "The maximum number of messages to return on one poll. Valid values: 1 to 10. Default: 10" }, "region": { "type": "string", "description": "The AWS region to target. Default: " }, "reset_visibility": { "type": "boolean", "description": "Whether to set the visibility timeout of the consumed message to zero once it is nacked. Disabling honors the preset visibility timeout specified for the queue. Default: true" }, "update_visibility": { "type": "boolean", "description": "Whether to periodically refresh the visibility timeout of in-flight messages to prevent more-than-once delivery while still processing. Default: true" }, "url": { "type": "string", "description": "The SQS URL to consume from." }, "wait_time_seconds": { "type": "integer", "description": "Whether to set the wait time. Enabling this activates long-polling. Valid values: 0 to 20. Default: 0" } } }, "azure_blob_storage": { "type": "object", "properties": { "codec": { "type": "string", "description": "The way in which the bytes of a data source should be converted into discrete messages, codecs are useful for specifying how large files or continuous streams of data might be processed in small chunks rather than loading it all in memory. It's possible to consume lines using a custom delimiter with the `delim:x` codec, where x is the character sequence custom delimiter. Codecs can be chained with `/`, for example a gzip compressed CSV file can be consumed with the codec `gzip/csv`.", "examples": [ "lines", "delim:\t", "delim:foobar", "gzip/csv" ] }, "container": { "type": "string", "description": "The name of the container from which to download blobs." }, "delete_objects": { "type": "boolean", "description": "Whether to delete downloaded objects from the blob once they are processed. Default: false" }, "max_buffer": { "type": "integer", "description": " Default: 1000000" }, "prefix": { "type": "string", "description": "An optional path prefix, if set only objects with the prefix are consumed. Default: " }, "scanner": { "type": "object", "description": "The [scanner](/docs/components/scanners/about) by which the stream of bytes consumed will be broken out into individual messages. Scanners are useful for processing large sources of data without holding the entirety of it within memory. For example, the `csv` scanner allows you to process individual CSV rows without loading the entire CSV file in memory at once. Default: map[to_the_end:map[]]" }, "storage_access_key": { "type": "string", "description": "The storage account access key. This field is ignored if `storage_connection_string` is set. Default: " }, "storage_account": { "type": "string", "description": "The storage account to access. This field is ignored if `storage_connection_string` is set. Default: " }, "storage_connection_string": { "type": "string", "description": "A storage account connection string. This field is required if `storage_account` and `storage_access_key` / `storage_sas_token` are not set. Default: " }, "storage_sas_token": { "type": "string", "description": "The storage account SAS token. This field is ignored if `storage_connection_string` or `storage_access_key` are set. Default: " }, "targets_input": { "type": "object", "description": "EXPERIMENTAL: An optional source of download targets, configured as a [regular Bento input](/docs/components/inputs/about). Each message yielded by this input should be a single structured object containing a field `name`, which represents the blob to be downloaded.", "examples": [ { "mqtt": { "topics": [ "some-topic" ], "urls": [ "example.westeurope-1.ts.eventgrid.azure.net:8883" ] }, "processors": [ { "unarchive": { "format": "json_array" } }, { "mapping": "if this.eventType == \"Microsoft.Storage.BlobCreated\" {\n root.name = this.data.url.parse_url().path.trim_prefix(\"/foocontainer/\")\n} else {\n root = deleted()\n}" } ] } ] } } }, "azure_cosmosdb": { "type": "object", "properties": { "account_key": { "type": "string", "description": "Account key.", "examples": [ "C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==" ] }, "args_mapping": { "type": "string", "description": "A [Bloblang mapping](/docs/guides/bloblang/about) that, for each message, creates a list of arguments to use with the query.", "examples": [ "root = [\n { \"Name\": \"@name\", \"Value\": \"bento\" },\n]" ] }, "auto_replay_nacks": { "type": "boolean", "description": "Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation. Default: true" }, "batch_count": { "type": "integer", "description": "The maximum number of messages that should be accumulated into each batch. Use '-1' specify dynamic page size. Default: -1" }, "connection_string": { "type": "string", "description": "Connection string.", "examples": [ "AccountEndpoint=https://localhost:8081/;AccountKey=C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==;" ] }, "container": { "type": "string", "description": "Container.", "examples": [ "testcontainer" ] }, "database": { "type": "string", "description": "Database.", "examples": [ "testdb" ] }, "endpoint": { "type": "string", "description": "CosmosDB endpoint.", "examples": [ "https://localhost:8081" ] }, "partition_keys_map": { "type": "string", "description": "A [Bloblang mapping](/docs/guides/bloblang/about) which should evaluate to a single partition key value or an array of partition key values of type string, integer or boolean. Currently, hierarchical partition keys are not supported so only one value may be provided.", "examples": [ "root = \"blobfish\"", "root = 41", "root = true", "root = null", "root = now().ts_format(\"2006-01-02\")" ] }, "query": { "type": "string", "description": "The query to execute", "examples": [ "SELECT c.foo FROM testcontainer AS c WHERE c.bar = \"baz\" AND c.timestamp \u003c @timestamp" ] } } }, "azure_queue_storage": { "type": "object", "properties": { "dequeue_visibility_timeout": { "type": "string", "description": "The timeout duration until a dequeued message gets visible again, 30s by default Default: 30s" }, "max_in_flight": { "type": "integer", "description": "The maximum number of unprocessed messages to fetch at a given time. Default: 10" }, "queue_name": { "type": "string", "description": "The name of the source storage queue.", "examples": [ "foo_queue", "${! env(\"MESSAGE_TYPE\").lowercase() }" ] }, "storage_access_key": { "type": "string", "description": "The storage account access key. This field is ignored if `storage_connection_string` is set. Default: " }, "storage_account": { "type": "string", "description": "The storage account to access. This field is ignored if `storage_connection_string` is set. Default: " }, "storage_connection_string": { "type": "string", "description": "A storage account connection string. This field is required if `storage_account` and `storage_access_key` / `storage_sas_token` are not set. Default: " }, "storage_sas_token": { "type": "string", "description": " Default: " }, "track_properties": { "type": "boolean", "description": "If set to `true` the queue is polled on each read request for information such as the queue message lag. These properties are added to consumed messages as metadata, but will also have a negative performance impact. Default: false" } } }, "azure_service_bus_queue": { "type": "object", "properties": { "auto_ack": { "type": "boolean", "description": "Whether to automatically acknowledge messages as they are consumed rather than waiting for acknowledgments from downstream. This can improve throughput but at the cost of eliminating delivery guarantees. Default: false" }, "connection_string": { "type": "string", "description": "The Service Bus connection string. This can be obtained from the Azure portal. If not provided, namespace and default credentials will be used. Default: ", "examples": [ "Endpoint=sb://example.servicebus.windows.net/;SharedAccessKeyName=RootManageSharedAccessKey;SharedAccessKey=..." ] }, "max_in_flight": { "type": "integer", "description": "The maximum number of unprocessed messages to fetch at a given time. Default: 10" }, "nack_reject_patterns": { "type": "array", "description": "A list of regular expression patterns whereby if a message that has failed to be delivered by Bento has an error that matches it will be nacked (or sent to dead letter queue if configured). By default failed messages are nacked with requeue enabled. Default: []", "items": { "type": "string" }, "examples": [ [ "^reject me please:.+$" ] ] }, "namespace": { "type": "string", "description": "The Service Bus namespace. Required when connection_string is not provided. Default: ", "examples": [ "myservicebus.servicebus.windows.net" ] }, "queue": { "type": "string", "description": "The name of the Service Bus queue to consume from." }, "renew_lock": { "type": "boolean", "description": "Automatically renew message locks to prevent lock expiration during processing. Useful for long-running message processing. Default: true" } } }, "azure_table_storage": { "type": "object", "properties": { "filter": { "type": "string", "description": "OData filter expression. Is not set all rows are returned. Valid operators are `eq, ne, gt, lt, ge and le` Default: ", "examples": [ "PartitionKey eq 'foo' and RowKey gt '1000'" ] }, "page_size": { "type": "integer", "description": "Maximum number of records to return on each page. Default: 1000" }, "select": { "type": "string", "description": "Select expression using OData notation. Limits the columns on each record to just those requested. Default: ", "examples": [ "PartitionKey,RowKey,Foo,Bar,Timestamp" ] }, "storage_access_key": { "type": "string", "description": "The storage account access key. This field is ignored if `storage_connection_string` is set. Default: " }, "storage_account": { "type": "string", "description": "The storage account to access. This field is ignored if `storage_connection_string` is set. Default: " }, "storage_connection_string": { "type": "string", "description": "A storage account connection string. This field is required if `storage_account` and `storage_access_key` / `storage_sas_token` are not set. Default: " }, "storage_sas_token": { "type": "string", "description": "The storage account SAS token. This field is ignored if `storage_connection_string` or `storage_access_key` are set. Default: " }, "table_name": { "type": "string", "description": "The table to read messages from.", "examples": [ "Foo" ] } } }, "batched": { "type": "object", "properties": { "child": { "type": "object", "description": "The child input." }, "policy": { "type": "object", "description": "\nAllows you to configure a [batching policy](/docs/configuration/batching).", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] } } }, "beanstalkd": { "type": "object", "properties": { "address": { "type": "string", "description": "An address to connect to.", "examples": [ "127.0.0.1:11300" ] } } }, "broker": { "type": "object", "properties": { "batching": { "type": "object", "description": "\nAllows you to configure a [batching policy](/docs/configuration/batching).", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] }, "copies": { "type": "integer", "description": "Whatever is specified within `inputs` will be created this many times. Default: 1" }, "inputs": { "type": "array", "description": "A list of inputs to create.", "items": { "type": "object" } } } }, "cassandra": { "type": "object", "properties": { "addresses": { "type": "array", "description": "A list of Cassandra nodes to connect to. Multiple comma separated addresses can be specified on a single line.", "items": { "type": "string" }, "examples": [ [ "localhost:9042" ], [ "foo:9042", "bar:9042" ], [ "foo:9042,bar:9042" ] ] }, "auto_replay_nacks": { "type": "boolean", "description": "Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation. Default: true" }, "backoff": { "type": "object", "description": "Control time intervals between retry attempts.", "properties": { "initial_interval": { "type": "string", "description": "The initial period to wait between retry attempts. Default: 1s" }, "max_interval": { "type": "string", "description": "The maximum period to wait between retry attempts. Default: 5s" } } }, "disable_initial_host_lookup": { "type": "boolean", "description": "If enabled the driver will not attempt to get host info from the system.peers table. This can speed up queries but will mean that data_centre, rack and token information will not be available. Default: false" }, "max_retries": { "type": "integer", "description": "The maximum number of retries before giving up on a request. Default: 3" }, "password_authenticator": { "type": "object", "description": "Optional configuration of Cassandra authentication parameters.", "properties": { "enabled": { "type": "boolean", "description": "Whether to use password authentication Default: false" }, "password": { "type": "string", "description": "The password to authenticate with. Default: " }, "username": { "type": "string", "description": "The username to authenticate as. Default: " } } }, "query": { "type": "string", "description": "A query to execute." }, "timeout": { "type": "string", "description": "The client connection timeout. Default: 600ms" }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } } } }, "cockroachdb_changefeed": { "type": "object", "properties": { "auto_replay_nacks": { "type": "boolean", "description": "Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation. Default: true" }, "cursor_cache": { "type": "string", "description": "A [cache resource](https://warpstreamlabs.github.io/bento/docs/components/caches/about) to use for storing the current latest cursor that has been successfully delivered, this allows Bento to continue from that cursor upon restart, rather than consume the entire state of the table." }, "dsn": { "type": "string", "description": "A Data Source Name to identify the target database.", "examples": [ "postgres://user:password@example.com:26257/defaultdb?sslmode=require" ] }, "options": { "type": "array", "description": "A list of options to be included in the changefeed (WITH X, Y...).\n**NOTE: Both the CURSOR option and UPDATED will be ignored from these options when a `cursor_cache` is specified, as they are set explicitly by Bento in this case.**", "items": { "type": "string" }, "examples": [ [ "virtual_columns=\"omitted\"" ] ] }, "tables": { "type": "array", "description": "CSV of tables to be included in the changefeed", "items": { "type": "string" }, "examples": [ [ "table1", "table2" ] ] }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } } } }, "csv": { "type": "object", "properties": { "auto_replay_nacks": { "type": "boolean", "description": "Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation. Default: true" }, "batch_count": { "type": "integer", "description": "Optionally process records in batches. This can help to speed up the consumption of exceptionally large CSV files. When the end of the file is reached the remaining records are processed as a (potentially smaller) batch. Default: 1" }, "delete_on_finish": { "type": "boolean", "description": "Whether to delete input files from the disk once they are fully consumed. Default: false" }, "delimiter": { "type": "string", "description": "The delimiter to use for splitting values in each record. It must be a single character. Default: ," }, "expected_headers": { "type": "array", "description": "An optional list of expected headers in the header row. If provided, the scanner will check the file contents and emit an error if any expected headers don't match.", "items": { "type": "string" }, "examples": [ [ "first_name", "last_name", "age" ] ] }, "expected_number_of_fields": { "type": "integer", "description": "The number of expected fields in the csv file." }, "lazy_quotes": { "type": "boolean", "description": "If set to `true`, a quote may appear in an unquoted field and a non-doubled quote may appear in a quoted field. Default: false" }, "parse_header_row": { "type": "boolean", "description": "Whether to reference the first row as a header row. If set to true the output structure for messages will be an object where field keys are determined by the header row. Otherwise, each message will consist of an array of values from the corresponding CSV row. Default: true" }, "paths": { "type": "array", "description": "A list of file paths to read from. Each file will be read sequentially until the list is exhausted, at which point the input will close. Glob patterns are supported, including super globs (double star).", "items": { "type": "string" }, "examples": [ [ "/tmp/foo.csv", "/tmp/bar/*.csv", "/tmp/data/**/*.csv" ] ] } } }, "cypher": { "type": "object", "properties": { "auto_replay_nacks": { "type": "boolean", "description": "Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation. Default: true" }, "basic_auth": { "type": "object", "description": "Basic Authentication fields", "properties": { "password": { "type": "string", "description": "The password for basic auth. Default: " }, "realm": { "type": "string", "description": "The realm for basic auth. Default: " }, "user": { "type": "string", "description": "The username for basic auth. Default: " } } }, "database": { "type": "string", "description": "The name of the database to connect to.", "examples": [ "neo4j" ] }, "no_auth": { "type": "boolean", "description": "Set to true to connect without authentication. Default: false" }, "query": { "type": "string", "description": "The cypher query to execute.", "examples": [ "MATCH (n) RETURN n" ] }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "uri": { "type": "string", "description": "The URL of the database engine.", "examples": [ "bolt://localhost:7687" ] } } }, "discord": { "type": "object", "properties": { "auto_replay_nacks": { "type": "boolean", "description": "Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation. Default: true" }, "bot_token": { "type": "string", "description": "A bot token used for authentication." }, "cache": { "type": "string", "description": "A cache resource to use for performing unread message backfills, the ID of the last message received will be stored in this cache and used for subsequent requests." }, "cache_key": { "type": "string", "description": "The key identifier used when storing the ID of the last message received. Default: last_message_id" }, "channel_id": { "type": "string", "description": "A discord channel ID to consume messages from." }, "limit": { "type": "integer", "description": "The maximum number of messages to receive in a single request. Default: 100" }, "poll_period": { "type": "string", "description": "The length of time (as a duration string) to wait between each poll for backlogged messages. This field can be set empty, in which case requests are made at the limit set by the rate limit. This field also supports cron expressions. Default: 1m" }, "rate_limit": { "type": "string", "description": " Default: An optional rate limit resource to restrict API requests with." } } }, "dynamic": { "type": "object", "properties": { "inputs": { "type": "object", "description": "A map of inputs to statically create. Default: map[]" }, "prefix": { "type": "string", "description": "A path prefix for HTTP endpoints that are registered. Default: " } } }, "etcd": { "type": "object", "properties": { "auth": { "type": "object", "description": "Optional configuration of etcd authentication headers.", "properties": { "enabled": { "type": "boolean", "description": "Whether to use password authentication Default: false" }, "password": { "type": "string", "description": "The password to authenticate with. Default: " }, "username": { "type": "string", "description": "The username to authenticate as. Default: " } } }, "auto_replay_nacks": { "type": "boolean", "description": "Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation. Default: true" }, "auto_sync_interval": { "type": "string", "description": "The interval to update endpoints with its latest members. 0 disables auto-sync. By default auto-sync is disabled." }, "backoff_jitter_fraction": { "type": "number", "description": "The jitter fraction to randomize backoff wait time." }, "backoff_wait_between": { "type": "string", "description": "The wait time before retrying an RPC." }, "dial_timeout": { "type": "string", "description": "Timeout for failing to establish a connection. Default: 5s" }, "endpoints": { "type": "array", "description": "A set of URLs (schemes, hosts and ports only) that can be used to communicate with a logical etcd cluster. If multiple endpoints are provided, the Client will attempt to use them all in the event that one or more of them are unusable.", "items": { "type": "string" }, "examples": [ [ "etcd://:2379" ], [ "etcd://localhost:2379" ], [ "etcd://localhost:2379", "etcd://localhost:22379", "etcd://localhost:32379" ] ] }, "keep_alive_time": { "type": "string", "description": "Time after which client pings the server to see if transport is alive. Default: 5s" }, "keep_alive_timeout": { "type": "string", "description": "Time that the client waits for a response for the keep-alive probe. If the response is not received in this time, the connection is closed. Default: 1s" }, "key": { "type": "string", "description": "The key or prefix being watched. For prefix watching, options.with_prefix should be `true`" }, "max_call_recv_msg_size": { "type": "integer", "description": "The client-side response receive limit. If 0, it defaults to math.MaxInt32." }, "max_call_send_msg_size": { "type": "integer", "description": "The client-side request send limit in bytes. If 0, it defaults to 2.0 MiB (2 * 1024 * 1024)." }, "max_unary_retries": { "type": "integer", "description": "The maximum number of retries for unary RPCs." }, "options": { "type": "object", "description": "Collection of options to configure an etcd watcher.", "properties": { "with_created_notify": { "type": "boolean", "description": "Whether to send CREATED notify events to the watcher. Default: false" }, "with_delete_filter": { "type": "boolean", "description": "Whether to discard DELETE events from the watcher. Default: false" }, "with_prefix": { "type": "boolean", "description": "Whether to watch for events on a prefix. Default: false" }, "with_progress_notify": { "type": "boolean", "description": "Whether to send periodic progress updates every 10 minutes when there is no incoming events. Default: false" }, "with_put_filter": { "type": "boolean", "description": "Whether to discard PUT events from the watcher. Default: false" }, "with_range": { "type": "string", "description": "Will cause the watcher to return a range of lexicographically sorted keys to return in the form `[key, end)` where `end` is the passed parameter. Default: " } } }, "permit_without_stream": { "type": "boolean", "description": "When set, will allow client to send keepalive pings to server without any active streams (RPCs). Default: false" }, "reject_old_cluster": { "type": "boolean", "description": "When set, will refuse to create a client against an outdated cluster. Default: false" }, "request_timeout": { "type": "string", "description": "Timeout for a single request. This includes connection time, any redirects, and header wait time. Default: 1s" }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } } } }, "file": { "type": "object", "properties": { "auto_replay_nacks": { "type": "boolean", "description": "Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation. Default: true" }, "codec": { "type": "string", "description": "The way in which the bytes of a data source should be converted into discrete messages, codecs are useful for specifying how large files or continuous streams of data might be processed in small chunks rather than loading it all in memory. It's possible to consume lines using a custom delimiter with the `delim:x` codec, where x is the character sequence custom delimiter. Codecs can be chained with `/`, for example a gzip compressed CSV file can be consumed with the codec `gzip/csv`.", "examples": [ "lines", "delim:\t", "delim:foobar", "gzip/csv" ] }, "delete_on_finish": { "type": "boolean", "description": "Whether to delete input files from the disk once they are fully consumed. Default: false" }, "max_buffer": { "type": "integer", "description": " Default: 1000000" }, "paths": { "type": "array", "description": "A list of paths to consume sequentially. Glob patterns are supported, including super globs (double star).", "items": { "type": "string" } }, "scanner": { "type": "object", "description": "The [scanner](/docs/components/scanners/about) by which the stream of bytes consumed will be broken out into individual messages. Scanners are useful for processing large sources of data without holding the entirety of it within memory. For example, the `csv` scanner allows you to process individual CSV rows without loading the entire CSV file in memory at once. Default: map[lines:map[]]" } } }, "gcp_bigquery_select": { "type": "object", "properties": { "args_mapping": { "type": "string", "description": "An optional [Bloblang mapping](/docs/guides/bloblang/about) which should evaluate to an array of values matching in size to the number of placeholder arguments in the field `where`.", "examples": [ "root = [ \"article\", now().ts_format(\"2006-01-02\") ]" ] }, "auto_replay_nacks": { "type": "boolean", "description": "Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation. Default: true" }, "columns": { "type": "array", "description": "A list of columns to query.", "items": { "type": "string" } }, "job_labels": { "type": "object", "description": "A list of labels to add to the query job. Default: map[]" }, "prefix": { "type": "string", "description": "An optional prefix to prepend to the select query (before SELECT)." }, "priority": { "type": "string", "description": "The priority with which to schedule the query. Default: " }, "project": { "type": "string", "description": "GCP project where the query job will execute." }, "suffix": { "type": "string", "description": "An optional suffix to append to the select query." }, "table": { "type": "string", "description": "Fully-qualified BigQuery table name to query.", "examples": [ "bigquery-public-data.samples.shakespeare" ] }, "where": { "type": "string", "description": "An optional where clause to add. Placeholder arguments are populated with the `args_mapping` field. Placeholders should always be question marks (`?`).", "examples": [ "type = ? and created_at \u003e ?", "user_id = ?" ] } } }, "gcp_cloud_storage": { "type": "object", "properties": { "bucket": { "type": "string", "description": "The name of the bucket from which to download objects." }, "codec": { "type": "string", "description": "The way in which the bytes of a data source should be converted into discrete messages, codecs are useful for specifying how large files or continuous streams of data might be processed in small chunks rather than loading it all in memory. It's possible to consume lines using a custom delimiter with the `delim:x` codec, where x is the character sequence custom delimiter. Codecs can be chained with `/`, for example a gzip compressed CSV file can be consumed with the codec `gzip/csv`.", "examples": [ "lines", "delim:\t", "delim:foobar", "gzip/csv" ] }, "delete_objects": { "type": "boolean", "description": "Whether to delete downloaded objects from the bucket once they are processed. Default: false" }, "max_buffer": { "type": "integer", "description": " Default: 1000000" }, "prefix": { "type": "string", "description": "An optional path prefix, if set only objects with the prefix are consumed. Default: " }, "scanner": { "type": "object", "description": "The [scanner](/docs/components/scanners/about) by which the stream of bytes consumed will be broken out into individual messages. Scanners are useful for processing large sources of data without holding the entirety of it within memory. For example, the `csv` scanner allows you to process individual CSV rows without loading the entire CSV file in memory at once. Default: map[to_the_end:map[]]" } } }, "gcp_pubsub": { "type": "object", "properties": { "create_subscription": { "type": "object", "description": "Allows you to configure the input subscription and creates if it doesn't exist.", "properties": { "enabled": { "type": "boolean", "description": "Whether to configure subscription or not. Default: false" }, "topic": { "type": "string", "description": "Defines the topic that the subscription should be vinculated to. Default: " } } }, "endpoint": { "type": "string", "description": "An optional endpoint to override the default of `pubsub.googleapis.com:443`. This can be used to connect to a region specific pubsub endpoint. For a list of valid values check out [this document.](https://cloud.google.com/pubsub/docs/reference/service_apis_overview#list_of_regional_endpoints) Default: ", "examples": [ "us-central1-pubsub.googleapis.com:443", "us-west3-pubsub.googleapis.com:443" ] }, "max_outstanding_bytes": { "type": "integer", "description": "The maximum number of outstanding pending messages to be consumed measured in bytes. Default: 1e+09" }, "max_outstanding_messages": { "type": "integer", "description": "The maximum number of outstanding pending messages to be consumed at a given time. Default: 1000" }, "project": { "type": "string", "description": "The project ID of the target subscription." }, "subscription": { "type": "string", "description": "The target subscription ID." }, "sync": { "type": "boolean", "description": "Enable synchronous pull mode. Default: false" } } }, "gcp_spanner_cdc": { "type": "object", "properties": { "end_time": { "type": "string", "description": "An optional field to define the end time to read from the changestreams, timestamp format should conform to RFC3339", "examples": [ "2006-01-02T15:04:05Z07:00" ] }, "heartbeat_interval": { "type": "string", "description": "An optional field to configure the heartbeat interval for partitions. Default: 3s" }, "prefetch_count": { "type": "integer", "description": "The maximum number of messages to have pulled in. Default: 1024", "examples": [ 1024 ] }, "spanner_dsn": { "type": "string", "description": "The dsn for spanner from where to read the changestream.", "examples": [ "projects/{projectId}/instances/{instanceId}/databases/{databaseName}" ] }, "start_time": { "type": "string", "description": "An optional field to define the start point to read from the changestreams. If not set then the current time is used. The timestamp format should conform to RFC3339, for details on valid start times please see [this document](https://cloud.google.com/spanner/docs/change-streams#data-retention)", "examples": [ "2006-01-02T15:04:05Z07:00" ] }, "stream_name": { "type": "string", "description": "The name of the stream to track changes on." } } }, "generate": { "type": "object", "properties": { "auto_replay_nacks": { "type": "boolean", "description": "Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation. Default: true" }, "batch_size": { "type": "integer", "description": "The number of generated messages that should be accumulated into each batch flushed at the specified interval. Default: 1" }, "count": { "type": "integer", "description": "An optional number of messages to generate, if set above 0 the specified number of messages is generated and then the input will shut down. Default: 0" }, "interval": { "type": "string", "description": "The time interval at which messages should be generated, expressed either as a duration string or as a cron expression. If set to an empty string messages will be generated as fast as downstream services can process them. Cron expressions can specify a timezone by prefixing the expression with `TZ=\u003clocation name\u003e`, where the location name corresponds to a file within the IANA Time Zone database. Default: 1s", "examples": [ "5s", "1m", "1h", "@every 1s", "0,30 */2 * * * *", "TZ=Europe/London 30 3-6,20-23 * * *" ] }, "mapping": { "type": "string", "description": "A [bloblang](/docs/guides/bloblang/about) mapping to use for generating messages.", "examples": [ "root = \"hello world\"", "root = {\"test\":\"message\",\"id\":uuid_v4()}" ] } } }, "hdfs": { "type": "object", "properties": { "directory": { "type": "string", "description": "The directory to consume from." }, "hosts": { "type": "array", "description": "A list of target host addresses to connect to.", "items": { "type": "string" }, "examples": [ "localhost:9000" ] }, "user": { "type": "string", "description": "A user ID to connect as. Default: " } } }, "http_client": { "type": "object", "properties": { "auto_replay_nacks": { "type": "boolean", "description": "Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation. Default: true" }, "backoff_on": { "type": "integer", "description": "A list of status codes whereby the request should be considered to have failed and retries should be attempted, but the period between them should be increased gradually. Default: [429]", "items": { "type": "integer" } }, "basic_auth": { "type": "object", "description": "Allows you to specify basic authentication.", "properties": { "enabled": { "type": "boolean", "description": "Whether to use basic authentication in requests. Default: false" }, "password": { "type": "string", "description": "A password to authenticate with. Default: " }, "username": { "type": "string", "description": "A username to authenticate as. Default: " } } }, "drop_empty_bodies": { "type": "boolean", "description": "Whether empty payloads received from the target server should be dropped. Default: true" }, "drop_on": { "type": "integer", "description": "A list of status codes whereby the request should be considered to have failed but retries should not be attempted. This is useful for preventing wasted retries for requests that will never succeed. Note that with these status codes the _request_ is dropped, but _message_ that caused the request will not be dropped. Default: []", "items": { "type": "integer" } }, "dump_request_log_level": { "type": "string", "description": "EXPERIMENTAL: Optionally set a level at which the request and response payload of each request made will be logged. Default: " }, "extract_headers": { "type": "object", "description": "Specify which response headers should be added to resulting messages as metadata. Header keys are lowercased before matching, so ensure that your patterns target lowercased versions of the header keys that you expect.", "properties": { "include_patterns": { "type": "array", "description": "Provide a list of explicit metadata key regular expression (re2) patterns to match against. Default: []", "items": { "type": "string" }, "examples": [ [ ".*" ], [ "_timestamp_unix$" ] ] }, "include_prefixes": { "type": "array", "description": "Provide a list of explicit metadata key prefixes to match against. Default: []", "items": { "type": "string" }, "examples": [ [ "foo_", "bar_" ], [ "kafka_" ], [ "content-" ] ] } } }, "headers": { "type": "object", "description": "A map of headers to add to the request. Default: map[]", "examples": [ { "Content-Type": "application/octet-stream", "traceparent": "${! tracing_span().traceparent }" } ] }, "jwt": { "type": "object", "description": "BETA: Allows you to specify JWT authentication.", "properties": { "claims": { "type": "object", "description": "A value used to identify the claims that issued the JWT. Default: map[]" }, "enabled": { "type": "boolean", "description": "Whether to use JWT authentication in requests. Default: false" }, "headers": { "type": "object", "description": "Add optional key/value headers to the JWT. Default: map[]" }, "private_key_file": { "type": "string", "description": "A file with the PEM encoded via PKCS1 or PKCS8 as private key. Default: " }, "signing_method": { "type": "string", "description": "A method used to sign the token such as RS256, RS384, RS512 or EdDSA. Default: " } } }, "max_retry_backoff": { "type": "string", "description": "The maximum period to wait between failed requests. Default: 300s" }, "metadata": { "type": "object", "description": "Specify optional matching rules to determine which metadata keys should be added to the HTTP request as headers.", "properties": { "include_patterns": { "type": "array", "description": "Provide a list of explicit metadata key regular expression (re2) patterns to match against. Default: []", "items": { "type": "string" }, "examples": [ [ ".*" ], [ "_timestamp_unix$" ] ] }, "include_prefixes": { "type": "array", "description": "Provide a list of explicit metadata key prefixes to match against. Default: []", "items": { "type": "string" }, "examples": [ [ "foo_", "bar_" ], [ "kafka_" ], [ "content-" ] ] } } }, "oauth": { "type": "object", "description": "Allows you to specify open authentication via OAuth version 1.", "properties": { "access_token": { "type": "string", "description": "A value used to gain access to the protected resources on behalf of the user. Default: " }, "access_token_secret": { "type": "string", "description": "A secret provided in order to establish ownership of a given access token. Default: " }, "consumer_key": { "type": "string", "description": "A value used to identify the client to the service provider. Default: " }, "consumer_secret": { "type": "string", "description": "A secret used to establish ownership of the consumer key. Default: " }, "enabled": { "type": "boolean", "description": "Whether to use OAuth version 1 in requests. Default: false" } } }, "oauth2": { "type": "object", "description": "Allows you to specify open authentication via OAuth version 2 using the client credentials token flow.", "properties": { "client_key": { "type": "string", "description": "A value used to identify the client to the token provider. Default: " }, "client_secret": { "type": "string", "description": "A secret used to establish ownership of the client key. Default: " }, "enabled": { "type": "boolean", "description": "Whether to use OAuth version 2 in requests. Default: false" }, "endpoint_params": { "type": "object", "description": "A list of optional endpoint parameters, values should be arrays of strings. Default: map[]", "examples": [ { "bar": [ "woof" ], "foo": [ "meow", "quack" ] } ] }, "scopes": { "type": "array", "description": "A list of optional requested permissions. Default: []", "items": { "type": "string" } }, "token_url": { "type": "string", "description": "The URL of the token provider. Default: " } } }, "payload": { "type": "string", "description": "An optional payload to deliver for each request." }, "proxy_url": { "type": "string", "description": "An optional HTTP proxy URL." }, "rate_limit": { "type": "string", "description": "An optional [rate limit](/docs/components/rate_limits/about) to throttle requests by." }, "retries": { "type": "integer", "description": "The maximum number of retry attempts to make. Default: 3" }, "retry_period": { "type": "string", "description": "The base period to wait between failed requests. Default: 1s" }, "stream": { "type": "object", "description": "Allows you to set streaming mode, where requests are kept open and messages are processed line-by-line.", "properties": { "codec": { "type": "string", "description": "The way in which the bytes of a data source should be converted into discrete messages, codecs are useful for specifying how large files or continuous streams of data might be processed in small chunks rather than loading it all in memory. It's possible to consume lines using a custom delimiter with the `delim:x` codec, where x is the character sequence custom delimiter. Codecs can be chained with `/`, for example a gzip compressed CSV file can be consumed with the codec `gzip/csv`.", "examples": [ "lines", "delim:\t", "delim:foobar", "gzip/csv" ] }, "enabled": { "type": "boolean", "description": "Enables streaming mode. Default: false" }, "max_buffer": { "type": "integer", "description": " Default: 1000000" }, "reconnect": { "type": "boolean", "description": "Sets whether to re-establish the connection once it is lost. Default: true" }, "scanner": { "type": "object", "description": "The [scanner](/docs/components/scanners/about) by which the stream of bytes consumed will be broken out into individual messages. Scanners are useful for processing large sources of data without holding the entirety of it within memory. For example, the `csv` scanner allows you to process individual CSV rows without loading the entire CSV file in memory at once. Default: map[lines:map[]]" } } }, "successful_on": { "type": "integer", "description": "A list of status codes whereby the attempt should be considered successful, this is useful for dropping requests that return non-2XX codes indicating that the message has been dealt with, such as a 303 See Other or a 409 Conflict. All 2XX codes are considered successful unless they are present within `backoff_on` or `drop_on`, regardless of this field. Default: []", "items": { "type": "integer" } }, "timeout": { "type": "string", "description": "A static timeout to apply to requests. Default: 5s" }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "transport": { "type": "object", "description": "Custom transport options.", "properties": { "dial_context": { "type": "object", "description": "Settings for the dialer used to create new connections.", "properties": { "keep_alive": { "type": "string", "description": "Keep-alive period for active network connections used by the dialer. Default: 30s" }, "timeout": { "type": "string", "description": "Timeout for establishing new network connections. Default: 30s" } } }, "expect_continue_timeout": { "type": "string", "description": "Time to wait for a server's first response headers after sending request headers when 'Expect: 100-continue' is used. Zero means send body immediately. Default: 1s" }, "force_http2": { "type": "boolean", "description": "If true, the transport will attempt to use HTTP/2. Default: true" }, "idle_connection_timeout": { "type": "string", "description": "Maximum time an idle keep-alive connection remains open before closing itself. Default: 90s" }, "max_idle_connections": { "type": "integer", "description": "Maximum number of idle keep-alive connections. Zero = unlimited. Default: 100" }, "tls_handshake_timeout": { "type": "string", "description": "Maximum time allowed for TLS handshake to complete. Default: 10s" } } }, "url": { "type": "string", "description": "The URL to connect to." }, "verb": { "type": "string", "description": "A verb to connect with Default: GET", "examples": [ "POST", "GET", "DELETE" ] } } }, "http_server": { "type": "object", "properties": { "address": { "type": "string", "description": "An alternative address to host from. If left empty the service wide address is used. Default: " }, "allowed_verbs": { "type": "array", "description": "An array of verbs that are allowed for the `path` endpoint. Default: [POST]", "items": { "type": "string" } }, "cert_file": { "type": "string", "description": "Enable TLS by specifying a certificate and key file. Only valid with a custom `address`. Default: " }, "cors": { "type": "object", "description": "Adds Cross-Origin Resource Sharing headers. Only valid with a custom `address`.", "properties": { "allowed_headers": { "type": "array", "description": "Appends additional headers to the list of default allowed headers: Accept, Accept-Language, Content-Language \u0026 Origin. These default headers are therefore always allowed. Default: []", "items": { "type": "string" } }, "allowed_methods": { "type": "array", "description": "Used to explicitly set allowed methods in the Access-Control-Allow-Methods header. Default: [GET HEAD POST PUT PATCH DELETE]", "items": { "type": "string" } }, "allowed_origins": { "type": "array", "description": "An explicit list of origins that are allowed for CORS requests. Default: []", "items": { "type": "string" } }, "enabled": { "type": "boolean", "description": "Whether to allow CORS requests. Default: false" } } }, "key_file": { "type": "string", "description": "Enable TLS by specifying a certificate and key file. Only valid with a custom `address`. Default: " }, "path": { "type": "string", "description": "The endpoint path to listen for POST requests. Default: /post" }, "rate_limit": { "type": "string", "description": "An optional [rate limit](/docs/components/rate_limits/about) to throttle requests by. Default: " }, "sync_response": { "type": "object", "description": "Customise messages returned via [synchronous responses](/docs/guides/sync_responses).", "properties": { "headers": { "type": "object", "description": "Specify headers to return with synchronous responses. Default: map[Content-Type:application/octet-stream]" }, "metadata_headers": { "type": "object", "description": "Specify criteria for which metadata values are added to the response as headers.", "properties": { "include_patterns": { "type": "array", "description": "Provide a list of explicit metadata key regular expression (re2) patterns to match against. Default: []", "items": { "type": "string" }, "examples": [ [ ".*" ], [ "_timestamp_unix$" ] ] }, "include_prefixes": { "type": "array", "description": "Provide a list of explicit metadata key prefixes to match against. Default: []", "items": { "type": "string" }, "examples": [ [ "foo_", "bar_" ], [ "kafka_" ], [ "content-" ] ] } } }, "status": { "type": "string", "description": "Specify the status code to return with synchronous responses. This is a string value, which allows you to customize it based on resulting payloads and their metadata. Default: 200", "examples": [ "${! json(\"status\") }", "${! metadata(\"status\").string() }" ] } } }, "timeout": { "type": "string", "description": "Timeout for requests. If a consumed messages takes longer than this to be delivered the connection is closed, but the message may still be delivered. Default: 5s" }, "ws_path": { "type": "string", "description": "The endpoint path to create websocket connections from. Default: /post/ws" }, "ws_rate_limit_message": { "type": "string", "description": "An optional message to delivery to websocket connections that are rate limited. Default: " }, "ws_welcome_message": { "type": "string", "description": "An optional message to deliver to fresh websocket connections. Default: " } } }, "inproc": { "type": "string", "description": " Default: " }, "kafka": { "type": "object", "properties": { "addresses": { "type": "array", "description": "A list of broker addresses to connect to. If an item of the list contains commas it will be expanded into multiple addresses.", "items": { "type": "string" }, "examples": [ [ "localhost:9092" ], [ "localhost:9041,localhost:9042" ], [ "localhost:9041", "localhost:9042" ] ] }, "auto_replay_nacks": { "type": "boolean", "description": "Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation. Default: true" }, "batching": { "type": "object", "description": "\nAllows you to configure a [batching policy](/docs/configuration/batching).", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] }, "checkpoint_limit": { "type": "integer", "description": "The maximum number of messages of the same topic and partition that can be processed at a given time. Increasing this limit enables parallel processing and batching at the output level to work on individual partitions. Any given offset will not be committed unless all messages under that offset are delivered in order to preserve at least once delivery guarantees. Default: 1024" }, "client_id": { "type": "string", "description": "An identifier for the client connection. Default: bento" }, "commit_period": { "type": "string", "description": "The period of time between each commit of the current partition offsets. Offsets are always committed during shutdown. Default: 1s" }, "consumer_group": { "type": "string", "description": "An identifier for the consumer group of the connection. This field can be explicitly made empty in order to disable stored offsets for the consumed topic partitions. Default: " }, "extract_tracing_map": { "type": "string", "description": "EXPERIMENTAL: A [Bloblang mapping](/docs/guides/bloblang/about) that attempts to extract an object containing tracing propagation information, which will then be used as the root tracing span for the message. The specification of the extracted fields must match the format used by the service wide tracer.", "examples": [ "root = @", "root = this.meta.span" ] }, "fetch_buffer_cap": { "type": "integer", "description": "The maximum number of unprocessed messages to fetch at a given time. Default: 256" }, "group": { "type": "object", "description": "Tuning parameters for consumer group synchronization.", "properties": { "heartbeat_interval": { "type": "string", "description": "A period in which heartbeats should be sent out. Default: 3s" }, "rebalance_timeout": { "type": "string", "description": "A period after which rebalancing is abandoned if unresolved. Default: 60s" }, "session_timeout": { "type": "string", "description": "A period after which a consumer of the group is kicked after no heartbeats. Default: 10s" } } }, "max_processing_period": { "type": "string", "description": "A maximum estimate for the time taken to process a message, this is used for tuning consumer group synchronization. Default: 100ms" }, "multi_header": { "type": "boolean", "description": "Decode headers into lists to allow handling of multiple values with the same key Default: false" }, "new_root_span_with_link": { "type": "boolean", "description": "EXPERIMENTAL: Starts a new root span with link to parent." }, "rack_id": { "type": "string", "description": "A rack identifier for this client. Default: " }, "sasl": { "type": "object", "description": "Enables SASL authentication.", "properties": { "access_token": { "type": "string", "description": "A static OAUTHBEARER access token Default: " }, "aws": { "type": "object", "description": "Contains AWS specific fields for when the `mechanism` is set to `AWS_MSK_IAM`.", "properties": { "credentials": { "type": "object", "description": "Optional manual configuration of AWS credentials to use. More information can be found [in this document](/docs/guides/cloud/aws).", "properties": { "expiry_window": { "type": "string", "description": "Allow the credentials to trigger refreshing prior to the credentials actually expiring. This is beneficial so race conditions with expiring credentials do not cause requests to fail. For example '10s' would refresh credentials ten seconds before expiration. Setting to a duration of `0` disables the expiry window. Default: " }, "from_ec2_role": { "type": "boolean", "description": "Use the credentials of a host EC2 machine configured to assume [an IAM role associated with the instance](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html). Default: false" }, "id": { "type": "string", "description": "The ID of credentials to use. Default: " }, "profile": { "type": "string", "description": "A profile from `~/.aws/credentials` to use. Default: " }, "role": { "type": "string", "description": "A role ARN to assume. Default: " }, "role_external_id": { "type": "string", "description": "An external ID to provide when assuming a role. Default: " }, "secret": { "type": "string", "description": "The secret for the credentials being used. Default: " }, "token": { "type": "string", "description": "The token for the credentials being used, required when using short term credentials. Default: " } } }, "endpoint": { "type": "string", "description": "Allows you to specify a custom endpoint for the AWS API. Default: " }, "region": { "type": "string", "description": "The AWS region to target. Default: " } } }, "mechanism": { "type": "string", "description": "The SASL authentication mechanism, if left empty SASL authentication is not used. Default: none" }, "password": { "type": "string", "description": "A PLAIN password. It is recommended that you use environment variables to populate this field. Default: ", "examples": [ "${PASSWORD}" ] }, "token_cache": { "type": "string", "description": "Instead of using a static `access_token` allows you to query a [`cache`](/docs/components/caches/about) resource to fetch OAUTHBEARER tokens from Default: " }, "token_key": { "type": "string", "description": "Required when using a `token_cache`, the key to query the cache with for tokens. Default: " }, "user": { "type": "string", "description": "A PLAIN username. It is recommended that you use environment variables to populate this field. Default: ", "examples": [ "${USER}" ] } } }, "start_from_oldest": { "type": "boolean", "description": "Determines whether to consume from the oldest available offset, otherwise messages are consumed from the latest offset. The setting is applied when creating a new consumer group or the saved offset no longer exists. Default: true" }, "target_version": { "type": "string", "description": "The version of the Kafka protocol to use. This limits the capabilities used by the client and should ideally match the version of your brokers. Defaults to the oldest supported stable version.", "examples": [ "2.1.0", "3.1.0" ] }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "topics": { "type": "array", "description": "A list of topics to consume from. Multiple comma separated topics can be listed in a single element. Partitions are automatically distributed across consumers of a topic. Alternatively, it's possible to specify explicit partitions to consume from with a colon after the topic name, e.g. `foo:0` would consume the partition 0 of the topic foo. This syntax supports ranges, e.g. `foo:0-10` would consume partitions 0 through to 10 inclusive.", "items": { "type": "string" }, "examples": [ [ "foo", "bar" ], [ "foo,bar" ], [ "foo:0", "bar:1", "bar:3" ], [ "foo:0,bar:1,bar:3" ], [ "foo:0-5" ] ] } } }, "kafka_franz": { "type": "object", "properties": { "auto_offset_reset": { "type": "string", "description": "Determines which offset to automatically consume from, matching Kafka's `auto.offset.reset` property. When specified, this takes precedence over `start_from_oldest`." }, "auto_replay_nacks": { "type": "boolean", "description": "Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation. Default: true" }, "batching": { "type": "object", "description": "Allows you to configure a [batching policy](/docs/configuration/batching) that applies to individual topic partitions in order to batch messages together before flushing them for processing. Batching can be beneficial for performance as well as useful for windowed processing, and doing so this way preserves the ordering of topic partitions.", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] }, "checkpoint_limit": { "type": "integer", "description": ":::caution \n\t\t\tSetting this `checkpoint_limit: 1`_will not_ enforce 'strict ordered' processing of records. Use the [kafka input processor](/docs/components/inputs/kafka/) for 'strict ordered' processing.\n:::\n\t\t\t\n\t\t\tDetermines how many messages of the same partition can be processed in parallel before applying back pressure. When a message of a given offset is delivered to the output the offset is only allowed to be committed when all messages of prior offsets have also been delivered, this ensures at-least-once delivery guarantees. However, this mechanism also increases the likelihood of duplicates in the event of crashes or server faults, reducing the checkpoint limit will mitigate this. Default: 1024" }, "client_id": { "type": "string", "description": "An identifier for the client connection. Default: bento" }, "commit_period": { "type": "string", "description": "The period of time between each commit of the current partition offsets. Offsets are always committed during shutdown. Default: 5s" }, "consumer_group": { "type": "string", "description": "An optional consumer group to consume as. When specified the partitions of specified topics are automatically distributed across consumers sharing a consumer group, and partition offsets are automatically committed and resumed under this name. Consumer groups are not supported when specifying explicit partitions to consume from in the `topics` field." }, "fetch_max_bytes": { "type": "string", "description": "This sets the maximum amount of bytes a broker will try to send during a fetch. Note that brokers may not obey this limit if it has records larger than this limit. Also note that this client sends a fetch to each broker concurrently, meaning the client will buffer up to `\u003cbrokers * max bytes\u003e` worth of memory. Equivalent to Kafka's `fetch.max.bytes` option. Default: 50MiB" }, "fetch_max_partition_bytes": { "type": "string", "description": "Sets the maximum amount of bytes that will be consumed for a single partition in a fetch request. Note that if a single batch is larger than this number, that batch will still be returned so the client can make progress. Equivalent to Kafka's `max.partition.fetch.bytes` option. Default: 1MiB" }, "fetch_max_wait": { "type": "string", "description": "This sets the maximum amount of time a broker will wait for a fetch response to hit the minimum number of required bytes before returning, overriding the default 5s. Default: 5s" }, "group_balancers": { "type": "array", "description": "Balancers sets the group balancers to use for dividing topic partitions among group members. This option is equivalent to Kafka's `partition.assignment.strategies` option. Default: [cooperative_sticky]", "items": { "type": "string" } }, "metadata_max_age": { "type": "string", "description": "This sets the maximum age for the client's cached metadata, to allow detection of new topics, partitions, etc. Default: 5m" }, "multi_header": { "type": "boolean", "description": "Decode headers into lists to allow handling of multiple values with the same key Default: false" }, "preferring_lag": { "type": "integer", "description": "\nThis allows you to re-order partitions before they are fetched, given each partition's current lag.\n\nBy default, the client rotates partitions fetched by one after every fetch request. Kafka answers fetch requests in the order that partitions are requested, filling the fetch response until`fetch_max_bytes` and `fetch_max_partition_bytes` are hit. All partitions eventually rotate to the front, ensuring no partition is starved.\n\nWith this option, you can return topic order and per-topic partition ordering. These orders will sort to the front (first by topic, then by partition). Any topic or partitions that you do not return are added to the end, preserving their original ordering." }, "rack_id": { "type": "string", "description": "A rack identifier for this client. Default: " }, "rate_limit": { "type": "string", "description": "An optional [`rate_limit`](/docs/components/rate_limits/about) to throttle invocations by. Default: " }, "reconnect_on_unknown_topic_or_partition": { "type": "boolean", "description": "Determines whether to close the client and force a reconnect after seeing an UNKNOWN_TOPIC_OR_PARTITION or UNKNOWN_TOPIC_ID error. Default: false" }, "regexp_topics": { "type": "boolean", "description": "Whether listed topics should be interpreted as regular expression patterns for matching multiple topics. When topics are specified with explicit partitions this field must remain set to `false`. Default: false" }, "sasl": { "type": "array", "description": "Specify one or more methods of SASL authentication. SASL is tried in order; if the broker supports the first mechanism, all connections will use that mechanism. If the first mechanism fails, the client will pick the first supported mechanism. If the broker does not support any client mechanisms, connections will fail.", "items": { "type": "object" }, "examples": [ [ { "mechanism": "SCRAM-SHA-512", "password": "bar", "username": "foo" } ] ] }, "seed_brokers": { "type": "array", "description": "A list of broker addresses to connect to in order to establish connections. If an item of the list contains commas it will be expanded into multiple addresses.", "items": { "type": "string" }, "examples": [ [ "localhost:9092" ], [ "foo:9092", "bar:9092" ], [ "foo:9092,bar:9092" ] ] }, "start_from_oldest": { "type": "boolean", "description": "Determines whether to consume from the oldest available offset, otherwise messages are consumed from the latest offset. The setting is applied when creating a new consumer group or the saved offset no longer exists. Default: true" }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "topics": { "type": "array", "description": "\nA list of topics to consume from. Multiple comma separated topics can be listed in a single element. When a `consumer_group` is specified partitions are automatically distributed across consumers of a topic, otherwise all partitions are consumed.\n\nAlternatively, it's possible to specify explicit partitions to consume from with a colon after the topic name, e.g. `foo:0` would consume the partition 0 of the topic foo. This syntax supports ranges, e.g. `foo:0-10` would consume partitions 0 through to 10 inclusive.\n\nFinally, it's also possible to specify an explicit offset to consume from by adding another colon after the partition, e.g. `foo:0:10` would consume the partition 0 of the topic foo starting from the offset 10. If the offset is not present (or remains unspecified) then the field `start_from_oldest` determines which offset to start from.", "items": { "type": "string" }, "examples": [ [ "foo", "bar" ], [ "things.*" ], [ "foo,bar" ], [ "foo:0", "bar:1", "bar:3" ], [ "foo:0,bar:1,bar:3" ], [ "foo:0-5" ] ] } } }, "kubernetes_watch": { "type": "object", "properties": { "api_server": { "type": "string", "description": "Kubernetes API server URL. Only used when providing explicit credentials. Default: " }, "auto_auth": { "type": "boolean", "description": "Automatically detect authentication method. Tries in-cluster config first, then falls back to kubeconfig. Default: true" }, "backoff": { "type": "object", "description": "Control time intervals between retry attempts.", "properties": { "initial_interval": { "type": "string", "description": "The initial period to wait between retry attempts. Default: 1s" }, "max_elapsed_time": { "type": "string", "description": "The maximum period to wait before retry attempts are abandoned. If zero then no limit is used. Default: 0s" }, "max_interval": { "type": "string", "description": "The maximum period to wait between retry attempts. Default: 60s" } } }, "ca_file": { "type": "string", "description": "Path to CA certificate file for verifying API server. Default: " }, "client_burst": { "type": "integer", "description": "Burst limit for Kubernetes API client. 0 uses the client-go default. Default: 0" }, "client_qps": { "type": "number", "description": "QPS limit for Kubernetes API client. 0 uses the client-go default. Default: 0" }, "context": { "type": "string", "description": "Kubernetes context to use from kubeconfig. If empty, uses the current context. Default: " }, "custom_resource": { "type": "object", "description": "Custom Resource Definition to watch. Use this for CRDs instead of 'resource'.", "properties": { "group": { "type": "string", "description": "API group for the custom resource (e.g., 'stable.example.com'). Default: " }, "resource": { "type": "string", "description": "Plural name of the custom resource (e.g., 'crontabs'). Default: " }, "version": { "type": "string", "description": "API version for the custom resource (e.g., 'v1', 'v1beta1'). Default: v1" } } }, "event_types": { "type": "array", "description": "Watch event types to include. Valid values are `ADDED`, `MODIFIED`, and `DELETED`. Default: [ADDED MODIFIED DELETED]", "items": { "type": "string" }, "examples": [ [ "ADDED", "DELETED" ], [ "MODIFIED" ] ] }, "field_selector": { "type": "object", "description": "Kubernetes field selector to filter resources. Default: map[]", "examples": [ { "status.phase": "Running" }, { "metadata.name": "my-pod" } ] }, "include_initial_list": { "type": "boolean", "description": "Emit ADDED events for all existing resources when starting. Default: true" }, "insecure_skip_verify": { "type": "boolean", "description": "Skip TLS certificate verification. Not recommended for production. Default: false" }, "kubeconfig": { "type": "string", "description": "Path to kubeconfig file. If empty, uses $KUBECONFIG (if set) or the default kubeconfig location (~/.kube/config). Default: " }, "kubeconfig_yaml": { "type": "string", "description": "Kubeconfig content as a YAML string. Use this to embed kubeconfig directly in your config (e.g., from a secret or environment variable) instead of referencing a file path. Default: " }, "label_selector": { "type": "object", "description": "Kubernetes label selector to filter resources. Default: map[]", "examples": [ { "app": "myapp" }, { "app": "myapp", "env": "prod" } ] }, "max_retries": { "type": "integer", "description": "The maximum number of retries before giving up on the request. If set to zero there is no discrete limit. Default: 0" }, "namespaces": { "type": "array", "description": "Namespaces to watch. Empty list means all namespaces. Default: []", "items": { "type": "string" }, "examples": [ [ "default" ], [ "production", "staging" ] ] }, "request_timeout": { "type": "string", "description": "Timeout for Kubernetes API requests such as list calls. Use \"0s\" to disable. Default: 30s" }, "resource": { "type": "string", "description": "Standard Kubernetes resource type to watch. Default: ", "examples": [ "pods", "deployments", "configmaps" ] }, "token": { "type": "string", "description": "Bearer token for authentication. Can be a service account token. Default: " }, "token_file": { "type": "string", "description": "Path to file containing bearer token. Default: " } } }, "mongodb": { "type": "object", "properties": { "auto_replay_nacks": { "type": "boolean", "description": "Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation. Default: true" }, "batch_size": { "type": "integer", "description": "A explicit number of documents to batch up before flushing them for processing. Must be greater than `0`. Operations: `find`, `aggregate`", "examples": [ 1000 ] }, "collection": { "type": "string", "description": "The collection to select from." }, "database": { "type": "string", "description": "The name of the target MongoDB database." }, "json_marshal_mode": { "type": "string", "description": "The json_marshal_mode setting is optional and controls the format of the output message. Default: canonical" }, "limit": { "type": "integer", "description": "An explicit maximum number of documents to return. Operations: `find`" }, "operation": { "type": "string", "description": "The mongodb operation to perform. Default: find" }, "password": { "type": "string", "description": "The password to connect to the database. Default: " }, "query": { "type": "string", "description": "Bloblang expression describing MongoDB query.", "examples": [ "\n root.from = {\"$lte\": timestamp_unix()}\n root.to = {\"$gte\": timestamp_unix()}\n" ] }, "sort": { "type": "object", "description": "An object specifying fields to sort by, and the respective sort order (`1` ascending, `-1` descending). Note: The driver currently appears to support only one sorting key. Operations: `find`", "examples": [ { "name": 1 }, { "age": -1 } ] }, "url": { "type": "string", "description": "The URL of the target MongoDB server.", "examples": [ "mongodb://localhost:27017" ] }, "username": { "type": "string", "description": "The username to connect to the database. Default: " } } }, "mqtt": { "type": "object", "properties": { "auto_replay_nacks": { "type": "boolean", "description": "Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation. Default: true" }, "clean_session": { "type": "boolean", "description": "Set whether the connection is non-persistent. Default: true" }, "client_id": { "type": "string", "description": "An identifier for the client connection. Default: " }, "connect_timeout": { "type": "string", "description": "The maximum amount of time to wait in order to establish a connection before the attempt is abandoned. Default: 30s", "examples": [ "1s", "500ms" ] }, "dynamic_client_id_suffix": { "type": "string", "description": "Append a dynamically generated suffix to the specified `client_id` on each run of the pipeline. This can be useful when clustering Bento producers." }, "keepalive": { "type": "integer", "description": "Max seconds of inactivity before a keepalive message is sent. Default: 30" }, "password": { "type": "string", "description": "A password to connect with. Default: " }, "qos": { "type": "integer", "description": "The level of delivery guarantee to enforce. Has options 0, 1, 2. Default: 1" }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "topics": { "type": "array", "description": "A list of topics to consume from.", "items": { "type": "string" } }, "urls": { "type": "array", "description": "A list of URLs to connect to. If an item of the list contains commas it will be expanded into multiple URLs.", "items": { "type": "string" }, "examples": [ [ "tcp://localhost:1883" ] ] }, "user": { "type": "string", "description": "A username to connect with. Default: " }, "will": { "type": "object", "description": "Set last will message in case of Bento failure", "properties": { "enabled": { "type": "boolean", "description": "Whether to enable last will messages. Default: false" }, "payload": { "type": "string", "description": "Set payload for last will message. Default: " }, "qos": { "type": "integer", "description": "Set QoS for last will message. Valid values are: 0, 1, 2. Default: 0" }, "retained": { "type": "boolean", "description": "Set retained for last will message. Default: false" }, "topic": { "type": "string", "description": "Set topic for last will message. Default: " } } } } }, "nanomsg": { "type": "object", "properties": { "auto_replay_nacks": { "type": "boolean", "description": "Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation. Default: true" }, "bind": { "type": "boolean", "description": "Whether the URLs provided should be connected to, or bound as. Default: true" }, "poll_timeout": { "type": "string", "description": "The period to wait until a poll is abandoned and reattempted. Default: 5s" }, "socket_type": { "type": "string", "description": "The socket type to use. Default: PULL" }, "sub_filters": { "type": "array", "description": "A list of subscription topic filters to use when consuming from a SUB socket. Specifying a single sub_filter of `''` will subscribe to everything. Default: []", "items": { "type": "string" } }, "urls": { "type": "array", "description": "A list of URLs to connect to (or as). If an item of the list contains commas it will be expanded into multiple URLs.", "items": { "type": "string" } } } }, "nats": { "type": "object", "properties": { "auth": { "type": "object", "description": "Optional configuration of NATS authentication parameters.", "properties": { "nkey_file": { "type": "string", "description": "An optional file containing a NKey seed.", "examples": [ "./seed.nk" ] }, "user_credentials_file": { "type": "string", "description": "An optional file containing user credentials which consist of an user JWT and corresponding NKey seed.", "examples": [ "./user.creds" ] }, "user_jwt": { "type": "string", "description": "An optional plain text user JWT (given along with the corresponding user NKey Seed)." }, "user_nkey_seed": { "type": "string", "description": "An optional plain text user NKey Seed (given along with the corresponding user JWT)." } } }, "auto_replay_nacks": { "type": "boolean", "description": "Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation. Default: true" }, "extract_tracing_map": { "type": "string", "description": "EXPERIMENTAL: A [Bloblang mapping](/docs/guides/bloblang/about) that attempts to extract an object containing tracing propagation information, which will then be used as the root tracing span for the message. The specification of the extracted fields must match the format used by the service wide tracer.", "examples": [ "root = @", "root = this.meta.span" ] }, "nak_delay": { "type": "string", "description": "An optional delay duration on redelivering a message when negatively acknowledged.", "examples": [ "1m" ] }, "new_root_span_with_link": { "type": "boolean", "description": "EXPERIMENTAL: Starts a new root span with link to parent." }, "prefetch_count": { "type": "integer", "description": "The maximum number of messages to pull at a time. Default: 524288" }, "queue": { "type": "string", "description": "An optional queue group to consume as." }, "subject": { "type": "string", "description": "A subject to consume from. Supports wildcards for consuming multiple subjects. Either a subject or stream must be specified.", "examples": [ "foo.bar.baz", "foo.*.baz", "foo.bar.*", "foo.\u003e" ] }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "urls": { "type": "array", "description": "A list of URLs to connect to. If an item of the list contains commas it will be expanded into multiple URLs.", "items": { "type": "string" }, "examples": [ [ "nats://127.0.0.1:4222" ], [ "nats://username:password@127.0.0.1:4222" ] ] } } }, "nats_jetstream": { "type": "object", "properties": { "ack_wait": { "type": "string", "description": "The maximum amount of time NATS server should wait for an ack from consumer. Default: 30s", "examples": [ "100ms", "5m" ] }, "auth": { "type": "object", "description": "Optional configuration of NATS authentication parameters.", "properties": { "nkey_file": { "type": "string", "description": "An optional file containing a NKey seed.", "examples": [ "./seed.nk" ] }, "user_credentials_file": { "type": "string", "description": "An optional file containing user credentials which consist of an user JWT and corresponding NKey seed.", "examples": [ "./user.creds" ] }, "user_jwt": { "type": "string", "description": "An optional plain text user JWT (given along with the corresponding user NKey Seed)." }, "user_nkey_seed": { "type": "string", "description": "An optional plain text user NKey Seed (given along with the corresponding user JWT)." } } }, "bind": { "type": "boolean", "description": "Indicates that the subscription should use an existing consumer." }, "deliver": { "type": "string", "description": "Determines which messages to deliver when consuming without a durable subscriber. Default: all" }, "durable": { "type": "string", "description": "Preserve the state of your consumer under a durable name." }, "extract_tracing_map": { "type": "string", "description": "EXPERIMENTAL: A [Bloblang mapping](/docs/guides/bloblang/about) that attempts to extract an object containing tracing propagation information, which will then be used as the root tracing span for the message. The specification of the extracted fields must match the format used by the service wide tracer.", "examples": [ "root = @", "root = this.meta.span" ] }, "max_ack_pending": { "type": "integer", "description": "The maximum number of outstanding acks to be allowed before consuming is halted. Default: 1024" }, "new_root_span_with_link": { "type": "boolean", "description": "EXPERIMENTAL: Starts a new root span with link to parent." }, "queue": { "type": "string", "description": "An optional queue group to consume as." }, "stream": { "type": "string", "description": "A stream to consume from. Either a subject or stream must be specified." }, "subject": { "type": "string", "description": "A subject to consume from. Supports wildcards for consuming multiple subjects. Either a subject or stream must be specified.", "examples": [ "foo.bar.baz", "foo.*.baz", "foo.bar.*", "foo.\u003e" ] }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "urls": { "type": "array", "description": "A list of URLs to connect to. If an item of the list contains commas it will be expanded into multiple URLs.", "items": { "type": "string" }, "examples": [ [ "nats://127.0.0.1:4222" ], [ "nats://username:password@127.0.0.1:4222" ] ] } } }, "nats_kv": { "type": "object", "properties": { "auth": { "type": "object", "description": "Optional configuration of NATS authentication parameters.", "properties": { "nkey_file": { "type": "string", "description": "An optional file containing a NKey seed.", "examples": [ "./seed.nk" ] }, "user_credentials_file": { "type": "string", "description": "An optional file containing user credentials which consist of an user JWT and corresponding NKey seed.", "examples": [ "./user.creds" ] }, "user_jwt": { "type": "string", "description": "An optional plain text user JWT (given along with the corresponding user NKey Seed)." }, "user_nkey_seed": { "type": "string", "description": "An optional plain text user NKey Seed (given along with the corresponding user JWT)." } } }, "auto_replay_nacks": { "type": "boolean", "description": "Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation. Default: true" }, "bucket": { "type": "string", "description": "The name of the KV bucket.", "examples": [ "my_kv_bucket" ] }, "ignore_deletes": { "type": "boolean", "description": "Do not send delete markers as messages. Default: false" }, "include_history": { "type": "boolean", "description": "Include all the history per key, not just the last one. Default: false" }, "key": { "type": "string", "description": "Key to watch for updates, can include wildcards. Default: \u003e", "examples": [ "foo.bar.baz", "foo.*.baz", "foo.bar.*", "foo.\u003e" ] }, "meta_only": { "type": "boolean", "description": "Retrieve only the metadata of the entry Default: false" }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "urls": { "type": "array", "description": "A list of URLs to connect to. If an item of the list contains commas it will be expanded into multiple URLs.", "items": { "type": "string" }, "examples": [ [ "nats://127.0.0.1:4222" ], [ "nats://username:password@127.0.0.1:4222" ] ] } } }, "nats_object_store": { "type": "object", "properties": { "auth": { "type": "object", "description": "Optional configuration of NATS authentication parameters.", "properties": { "nkey_file": { "type": "string", "description": "An optional file containing a NKey seed.", "examples": [ "./seed.nk" ] }, "user_credentials_file": { "type": "string", "description": "An optional file containing user credentials which consist of an user JWT and corresponding NKey seed.", "examples": [ "./user.creds" ] }, "user_jwt": { "type": "string", "description": "An optional plain text user JWT (given along with the corresponding user NKey Seed)." }, "user_nkey_seed": { "type": "string", "description": "An optional plain text user NKey Seed (given along with the corresponding user JWT)." } } }, "auto_replay_nacks": { "type": "boolean", "description": "Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation. Default: true" }, "bucket": { "type": "string", "description": "The name of the object store bucket.", "examples": [ "my_bucket" ] }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "urls": { "type": "array", "description": "A list of URLs to connect to. If an item of the list contains commas it will be expanded into multiple URLs.", "items": { "type": "string" }, "examples": [ [ "nats://127.0.0.1:4222" ], [ "nats://username:password@127.0.0.1:4222" ] ] } } }, "nats_stream": { "type": "object", "properties": { "ack_wait": { "type": "string", "description": "An optional duration to specify at which a message that is yet to be acked will be automatically retried. Default: 30s" }, "auth": { "type": "object", "description": "Optional configuration of NATS authentication parameters.", "properties": { "nkey_file": { "type": "string", "description": "An optional file containing a NKey seed.", "examples": [ "./seed.nk" ] }, "user_credentials_file": { "type": "string", "description": "An optional file containing user credentials which consist of an user JWT and corresponding NKey seed.", "examples": [ "./user.creds" ] }, "user_jwt": { "type": "string", "description": "An optional plain text user JWT (given along with the corresponding user NKey Seed)." }, "user_nkey_seed": { "type": "string", "description": "An optional plain text user NKey Seed (given along with the corresponding user JWT)." } } }, "client_id": { "type": "string", "description": "A client ID to connect as. Default: " }, "cluster_id": { "type": "string", "description": "The ID of the cluster to consume from." }, "durable_name": { "type": "string", "description": "Preserve the state of your consumer under a durable name. Default: " }, "extract_tracing_map": { "type": "string", "description": "EXPERIMENTAL: A [Bloblang mapping](/docs/guides/bloblang/about) that attempts to extract an object containing tracing propagation information, which will then be used as the root tracing span for the message. The specification of the extracted fields must match the format used by the service wide tracer.", "examples": [ "root = @", "root = this.meta.span" ] }, "max_inflight": { "type": "integer", "description": "The maximum number of unprocessed messages to fetch at a given time. Default: 1024" }, "new_root_span_with_link": { "type": "boolean", "description": "EXPERIMENTAL: Starts a new root span with link to parent." }, "queue": { "type": "string", "description": "The queue to consume from. Default: " }, "start_from_oldest": { "type": "boolean", "description": "If a position is not found for a queue, determines whether to consume from the oldest available message, otherwise messages are consumed from the latest. Default: true" }, "subject": { "type": "string", "description": "A subject to consume from. Default: " }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "unsubscribe_on_close": { "type": "boolean", "description": "Whether the subscription should be destroyed when this client disconnects. Default: false" }, "urls": { "type": "array", "description": "A list of URLs to connect to. If an item of the list contains commas it will be expanded into multiple URLs.", "items": { "type": "string" }, "examples": [ [ "nats://127.0.0.1:4222" ], [ "nats://username:password@127.0.0.1:4222" ] ] } } }, "nsq": { "type": "object", "properties": { "auth_secret": { "type": "string", "description": "An optional secret for NSQ authentication (requires nsqd 0.2.29+)." }, "channel": { "type": "string", "description": "The channel to consume from." }, "lookupd_http_addresses": { "type": "array", "description": "A list of nsqlookupd addresses to connect to.", "items": { "type": "string" } }, "max_attempts": { "type": "integer", "description": "The maximum number of attempts to successfully consume a messages. Default: 5" }, "max_in_flight": { "type": "integer", "description": "The maximum number of pending messages to consume at any given time. Default: 100" }, "nsqd_tcp_addresses": { "type": "array", "description": "A list of nsqd addresses to connect to.", "items": { "type": "string" } }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "topic": { "type": "string", "description": "The topic to consume from." }, "user_agent": { "type": "string", "description": "A user agent to assume when connecting." } } }, "parquet": { "type": "object", "properties": { "auto_replay_nacks": { "type": "boolean", "description": "Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation. Default: true" }, "batch_count": { "type": "integer", "description": "Optionally process records in batches. This can help to speed up the consumption of exceptionally large files. When the end of the file is reached the remaining records are processed as a (potentially smaller) batch. Default: 1" }, "paths": { "type": "array", "description": "A list of file paths to read from. Each file will be read sequentially until the list is exhausted, at which point the input will close. Glob patterns are supported, including super globs (double star).", "items": { "type": "string" }, "examples": [ "/tmp/foo.parquet", "/tmp/bar/*.parquet", "/tmp/data/**/*.parquet" ] }, "strict_schema": { "type": "boolean", "description": "Whether to enforce strict Parquet schema validation. When set to false, allows reading files with non-standard schema structures (such as non-standard LIST formats). Disabling strict mode may reduce validation but increases compatibility. Default: true" } } }, "pulsar": { "type": "object", "properties": { "auth": { "type": "object", "description": "Optional configuration of Pulsar authentication methods.", "properties": { "oauth2": { "type": "object", "description": "Parameters for Pulsar OAuth2 authentication.", "properties": { "audience": { "type": "string", "description": "OAuth2 audience. Default: " }, "enabled": { "type": "boolean", "description": "Whether OAuth2 is enabled. Default: false" }, "issuer_url": { "type": "string", "description": "OAuth2 issuer URL. Default: " }, "private_key_file": { "type": "string", "description": "The path to a file containing a private key. Default: " } } }, "token": { "type": "object", "description": "Parameters for Pulsar Token authentication.", "properties": { "enabled": { "type": "boolean", "description": "Whether Token Auth is enabled. Default: false" }, "token": { "type": "string", "description": "Actual base64 encoded token. Default: " } } } } }, "subscription_name": { "type": "string", "description": "Specify the subscription name for this consumer." }, "subscription_type": { "type": "string", "description": "Specify the subscription type for this consumer.\n\n\u003e NOTE: Using a `key_shared` subscription type will __allow out-of-order delivery__ since nack-ing messages sets non-zero nack delivery delay - this can potentially cause consumers to stall. See [Pulsar documentation](https://pulsar.apache.org/docs/en/2.8.1/concepts-messaging/#negative-acknowledgement) and [this Github issue](https://github.com/apache/pulsar/issues/12208) for more details. Default: shared" }, "tls": { "type": "object", "description": "Specify the path to a custom CA certificate to trust broker TLS service.", "properties": { "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] } } }, "topics": { "type": "array", "description": "A list of topics to subscribe to. This or topics_pattern must be set.", "items": { "type": "string" } }, "topics_pattern": { "type": "string", "description": "A regular expression matching the topics to subscribe to. This or topics must be set." }, "url": { "type": "string", "description": "A URL to connect to.", "examples": [ "pulsar://localhost:6650", "pulsar://pulsar.us-west.example.com:6650", "pulsar+ssl://pulsar.us-west.example.com:6651" ] } } }, "read_until": { "type": "object", "properties": { "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether the input should now be closed.", "examples": [ "this.type == \"foo\"", "count(\"messages\") \u003e= 100" ] }, "idle_timeout": { "type": "string", "description": "The maximum amount of time without receiving new messages after which the input is closed.", "examples": [ "5s" ] }, "input": { "type": "object", "description": "The child input to consume from." }, "restart_input": { "type": "boolean", "description": "Whether the input should be reopened if it closes itself before the condition has resolved to true. Default: false" } } }, "redis_list": { "type": "object", "properties": { "auto_replay_nacks": { "type": "boolean", "description": "Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation. Default: true" }, "command": { "type": "string", "description": "The command used to pop elements from the Redis list Default: blpop" }, "key": { "type": "string", "description": "The key of a list to read from." }, "kind": { "type": "string", "description": "Specifies a simple, cluster-aware, or failover-aware redis client. Default: simple" }, "master": { "type": "string", "description": "Name of the redis master when `kind` is `failover` Default: ", "examples": [ "mymaster" ] }, "max_in_flight": { "type": "integer", "description": "Optionally sets a limit on the number of messages that can be flowing through a Bento stream pending acknowledgment from the input at any given time. Once a message has been either acknowledged or rejected (nacked) it is no longer considered pending. If the input produces logical batches then each batch is considered a single count against the maximum. **WARNING**: Batching policies at the output level will stall if this field limits the number of messages below the batching threshold. Zero (default) or lower implies no limit. Default: 0" }, "timeout": { "type": "string", "description": "The length of time to poll for new messages before reattempting. Default: 5s" }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.\n\n**Troubleshooting**\n\nSome cloud hosted instances of Redis (such as Azure Cache) might need some hand holding in order to establish stable connections. Unfortunately, it is often the case that TLS issues will manifest as generic error messages such as \"i/o timeout\". If you're using TLS and are seeing connectivity problems consider setting `enable_renegotiation` to `true`, and ensuring that the server supports at least TLS version 1.2.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "url": { "type": "string", "description": "The URL of the target Redis server. Database is optional and is supplied as the URL path.", "examples": [ "redis://:6397", "redis://localhost:6379", "redis://foousername:foopassword@redisplace:6379", "redis://:foopassword@redisplace:6379", "redis://localhost:6379/1", "redis://localhost:6379/1,redis://localhost:6380/1" ] } } }, "redis_pubsub": { "type": "object", "properties": { "auto_replay_nacks": { "type": "boolean", "description": "Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation. Default: true" }, "channels": { "type": "array", "description": "A list of channels to consume from.", "items": { "type": "string" } }, "kind": { "type": "string", "description": "Specifies a simple, cluster-aware, or failover-aware redis client. Default: simple" }, "master": { "type": "string", "description": "Name of the redis master when `kind` is `failover` Default: ", "examples": [ "mymaster" ] }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.\n\n**Troubleshooting**\n\nSome cloud hosted instances of Redis (such as Azure Cache) might need some hand holding in order to establish stable connections. Unfortunately, it is often the case that TLS issues will manifest as generic error messages such as \"i/o timeout\". If you're using TLS and are seeing connectivity problems consider setting `enable_renegotiation` to `true`, and ensuring that the server supports at least TLS version 1.2.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "url": { "type": "string", "description": "The URL of the target Redis server. Database is optional and is supplied as the URL path.", "examples": [ "redis://:6397", "redis://localhost:6379", "redis://foousername:foopassword@redisplace:6379", "redis://:foopassword@redisplace:6379", "redis://localhost:6379/1", "redis://localhost:6379/1,redis://localhost:6380/1" ] }, "use_patterns": { "type": "boolean", "description": "Whether to use the PSUBSCRIBE command, allowing for glob-style patterns within target channel names. Default: false" } } }, "redis_scan": { "type": "object", "properties": { "auto_replay_nacks": { "type": "boolean", "description": "Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation. Default: true" }, "kind": { "type": "string", "description": "Specifies a simple, cluster-aware, or failover-aware redis client. Default: simple" }, "master": { "type": "string", "description": "Name of the redis master when `kind` is `failover` Default: ", "examples": [ "mymaster" ] }, "match": { "type": "string", "description": "Iterates only elements matching the optional glob-style pattern. By default, it matches all elements. Default: ", "examples": [ "*", "1*", "foo*", "foo", "*4*" ] }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.\n\n**Troubleshooting**\n\nSome cloud hosted instances of Redis (such as Azure Cache) might need some hand holding in order to establish stable connections. Unfortunately, it is often the case that TLS issues will manifest as generic error messages such as \"i/o timeout\". If you're using TLS and are seeing connectivity problems consider setting `enable_renegotiation` to `true`, and ensuring that the server supports at least TLS version 1.2.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "url": { "type": "string", "description": "The URL of the target Redis server. Database is optional and is supplied as the URL path.", "examples": [ "redis://:6397", "redis://localhost:6379", "redis://foousername:foopassword@redisplace:6379", "redis://:foopassword@redisplace:6379", "redis://localhost:6379/1", "redis://localhost:6379/1,redis://localhost:6380/1" ] } } }, "redis_streams": { "type": "object", "properties": { "auto_replay_nacks": { "type": "boolean", "description": "Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation. Default: true" }, "body_key": { "type": "string", "description": "The field key to extract the raw message from. All other keys will be stored in the message as metadata. Default: body" }, "client_id": { "type": "string", "description": "An identifier for the client connection. Default: " }, "commit_period": { "type": "string", "description": "The period of time between each commit of the current offset. Offsets are always committed during shutdown. Default: 1s" }, "consumer_group": { "type": "string", "description": "An identifier for the consumer group of the stream. Default: " }, "create_streams": { "type": "boolean", "description": "Create subscribed streams if they do not exist (MKSTREAM option). Default: true" }, "kind": { "type": "string", "description": "Specifies a simple, cluster-aware, or failover-aware redis client. Default: simple" }, "limit": { "type": "integer", "description": "The maximum number of messages to consume from a single request. Default: 10" }, "master": { "type": "string", "description": "Name of the redis master when `kind` is `failover` Default: ", "examples": [ "mymaster" ] }, "start_from_oldest": { "type": "boolean", "description": "If an offset is not found for a stream, determines whether to consume from the oldest available offset, otherwise messages are consumed from the latest offset. Default: true" }, "streams": { "type": "array", "description": "A list of streams to consume from.", "items": { "type": "string" } }, "timeout": { "type": "string", "description": "The length of time to poll for new messages before reattempting. Default: 1s" }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.\n\n**Troubleshooting**\n\nSome cloud hosted instances of Redis (such as Azure Cache) might need some hand holding in order to establish stable connections. Unfortunately, it is often the case that TLS issues will manifest as generic error messages such as \"i/o timeout\". If you're using TLS and are seeing connectivity problems consider setting `enable_renegotiation` to `true`, and ensuring that the server supports at least TLS version 1.2.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "url": { "type": "string", "description": "The URL of the target Redis server. Database is optional and is supplied as the URL path.", "examples": [ "redis://:6397", "redis://localhost:6379", "redis://foousername:foopassword@redisplace:6379", "redis://:foopassword@redisplace:6379", "redis://localhost:6379/1", "redis://localhost:6379/1,redis://localhost:6380/1" ] } } }, "resource": { "type": "string", "description": " Default: " }, "s2": { "type": "object", "properties": { "access_token": { "type": "string", "description": "Access token for S2 account" }, "backoff_duration": { "type": "string", "description": "Interval to backoff for before reconnecting to a stream Default: 100ms" }, "basin": { "type": "string", "description": "Basin name" }, "cache": { "type": "string", "description": "Cache resource label for storing sequence number" }, "max_in_flight": { "type": "integer", "description": "Optionally sets a limit on the number of messages that can be flowing through a Bento stream pending acknowledgment from the input at any given time. Once a message has been either acknowledged or rejected (nacked) it is no longer considered pending. If the input produces logical batches then each batch is considered a single count against the maximum. **WARNING**: Batching policies at the output level will stall if this field limits the number of messages below the batching threshold. Zero (default) or lower implies no limit. Default: 0" }, "start_seq_num": { "type": "string", "description": "Start consuming the stream from either the earliest or the latest sequence number Default: earliest" }, "streams": { "description": "Streams prefix or list of streams to subscribe to" }, "update_streams_interval": { "type": "string", "description": "Interval after which the streams list should update dynamically Default: 1m" } } }, "sequence": { "type": "object", "properties": { "inputs": { "type": "array", "description": "An array of inputs to read from sequentially.", "items": { "type": "object" } }, "sharded_join": { "type": "object", "description": "EXPERIMENTAL: Provides a way to perform outer joins of arbitrarily structured and unordered data resulting from the input sequence, even when the overall size of the data surpasses the memory available on the machine.\n\nWhen configured the sequence of inputs will be consumed one or more times according to the number of iterations, and when more than one iteration is specified each iteration will process an entirely different set of messages by sharding them by the ID field. Increasing the number of iterations reduces the memory consumption at the cost of needing to fully parse the data each time.\n\nEach message must be structured (JSON or otherwise processed into a structured form) and the fields will be aggregated with those of other messages sharing the ID. At the end of each iteration the joined messages are flushed downstream before the next iteration begins, hence keeping memory usage limited.", "properties": { "id_path": { "type": "string", "description": "A [dot path](/docs/configuration/field_paths) that points to a common field within messages of each fragmented data set and can be used to join them. Messages that are not structured or are missing this field will be dropped. This field must be set in order to enable joins. Default: " }, "iterations": { "type": "integer", "description": "The total number of iterations (shards), increasing this number will increase the overall time taken to process the data, but reduces the memory used in the process. The real memory usage required is significantly higher than the real size of the data and therefore the number of iterations should be at least an order of magnitude higher than the available memory divided by the overall size of the dataset. Default: 1" }, "merge_strategy": { "type": "string", "description": "The chosen strategy to use when a data join would otherwise result in a collision of field values. The strategy `array` means non-array colliding values are placed into an array and colliding arrays are merged. The strategy `replace` replaces old values with new values. The strategy `keep` keeps the old value. Default: array" }, "type": { "type": "string", "description": "The type of join to perform. A `full-outer` ensures that all identifiers seen in any of the input sequences are sent, and is performed by consuming all input sequences before flushing the joined results. An `outer` join consumes all input sequences but only writes data joined from the last input in the sequence, similar to a left or right outer join. With an `outer` join if an identifier appears multiple times within the final sequence input it will be flushed each time it appears. `full-outter` and `outter` have been deprecated in favour of `full-outer` and `outer`. Default: none" } } } } }, "sftp": { "type": "object", "properties": { "address": { "type": "string", "description": "The address of the server to connect to." }, "auto_replay_nacks": { "type": "boolean", "description": "Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation. Default: true" }, "codec": { "type": "string", "description": "The way in which the bytes of a data source should be converted into discrete messages, codecs are useful for specifying how large files or continuous streams of data might be processed in small chunks rather than loading it all in memory. It's possible to consume lines using a custom delimiter with the `delim:x` codec, where x is the character sequence custom delimiter. Codecs can be chained with `/`, for example a gzip compressed CSV file can be consumed with the codec `gzip/csv`.", "examples": [ "lines", "delim:\t", "delim:foobar", "gzip/csv" ] }, "credentials": { "type": "object", "description": "The credentials to use to log into the target server.", "properties": { "password": { "type": "string", "description": "The password for the username to connect to the SFTP server. Default: " }, "private_key_file": { "type": "string", "description": "The private key for the username to connect to the SFTP server. Default: " }, "private_key_pass": { "type": "string", "description": "Optional passphrase for private key. Default: " }, "username": { "type": "string", "description": "The username to connect to the SFTP server. Default: " } } }, "delete_on_finish": { "type": "boolean", "description": "Whether to delete files from the server once they are processed. Default: false" }, "max_buffer": { "type": "integer", "description": " Default: 1000000" }, "paths": { "type": "array", "description": "A list of paths to consume sequentially. Glob patterns are supported.", "items": { "type": "string" } }, "scanner": { "type": "object", "description": "The [scanner](/docs/components/scanners/about) by which the stream of bytes consumed will be broken out into individual messages. Scanners are useful for processing large sources of data without holding the entirety of it within memory. For example, the `csv` scanner allows you to process individual CSV rows without loading the entire CSV file in memory at once. Default: map[to_the_end:map[]]" }, "watcher": { "type": "object", "description": "An experimental mode whereby the input will periodically scan the target paths for new files and consume them, when all files are consumed the input will continue polling for new files.", "properties": { "cache": { "type": "string", "description": "A [cache resource](/docs/components/caches/about) for storing the paths of files already consumed. Default: " }, "enabled": { "type": "boolean", "description": "Whether file watching is enabled. Default: false" }, "minimum_age": { "type": "string", "description": "The minimum period of time since a file was last updated before attempting to consume it. Increasing this period decreases the likelihood that a file will be consumed whilst it is still being written to. Default: 1s", "examples": [ "10s", "1m", "10m" ] }, "poll_interval": { "type": "string", "description": "The interval between each attempt to scan the target paths for new files. Default: 1s", "examples": [ "100ms", "1s" ] } } } } }, "socket": { "type": "object", "properties": { "address": { "type": "string", "description": "The address to connect to.", "examples": [ "/tmp/bento.sock", "127.0.0.1:6000" ] }, "auto_replay_nacks": { "type": "boolean", "description": "Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation. Default: true" }, "codec": { "type": "string", "description": "The way in which the bytes of a data source should be converted into discrete messages, codecs are useful for specifying how large files or continuous streams of data might be processed in small chunks rather than loading it all in memory. It's possible to consume lines using a custom delimiter with the `delim:x` codec, where x is the character sequence custom delimiter. Codecs can be chained with `/`, for example a gzip compressed CSV file can be consumed with the codec `gzip/csv`.", "examples": [ "lines", "delim:\t", "delim:foobar", "gzip/csv" ] }, "max_buffer": { "type": "integer", "description": " Default: 1000000" }, "network": { "type": "string", "description": "A network type to assume (unix|tcp)." }, "scanner": { "type": "object", "description": "The [scanner](/docs/components/scanners/about) by which the stream of bytes consumed will be broken out into individual messages. Scanners are useful for processing large sources of data without holding the entirety of it within memory. For example, the `csv` scanner allows you to process individual CSV rows without loading the entire CSV file in memory at once. Default: map[lines:map[]]" } } }, "socket_server": { "type": "object", "properties": { "address": { "type": "string", "description": "The address to listen from.", "examples": [ "/tmp/bento.sock", "0.0.0.0:6000" ] }, "address_cache": { "type": "string", "description": "An optional [`cache`](/docs/components/caches/about) within which this input should write it's bound address once known. The key of the cache item containing the address will be the label of the component suffixed with `_address` (e.g. `foo_address`), or `socket_server_address` when a label has not been provided. This is useful in situations where the address is dynamically allocated by the server (`127.0.0.1:0`) and you want to store the allocated address somewhere for reference by other systems and components." }, "auto_replay_nacks": { "type": "boolean", "description": "Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation. Default: true" }, "codec": { "type": "string", "description": "The way in which the bytes of a data source should be converted into discrete messages, codecs are useful for specifying how large files or continuous streams of data might be processed in small chunks rather than loading it all in memory. It's possible to consume lines using a custom delimiter with the `delim:x` codec, where x is the character sequence custom delimiter. Codecs can be chained with `/`, for example a gzip compressed CSV file can be consumed with the codec `gzip/csv`.", "examples": [ "lines", "delim:\t", "delim:foobar", "gzip/csv" ] }, "max_buffer": { "type": "integer", "description": " Default: 1000000" }, "network": { "type": "string", "description": "A network type to accept." }, "scanner": { "type": "object", "description": "The [scanner](/docs/components/scanners/about) by which the stream of bytes consumed will be broken out into individual messages. Scanners are useful for processing large sources of data without holding the entirety of it within memory. For example, the `csv` scanner allows you to process individual CSV rows without loading the entire CSV file in memory at once. Default: map[lines:map[]]" }, "tls": { "type": "object", "description": "TLS specific configuration, valid when the `network` is set to `tls`.", "properties": { "cert_file": { "type": "string", "description": "PEM encoded certificate for use with TLS." }, "key_file": { "type": "string", "description": "PEM encoded private key for use with TLS." }, "self_signed": { "type": "boolean", "description": "Whether to generate self signed certificates. Default: false" } } } } }, "sql_raw": { "type": "object", "properties": { "args_mapping": { "type": "string", "description": "A [Bloblang mapping](/docs/guides/bloblang/about) which should evaluate to an array of values matching in size to the number of columns specified.", "examples": [ "root = [ this.cat.meow, this.doc.woofs[0] ]", "root = [ metadata(\"user.id\").string() ]" ] }, "auto_replay_nacks": { "type": "boolean", "description": "Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation. Default: true" }, "azure": { "type": "object", "description": "Optional Fields that can be set to use Azure based authentication for Azure Postgres SQL", "properties": { "entra_enabled": { "type": "boolean", "description": "An optional field used to generate an entra token to connect to 'Azure Database for PostgreSQL flexible server', This will create a new connection string with the host, user and database from the DSN field - you may need to URL encode the dsn! The [Default Azure Credential Chain](https://learn.microsoft.com/en-gb/azure/developer/go/sdk/authentication/authentication-overview#defaultazurecredential) is used from the Azure SDK. Default: false" }, "token_request_options": { "type": "object", "properties": { "claims": { "type": "string", "description": "Set additional claims for the token. Default: " }, "enable_cae": { "type": "boolean", "description": "Indicates whether to enable Continuous Access Evaluation (CAE) for the requested token Default: false" }, "scopes": { "type": "array", "description": "Scopes contains the list of permission scopes required for the token. Default: [https://ossrdbms-aad.database.windows.net/.default]", "items": { "type": "string" } }, "tenant_id": { "type": "string", "description": "tenant_id identifies the tenant from which to request the token. azure credentials authenticate in their configured default tenants when this field isn't set. Default: " } } } } }, "conn_max_idle": { "type": "integer", "description": "An optional maximum number of connections in the idle connection pool. If conn_max_open is greater than 0 but less than the new conn_max_idle, then the new conn_max_idle will be reduced to match the conn_max_open limit. If `value \u003c= 0`, no idle connections are retained. The default max idle connections is currently 2. This may change in a future release. Default: 2" }, "conn_max_idle_time": { "type": "string", "description": "An optional maximum amount of time a connection may be idle. Expired connections may be closed lazily before reuse. If `value \u003c= 0`, connections are not closed due to a connections idle time." }, "conn_max_life_time": { "type": "string", "description": "An optional maximum amount of time a connection may be reused. Expired connections may be closed lazily before reuse. If `value \u003c= 0`, connections are not closed due to a connections age." }, "conn_max_open": { "type": "integer", "description": "An optional maximum number of open connections to the database. If conn_max_idle is greater than 0 and the new conn_max_open is less than conn_max_idle, then conn_max_idle will be reduced to match the new conn_max_open limit. If `value \u003c= 0`, then there is no limit on the number of open connections. The default is 0 (unlimited)." }, "credentials": { "type": "object", "description": "Optional manual configuration of AWS credentials to use. More information can be found [in this document](/docs/guides/cloud/aws).", "properties": { "expiry_window": { "type": "string", "description": "Allow the credentials to trigger refreshing prior to the credentials actually expiring. This is beneficial so race conditions with expiring credentials do not cause requests to fail. For example '10s' would refresh credentials ten seconds before expiration. Setting to a duration of `0` disables the expiry window. Default: " }, "from_ec2_role": { "type": "boolean", "description": "Use the credentials of a host EC2 machine configured to assume [an IAM role associated with the instance](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html). Default: false" }, "id": { "type": "string", "description": "The ID of credentials to use. Default: " }, "profile": { "type": "string", "description": "A profile from `~/.aws/credentials` to use. Default: " }, "role": { "type": "string", "description": "A role ARN to assume. Default: " }, "role_external_id": { "type": "string", "description": "An external ID to provide when assuming a role. Default: " }, "secret": { "type": "string", "description": "The secret for the credentials being used. Default: " }, "token": { "type": "string", "description": "The token for the credentials being used, required when using short term credentials. Default: " } } }, "driver": { "type": "string", "description": "A database [driver](#drivers) to use." }, "dsn": { "type": "string", "description": "A Data Source Name to identify the target database.\n\n#### Drivers\n\nThe following is a list of supported drivers, their placeholder style, and their respective DSN formats:\n\n| Driver | Data Source Name Format |\n|---|---|\n| `clickhouse` | [`clickhouse://[username[:password]@][netloc][:port]/dbname[?param1=value1\u0026...\u0026paramN=valueN]`](https://github.com/ClickHouse/clickhouse-go#dsn) |\n| `mysql` | `[username[:password]@][protocol[(address)]]/dbname[?param1=value1\u0026...\u0026paramN=valueN]` |\n| `postgres` | `postgres://[user[:password]@][netloc][:port][/dbname][?param1=value1\u0026...]` |\n| `mssql` | `sqlserver://[user[:password]@][netloc][:port][?database=dbname\u0026param1=value1\u0026...]` |\n| `sqlite` | `file:/path/to/filename.db[?param\u0026=value1\u0026...]` |\n| `oracle` | `oracle://[username[:password]@][netloc][:port]/service_name?server=server2\u0026server=server3` |\n| `snowflake` | `username[:password]@account_identifier/dbname/schemaname[?param1=value\u0026...\u0026paramN=valueN]` |\n| `spanner` | `projects/[project]/instances/[instance]/databases/dbname` |\n| `trino` | [`http[s]://user[:pass]@host[:port][?parameters]`](https://github.com/trinodb/trino-go-client#dsn-data-source-name) |\n| `gocosmos` | [`AccountEndpoint=\u003ccosmosdb-endpoint\u003e;AccountKey=\u003ccosmosdb-account-key\u003e[;TimeoutMs=\u003ctimeout-in-ms\u003e][;Version=\u003ccosmosdb-api-version\u003e][;DefaultDb/Db=\u003cdb-name\u003e][;AutoId=\u003ctrue/false\u003e][;InsecureSkipVerify=\u003ctrue/false\u003e]`](https://pkg.go.dev/github.com/microsoft/gocosmos#readme-example-usage) |\n\nPlease note that the `postgres` driver enforces SSL by default, you can override this with the parameter `sslmode=disable` if required.\n\nThe `snowflake` driver supports multiple DSN formats. Please consult [the docs](https://pkg.go.dev/github.com/snowflakedb/gosnowflake#hdr-Connection_String) for more details. For [key pair authentication](https://docs.snowflake.com/en/user-guide/key-pair-auth.html#configuring-key-pair-authentication), the DSN has the following format: `\u003csnowflake_user\u003e@\u003csnowflake_account\u003e/\u003cdb_name\u003e/\u003cschema_name\u003e?warehouse=\u003cwarehouse\u003e\u0026role=\u003crole\u003e\u0026authenticator=snowflake_jwt\u0026privateKey=\u003cbase64_url_encoded_private_key\u003e`, where the value for the `privateKey` parameter can be constructed from an unencrypted RSA private key file `rsa_key.p8` using `openssl enc -d -base64 -in rsa_key.p8 | basenc --base64url -w0` (you can use `gbasenc` insted of `basenc` on OSX if you install `coreutils` via Homebrew). If you have a password-encrypted private key, you can decrypt it using `openssl pkcs8 -in rsa_key_encrypted.p8 -out rsa_key.p8`. Also, make sure fields such as the username are URL-encoded.\n\nThe [`gocosmos`](https://pkg.go.dev/github.com/microsoft/gocosmos) driver is still experimental, but it has support for [hierarchical partition keys](https://learn.microsoft.com/en-us/azure/cosmos-db/hierarchical-partition-keys) as well as [cross-partition queries](https://learn.microsoft.com/en-us/azure/cosmos-db/nosql/how-to-query-container#cross-partition-query). Please refer to the [SQL notes](https://github.com/microsoft/gocosmos/blob/main/SQL.md) for details.", "examples": [ "clickhouse://username:password@host1:9000,host2:9000/database?dial_timeout=200ms\u0026max_execution_time=60", "foouser:foopassword@tcp(localhost:3306)/foodb", "postgres://foouser:foopass@localhost:5432/foodb?sslmode=disable", "oracle://foouser:foopass@localhost:1521/service_name" ] }, "endpoint": { "type": "string", "description": "Allows you to specify a custom endpoint for the AWS API. Default: " }, "iam_enabled": { "type": "boolean", "description": "An optional field used to generate an IAM authentication token to connect to an Amazon Relational Database (RDS) DB instance. This will overwrite the Password in the DSN with the generated token only if the drivers are `mysql` or `postgres`. Default: false" }, "init_files": { "type": "array", "description": "\nAn optional list of file paths containing SQL statements to execute immediately upon the first connection to the target database. This is a useful way to initialise tables before processing data. Glob patterns are supported, including super globs (double star).\n\nCare should be taken to ensure that the statements are idempotent, and therefore would not cause issues when run multiple times after service restarts. If both `init_statement` and `init_files` are specified the `init_statement` is executed _after_ the `init_files`.\n\nIf a statement fails for any reason a warning log will be emitted but the operation of this component will not be stopped.\n", "items": { "type": "string" }, "examples": [ [ "./init/*.sql" ], [ "./foo.sql", "./bar.sql" ] ] }, "init_statement": { "type": "string", "description": "\nAn optional SQL statement to execute immediately upon the first connection to the target database. This is a useful way to initialise tables before processing data. Care should be taken to ensure that the statement is idempotent, and therefore would not cause issues when run multiple times after service restarts.\n\nIf both `init_statement` and `init_files` are specified the `init_statement` is executed _after_ the `init_files`.\n\nIf the statement fails for any reason a warning log will be emitted but the operation of this component will not be stopped.\n", "examples": [ "\nCREATE TABLE IF NOT EXISTS some_table (\n foo varchar(50) not null,\n bar integer,\n baz varchar(50),\n primary key (foo)\n) WITHOUT ROWID;\n" ] }, "init_verify_conn": { "type": "boolean", "description": "Whether to verify the database connection on startup by performing a simple ping, by default this is disabled. Default: false" }, "query": { "type": "string", "description": "The query to execute. The style of placeholder to use depends on the driver, some drivers require question marks (`?`) whereas others expect incrementing dollar signs (`$1`, `$2`, and so on) or colons (`:1`, `:2` and so on). The style to use is outlined in this table:\n\n| Driver | Placeholder Style |\n|---|---|\n| `clickhouse` | Dollar sign |\n| `mysql` | Question mark |\n| `postgres` | Dollar sign |\n| `mssql` | Question mark |\n| `sqlite` | Question mark |\n| `oracle` | Colon |\n| `snowflake` | Question mark |\n| `spanner` | Question mark |\n| `trino` | Question mark |\n| `gocosmos` | Colon |\n", "examples": [ "SELECT * FROM footable WHERE user_id = $1;" ] }, "region": { "type": "string", "description": "The AWS region to target. Default: " }, "secret_name": { "type": "string", "description": "An optional field that can be used to get the Username + Password from AWS Secrets Manager. This will overwrite the Username + Password in the DSN with the values from the Secret only if the driver is set to `postgres`." } } }, "sql_select": { "type": "object", "properties": { "args_mapping": { "type": "string", "description": "An optional [Bloblang mapping](/docs/guides/bloblang/about) which should evaluate to an array of values matching in size to the number of placeholder arguments in the field `where`.", "examples": [ "root = [ \"article\", now().ts_format(\"2006-01-02\") ]" ] }, "auto_replay_nacks": { "type": "boolean", "description": "Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation. Default: true" }, "azure": { "type": "object", "description": "Optional Fields that can be set to use Azure based authentication for Azure Postgres SQL", "properties": { "entra_enabled": { "type": "boolean", "description": "An optional field used to generate an entra token to connect to 'Azure Database for PostgreSQL flexible server', This will create a new connection string with the host, user and database from the DSN field - you may need to URL encode the dsn! The [Default Azure Credential Chain](https://learn.microsoft.com/en-gb/azure/developer/go/sdk/authentication/authentication-overview#defaultazurecredential) is used from the Azure SDK. Default: false" }, "token_request_options": { "type": "object", "properties": { "claims": { "type": "string", "description": "Set additional claims for the token. Default: " }, "enable_cae": { "type": "boolean", "description": "Indicates whether to enable Continuous Access Evaluation (CAE) for the requested token Default: false" }, "scopes": { "type": "array", "description": "Scopes contains the list of permission scopes required for the token. Default: [https://ossrdbms-aad.database.windows.net/.default]", "items": { "type": "string" } }, "tenant_id": { "type": "string", "description": "tenant_id identifies the tenant from which to request the token. azure credentials authenticate in their configured default tenants when this field isn't set. Default: " } } } } }, "columns": { "type": "array", "description": "A list of columns to select.", "items": { "type": "string" }, "examples": [ [ "*" ], [ "foo", "bar", "baz" ] ] }, "conn_max_idle": { "type": "integer", "description": "An optional maximum number of connections in the idle connection pool. If conn_max_open is greater than 0 but less than the new conn_max_idle, then the new conn_max_idle will be reduced to match the conn_max_open limit. If `value \u003c= 0`, no idle connections are retained. The default max idle connections is currently 2. This may change in a future release. Default: 2" }, "conn_max_idle_time": { "type": "string", "description": "An optional maximum amount of time a connection may be idle. Expired connections may be closed lazily before reuse. If `value \u003c= 0`, connections are not closed due to a connections idle time." }, "conn_max_life_time": { "type": "string", "description": "An optional maximum amount of time a connection may be reused. Expired connections may be closed lazily before reuse. If `value \u003c= 0`, connections are not closed due to a connections age." }, "conn_max_open": { "type": "integer", "description": "An optional maximum number of open connections to the database. If conn_max_idle is greater than 0 and the new conn_max_open is less than conn_max_idle, then conn_max_idle will be reduced to match the new conn_max_open limit. If `value \u003c= 0`, then there is no limit on the number of open connections. The default is 0 (unlimited)." }, "credentials": { "type": "object", "description": "Optional manual configuration of AWS credentials to use. More information can be found [in this document](/docs/guides/cloud/aws).", "properties": { "expiry_window": { "type": "string", "description": "Allow the credentials to trigger refreshing prior to the credentials actually expiring. This is beneficial so race conditions with expiring credentials do not cause requests to fail. For example '10s' would refresh credentials ten seconds before expiration. Setting to a duration of `0` disables the expiry window. Default: " }, "from_ec2_role": { "type": "boolean", "description": "Use the credentials of a host EC2 machine configured to assume [an IAM role associated with the instance](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html). Default: false" }, "id": { "type": "string", "description": "The ID of credentials to use. Default: " }, "profile": { "type": "string", "description": "A profile from `~/.aws/credentials` to use. Default: " }, "role": { "type": "string", "description": "A role ARN to assume. Default: " }, "role_external_id": { "type": "string", "description": "An external ID to provide when assuming a role. Default: " }, "secret": { "type": "string", "description": "The secret for the credentials being used. Default: " }, "token": { "type": "string", "description": "The token for the credentials being used, required when using short term credentials. Default: " } } }, "driver": { "type": "string", "description": "A database [driver](#drivers) to use." }, "dsn": { "type": "string", "description": "A Data Source Name to identify the target database.\n\n#### Drivers\n\nThe following is a list of supported drivers, their placeholder style, and their respective DSN formats:\n\n| Driver | Data Source Name Format |\n|---|---|\n| `clickhouse` | [`clickhouse://[username[:password]@][netloc][:port]/dbname[?param1=value1\u0026...\u0026paramN=valueN]`](https://github.com/ClickHouse/clickhouse-go#dsn) |\n| `mysql` | `[username[:password]@][protocol[(address)]]/dbname[?param1=value1\u0026...\u0026paramN=valueN]` |\n| `postgres` | `postgres://[user[:password]@][netloc][:port][/dbname][?param1=value1\u0026...]` |\n| `mssql` | `sqlserver://[user[:password]@][netloc][:port][?database=dbname\u0026param1=value1\u0026...]` |\n| `sqlite` | `file:/path/to/filename.db[?param\u0026=value1\u0026...]` |\n| `oracle` | `oracle://[username[:password]@][netloc][:port]/service_name?server=server2\u0026server=server3` |\n| `snowflake` | `username[:password]@account_identifier/dbname/schemaname[?param1=value\u0026...\u0026paramN=valueN]` |\n| `spanner` | `projects/[project]/instances/[instance]/databases/dbname` |\n| `trino` | [`http[s]://user[:pass]@host[:port][?parameters]`](https://github.com/trinodb/trino-go-client#dsn-data-source-name) |\n| `gocosmos` | [`AccountEndpoint=\u003ccosmosdb-endpoint\u003e;AccountKey=\u003ccosmosdb-account-key\u003e[;TimeoutMs=\u003ctimeout-in-ms\u003e][;Version=\u003ccosmosdb-api-version\u003e][;DefaultDb/Db=\u003cdb-name\u003e][;AutoId=\u003ctrue/false\u003e][;InsecureSkipVerify=\u003ctrue/false\u003e]`](https://pkg.go.dev/github.com/microsoft/gocosmos#readme-example-usage) |\n\nPlease note that the `postgres` driver enforces SSL by default, you can override this with the parameter `sslmode=disable` if required.\n\nThe `snowflake` driver supports multiple DSN formats. Please consult [the docs](https://pkg.go.dev/github.com/snowflakedb/gosnowflake#hdr-Connection_String) for more details. For [key pair authentication](https://docs.snowflake.com/en/user-guide/key-pair-auth.html#configuring-key-pair-authentication), the DSN has the following format: `\u003csnowflake_user\u003e@\u003csnowflake_account\u003e/\u003cdb_name\u003e/\u003cschema_name\u003e?warehouse=\u003cwarehouse\u003e\u0026role=\u003crole\u003e\u0026authenticator=snowflake_jwt\u0026privateKey=\u003cbase64_url_encoded_private_key\u003e`, where the value for the `privateKey` parameter can be constructed from an unencrypted RSA private key file `rsa_key.p8` using `openssl enc -d -base64 -in rsa_key.p8 | basenc --base64url -w0` (you can use `gbasenc` insted of `basenc` on OSX if you install `coreutils` via Homebrew). If you have a password-encrypted private key, you can decrypt it using `openssl pkcs8 -in rsa_key_encrypted.p8 -out rsa_key.p8`. Also, make sure fields such as the username are URL-encoded.\n\nThe [`gocosmos`](https://pkg.go.dev/github.com/microsoft/gocosmos) driver is still experimental, but it has support for [hierarchical partition keys](https://learn.microsoft.com/en-us/azure/cosmos-db/hierarchical-partition-keys) as well as [cross-partition queries](https://learn.microsoft.com/en-us/azure/cosmos-db/nosql/how-to-query-container#cross-partition-query). Please refer to the [SQL notes](https://github.com/microsoft/gocosmos/blob/main/SQL.md) for details.", "examples": [ "clickhouse://username:password@host1:9000,host2:9000/database?dial_timeout=200ms\u0026max_execution_time=60", "foouser:foopassword@tcp(localhost:3306)/foodb", "postgres://foouser:foopass@localhost:5432/foodb?sslmode=disable", "oracle://foouser:foopass@localhost:1521/service_name" ] }, "endpoint": { "type": "string", "description": "Allows you to specify a custom endpoint for the AWS API. Default: " }, "iam_enabled": { "type": "boolean", "description": "An optional field used to generate an IAM authentication token to connect to an Amazon Relational Database (RDS) DB instance. This will overwrite the Password in the DSN with the generated token only if the drivers are `mysql` or `postgres`. Default: false" }, "init_files": { "type": "array", "description": "\nAn optional list of file paths containing SQL statements to execute immediately upon the first connection to the target database. This is a useful way to initialise tables before processing data. Glob patterns are supported, including super globs (double star).\n\nCare should be taken to ensure that the statements are idempotent, and therefore would not cause issues when run multiple times after service restarts. If both `init_statement` and `init_files` are specified the `init_statement` is executed _after_ the `init_files`.\n\nIf a statement fails for any reason a warning log will be emitted but the operation of this component will not be stopped.\n", "items": { "type": "string" }, "examples": [ [ "./init/*.sql" ], [ "./foo.sql", "./bar.sql" ] ] }, "init_statement": { "type": "string", "description": "\nAn optional SQL statement to execute immediately upon the first connection to the target database. This is a useful way to initialise tables before processing data. Care should be taken to ensure that the statement is idempotent, and therefore would not cause issues when run multiple times after service restarts.\n\nIf both `init_statement` and `init_files` are specified the `init_statement` is executed _after_ the `init_files`.\n\nIf the statement fails for any reason a warning log will be emitted but the operation of this component will not be stopped.\n", "examples": [ "\nCREATE TABLE IF NOT EXISTS some_table (\n foo varchar(50) not null,\n bar integer,\n baz varchar(50),\n primary key (foo)\n) WITHOUT ROWID;\n" ] }, "init_verify_conn": { "type": "boolean", "description": "Whether to verify the database connection on startup by performing a simple ping, by default this is disabled. Default: false" }, "prefix": { "type": "string", "description": "An optional prefix to prepend to the select query (before SELECT)." }, "region": { "type": "string", "description": "The AWS region to target. Default: " }, "secret_name": { "type": "string", "description": "An optional field that can be used to get the Username + Password from AWS Secrets Manager. This will overwrite the Username + Password in the DSN with the values from the Secret only if the driver is set to `postgres`." }, "suffix": { "type": "string", "description": "An optional suffix to append to the select query." }, "table": { "type": "string", "description": "The table to select from.", "examples": [ "foo" ] }, "where": { "type": "string", "description": "An optional where clause to add. Placeholder arguments are populated with the `args_mapping` field. Placeholders should always be question marks, and will automatically be converted to dollar syntax when the postgres or clickhouse drivers are used.", "examples": [ "type = ? and created_at \u003e ?", "user_id = ?" ] } } }, "stdin": { "type": "object", "properties": { "auto_replay_nacks": { "type": "boolean", "description": "Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation. Default: true" }, "codec": { "type": "string", "description": "The way in which the bytes of a data source should be converted into discrete messages, codecs are useful for specifying how large files or continuous streams of data might be processed in small chunks rather than loading it all in memory. It's possible to consume lines using a custom delimiter with the `delim:x` codec, where x is the character sequence custom delimiter. Codecs can be chained with `/`, for example a gzip compressed CSV file can be consumed with the codec `gzip/csv`.", "examples": [ "lines", "delim:\t", "delim:foobar", "gzip/csv" ] }, "max_buffer": { "type": "integer", "description": " Default: 1000000" }, "scanner": { "type": "object", "description": "The [scanner](/docs/components/scanners/about) by which the stream of bytes consumed will be broken out into individual messages. Scanners are useful for processing large sources of data without holding the entirety of it within memory. For example, the `csv` scanner allows you to process individual CSV rows without loading the entire CSV file in memory at once. Default: map[lines:map[]]" } } }, "subprocess": { "type": "object", "properties": { "args": { "type": "array", "description": "A list of arguments to provide the command. Default: []", "items": { "type": "string" } }, "codec": { "type": "string", "description": "The way in which messages should be consumed from the subprocess. Default: lines" }, "max_buffer": { "type": "integer", "description": "The maximum expected size of an individual message. Default: 65536" }, "name": { "type": "string", "description": "The command to execute as a subprocess.", "examples": [ "cat", "sed", "awk" ] }, "restart_on_exit": { "type": "boolean", "description": "Whether the command should be re-executed each time the subprocess ends. Default: false" } } }, "twitter_search": { "type": "object", "properties": { "api_key": { "type": "string", "description": "An API key for OAuth 2.0 authentication. It is recommended that you populate this field using [environment variables](/docs/configuration/interpolation)." }, "api_secret": { "type": "string", "description": "An API secret for OAuth 2.0 authentication. It is recommended that you populate this field using [environment variables](/docs/configuration/interpolation)." }, "backfill_period": { "type": "string", "description": "A duration string indicating the maximum age of tweets to acquire when starting a search. Default: 5m" }, "cache": { "type": "string", "description": "A cache resource to use for request pagination." }, "cache_key": { "type": "string", "description": "The key identifier used when storing the ID of the last tweet received. Default: last_tweet_id" }, "poll_period": { "type": "string", "description": "The length of time (as a duration string) to wait between each search request. This field can be set empty, in which case requests are made at the limit set by the rate limit. This field also supports cron expressions. Default: 1m" }, "query": { "type": "string", "description": "A search expression to use." }, "rate_limit": { "type": "string", "description": "An optional rate limit resource to restrict API requests with. Default: " }, "tweet_fields": { "type": "array", "description": "An optional list of additional fields to obtain for each tweet, by default only the fields `id` and `text` are returned. For more info refer to the [twitter API docs.](https://developer.twitter.com/en/docs/twitter-api/fields) Default: []", "items": { "type": "string" } } } }, "websocket": { "type": "object", "properties": { "auto_replay_nacks": { "type": "boolean", "description": "Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation. Default: true" }, "basic_auth": { "type": "object", "description": "Allows you to specify basic authentication.", "properties": { "enabled": { "type": "boolean", "description": "Whether to use basic authentication in requests. Default: false" }, "password": { "type": "string", "description": "A password to authenticate with. Default: " }, "username": { "type": "string", "description": "A username to authenticate as. Default: " } } }, "connection": { "type": "object", "description": "Customise how websocket connection attempts are made.", "properties": { "max_retries": { "type": "integer", "description": "An optional limit to the number of consecutive retry attempts that will be made before abandoning the connection altogether and gracefully terminating the input. When all inputs terminate in this way the service (or stream) will shut down. If set to zero connections will never be reattempted upon a failure. If set below zero this field is ignored (effectively unset).", "examples": [ -1, 10 ] } } }, "headers": { "type": "object", "description": "A map of custom headers to add to the websocket handshake. Default: map[]", "examples": [ { "Sec-WebSocket-Protocol": "graphql-ws", "User-Agent": "${! uuid_v4() }", "X-Client-ID": "${CLIENT_ID}" } ] }, "jwt": { "type": "object", "description": "BETA: Allows you to specify JWT authentication.", "properties": { "claims": { "type": "object", "description": "A value used to identify the claims that issued the JWT. Default: map[]" }, "enabled": { "type": "boolean", "description": "Whether to use JWT authentication in requests. Default: false" }, "headers": { "type": "object", "description": "Add optional key/value headers to the JWT. Default: map[]" }, "private_key_file": { "type": "string", "description": "A file with the PEM encoded via PKCS1 or PKCS8 as private key. Default: " }, "signing_method": { "type": "string", "description": "A method used to sign the token such as RS256, RS384, RS512 or EdDSA. Default: " } } }, "oauth": { "type": "object", "description": "Allows you to specify open authentication via OAuth version 1.", "properties": { "access_token": { "type": "string", "description": "A value used to gain access to the protected resources on behalf of the user. Default: " }, "access_token_secret": { "type": "string", "description": "A secret provided in order to establish ownership of a given access token. Default: " }, "consumer_key": { "type": "string", "description": "A value used to identify the client to the service provider. Default: " }, "consumer_secret": { "type": "string", "description": "A secret used to establish ownership of the consumer key. Default: " }, "enabled": { "type": "boolean", "description": "Whether to use OAuth version 1 in requests. Default: false" } } }, "open_message": { "type": "string", "description": "An optional message to send to the server upon connection." }, "open_message_type": { "type": "string", "description": "An optional flag to indicate the data type of open_message. Default: binary" }, "open_messages": { "type": "array", "description": "An optional list of messages to send to the server upon connection. This field replaces `open_message`, which will be removed in a future version.", "items": { "type": "string" } }, "proxy_url": { "type": "string", "description": "An optional HTTP proxy URL." }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "url": { "type": "string", "description": "The URL to connect to.", "examples": [ "ws://localhost:4195/get/ws" ] } } }, "zmq4n": { "type": "object", "properties": { "bind": { "type": "boolean", "description": "Whether to bind to the specified URLs (otherwise they are connected to). Default: false" }, "high_water_mark": { "type": "integer", "description": "The message high water mark to use. (experimental i go-zeromq) Default: 0" }, "poll_timeout": { "type": "string", "description": "The poll timeout to use. Default: 5s" }, "socket_type": { "type": "string", "description": "The socket type to connect as." }, "sub_filters": { "type": "array", "description": "A list of subscription topic filters to use when consuming from a SUB socket. Specifying a single sub_filter of `''` will subscribe to everything. Default: []", "items": { "type": "string" } }, "urls": { "type": "array", "description": "A list of URLs to connect to. If an item of the list contains commas it will be expanded into multiple URLs.", "items": { "type": "string" }, "examples": [ [ "tcp://localhost:5555" ] ] } } } } }, "input_resources": { "type": "array", "description": "A list of input resources, each must have a unique label. Default: []", "properties": { "amqp_0_9": { "type": "object", "properties": { "auto_ack": { "type": "boolean", "description": "Acknowledge messages automatically as they are consumed rather than waiting for acknowledgments from downstream. This can improve throughput and prevent the pipeline from blocking but at the cost of eliminating delivery guarantees. Default: false" }, "bindings_declare": { "type": "array", "description": "Allows you to passively declare bindings for the target queue.", "items": { "type": "object" }, "examples": [ [ { "exchange": "foo", "key": "bar" } ] ] }, "consumer_tag": { "type": "string", "description": "A consumer tag. Default: " }, "nack_reject_patterns": { "type": "array", "description": "A list of regular expression patterns whereby if a message that has failed to be delivered by Bento has an error that matches it will be dropped (or delivered to a dead-letter queue if one exists). By default failed messages are nacked with requeue enabled. Default: []", "items": { "type": "string" }, "examples": [ [ "^reject me please:.+$" ] ] }, "prefetch_count": { "type": "integer", "description": "The maximum number of pending messages to have consumed at a time. Default: 10" }, "prefetch_size": { "type": "integer", "description": "The maximum amount of pending messages measured in bytes to have consumed at a time. Default: 0" }, "queue": { "type": "string", "description": "An AMQP queue to consume from." }, "queue_declare": { "type": "object", "description": "Allows you to passively declare the target queue. If the queue already exists then the declaration passively verifies that they match the target fields.", "properties": { "auto_delete": { "type": "boolean", "description": "Whether the declared queue will auto-delete. Default: false" }, "durable": { "type": "boolean", "description": "Whether the declared queue is durable. Default: true" }, "enabled": { "type": "boolean", "description": "Whether to enable queue declaration. Default: false" } } }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "urls": { "type": "array", "description": "A list of URLs to connect to. The first URL to successfully establish a connection will be used until the connection is closed. If an item of the list contains commas it will be expanded into multiple URLs.", "items": { "type": "string" }, "examples": [ [ "amqp://guest:guest@127.0.0.1:5672/" ], [ "amqp://127.0.0.1:5672/,amqp://127.0.0.2:5672/" ], [ "amqp://127.0.0.1:5672/", "amqp://127.0.0.2:5672/" ] ] } } }, "amqp_1": { "type": "object", "properties": { "azure_renew_lock": { "type": "boolean", "description": "Experimental: Azure service bus specific option to renew lock if processing takes more then configured lock time Default: false" }, "credit": { "type": "integer", "description": "Specifies the maximum number of unacknowledged messages the sender can transmit. Once this limit is reached, no more messages will arrive until messages are acknowledged and settled. Default: 64" }, "read_header": { "type": "boolean", "description": "Read additional message header fields into `amqp_*` metadata properties. Default: false" }, "sasl": { "type": "object", "description": "Enables SASL authentication.", "properties": { "mechanism": { "type": "string", "description": "The SASL authentication mechanism to use. Default: none" }, "password": { "type": "string", "description": "A SASL plain text password. It is recommended that you use environment variables to populate this field. Default: ", "examples": [ "${PASSWORD}" ] }, "user": { "type": "string", "description": "A SASL plain text username. It is recommended that you use environment variables to populate this field. Default: ", "examples": [ "${USER}" ] } } }, "source_address": { "type": "string", "description": "The source address to consume from.", "examples": [ "/foo", "queue:/bar", "topic:/baz" ] }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "url": { "type": "string", "description": "A URL to connect to.", "examples": [ "amqp://localhost:5672/", "amqps://guest:guest@localhost:5672/" ] }, "urls": { "type": "array", "description": "A list of URLs to connect to. The first URL to successfully establish a connection will be used until the connection is closed. If an item of the list contains commas it will be expanded into multiple URLs.", "items": { "type": "string" }, "examples": [ [ "amqp://guest:guest@127.0.0.1:5672/" ], [ "amqp://127.0.0.1:5672/,amqp://127.0.0.2:5672/" ], [ "amqp://127.0.0.1:5672/", "amqp://127.0.0.2:5672/" ] ] } } }, "aws_kinesis": { "type": "object", "properties": { "auto_replay_nacks": { "type": "boolean", "description": "Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation. Default: true" }, "batching": { "type": "object", "description": "\nAllows you to configure a [batching policy](/docs/configuration/batching).", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] }, "checkpoint_limit": { "type": "integer", "description": "The maximum gap between the in flight sequence versus the latest acknowledged sequence at a given time. Increasing this limit enables parallel processing and batching at the output level to work on individual shards. Any given sequence will not be committed unless all messages under that offset are delivered in order to preserve at least once delivery guarantees. Default: 1024" }, "commit_period": { "type": "string", "description": "The period of time between each update to the checkpoint table. Default: 5s" }, "credentials": { "type": "object", "description": "Optional manual configuration of AWS credentials to use. More information can be found [in this document](/docs/guides/cloud/aws).", "properties": { "expiry_window": { "type": "string", "description": "Allow the credentials to trigger refreshing prior to the credentials actually expiring. This is beneficial so race conditions with expiring credentials do not cause requests to fail. For example '10s' would refresh credentials ten seconds before expiration. Setting to a duration of `0` disables the expiry window. Default: " }, "from_ec2_role": { "type": "boolean", "description": "Use the credentials of a host EC2 machine configured to assume [an IAM role associated with the instance](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html). Default: false" }, "id": { "type": "string", "description": "The ID of credentials to use. Default: " }, "profile": { "type": "string", "description": "A profile from `~/.aws/credentials` to use. Default: " }, "role": { "type": "string", "description": "A role ARN to assume. Default: " }, "role_external_id": { "type": "string", "description": "An external ID to provide when assuming a role. Default: " }, "secret": { "type": "string", "description": "The secret for the credentials being used. Default: " }, "token": { "type": "string", "description": "The token for the credentials being used, required when using short term credentials. Default: " } } }, "dynamodb": { "type": "object", "description": "Determines the table used for storing and accessing the latest consumed sequence for shards, and for coordinating balanced consumers of streams.", "properties": { "billing_mode": { "type": "string", "description": "When creating the table determines the billing mode. Default: PAY_PER_REQUEST" }, "create": { "type": "boolean", "description": "Whether, if the table does not exist, it should be created. Default: false" }, "read_capacity_units": { "type": "integer", "description": "Set the provisioned read capacity when creating the table with a `billing_mode` of `PROVISIONED`. Default: 0" }, "table": { "type": "string", "description": "The name of the table to access. Default: " }, "write_capacity_units": { "type": "integer", "description": "Set the provisioned write capacity when creating the table with a `billing_mode` of `PROVISIONED`. Default: 0" } } }, "endpoint": { "type": "string", "description": "Allows you to specify a custom endpoint for the AWS API. Default: " }, "lease_period": { "type": "string", "description": "The period of time after which a client that has failed to update a shard checkpoint is assumed to be inactive. Default: 30s" }, "rebalance_period": { "type": "string", "description": "The period of time between each attempt to rebalance shards across clients. Default: 30s" }, "region": { "type": "string", "description": "The AWS region to target. Default: " }, "start_from_oldest": { "type": "boolean", "description": "Whether to consume from the oldest message when a sequence does not yet exist for the stream. Default: true" }, "streams": { "type": "array", "description": "One or more Kinesis data streams to consume from. Streams can either be specified by their name or full ARN. Shards of a stream are automatically balanced across consumers by coordinating through the provided DynamoDB table. Multiple comma separated streams can be listed in a single element. Shards are automatically distributed across consumers of a stream by coordinating through the provided DynamoDB table. Alternatively, it's possible to specify an explicit shard to consume from with a colon after the stream name, e.g. `foo:0` would consume the shard `0` of the stream `foo`.", "items": { "type": "string" }, "examples": [ [ "foo", "arn:aws:kinesis:*:111122223333:stream/my-stream" ] ] } } }, "aws_s3": { "type": "object", "properties": { "bucket": { "type": "string", "description": "The bucket to consume from. If the field `sqs.url` is specified this field is optional. Default: " }, "codec": { "type": "string", "description": "The way in which the bytes of a data source should be converted into discrete messages, codecs are useful for specifying how large files or continuous streams of data might be processed in small chunks rather than loading it all in memory. It's possible to consume lines using a custom delimiter with the `delim:x` codec, where x is the character sequence custom delimiter. Codecs can be chained with `/`, for example a gzip compressed CSV file can be consumed with the codec `gzip/csv`.", "examples": [ "lines", "delim:\t", "delim:foobar", "gzip/csv" ] }, "credentials": { "type": "object", "description": "Optional manual configuration of AWS credentials to use. More information can be found [in this document](/docs/guides/cloud/aws).", "properties": { "expiry_window": { "type": "string", "description": "Allow the credentials to trigger refreshing prior to the credentials actually expiring. This is beneficial so race conditions with expiring credentials do not cause requests to fail. For example '10s' would refresh credentials ten seconds before expiration. Setting to a duration of `0` disables the expiry window. Default: " }, "from_ec2_role": { "type": "boolean", "description": "Use the credentials of a host EC2 machine configured to assume [an IAM role associated with the instance](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html). Default: false" }, "id": { "type": "string", "description": "The ID of credentials to use. Default: " }, "profile": { "type": "string", "description": "A profile from `~/.aws/credentials` to use. Default: " }, "role": { "type": "string", "description": "A role ARN to assume. Default: " }, "role_external_id": { "type": "string", "description": "An external ID to provide when assuming a role. Default: " }, "secret": { "type": "string", "description": "The secret for the credentials being used. Default: " }, "token": { "type": "string", "description": "The token for the credentials being used, required when using short term credentials. Default: " } } }, "delete_objects": { "type": "boolean", "description": "Whether to delete downloaded objects from the bucket once they are processed. Default: false" }, "endpoint": { "type": "string", "description": "Allows you to specify a custom endpoint for the AWS API. Default: " }, "force_path_style_urls": { "type": "boolean", "description": "Forces the client API to use path style URLs for downloading keys, which is often required when connecting to custom endpoints. Default: false" }, "max_buffer": { "type": "integer", "description": " Default: 1000000" }, "prefix": { "type": "string", "description": "An optional path prefix, if set only objects with the prefix are consumed when walking a bucket. Default: " }, "region": { "type": "string", "description": "The AWS region to target. Default: " }, "scanner": { "type": "object", "description": "The [scanner](/docs/components/scanners/about) by which the stream of bytes consumed will be broken out into individual messages. Scanners are useful for processing large sources of data without holding the entirety of it within memory. For example, the `csv` scanner allows you to process individual CSV rows without loading the entire CSV file in memory at once. Default: map[to_the_end:map[]]" }, "sqs": { "type": "object", "description": "Consume SQS messages in order to trigger key downloads.", "properties": { "bucket_path": { "type": "string", "description": "A [dot path](/docs/configuration/field_paths) whereby the bucket name can be found in SQS messages. Default: Records.*.s3.bucket.name" }, "delay_period": { "type": "string", "description": "An optional period of time to wait from when a notification was originally sent to when the target key download is attempted. Default: ", "examples": [ "10s", "5m" ] }, "endpoint": { "type": "string", "description": "A custom endpoint to use when connecting to SQS. Default: " }, "envelope_path": { "type": "string", "description": "A [dot path](/docs/configuration/field_paths) of a field to extract an enveloped JSON payload for further extracting the key and bucket from SQS messages. This is specifically useful when subscribing an SQS queue to an SNS topic that receives bucket events. Default: ", "examples": [ "Message" ] }, "key_path": { "type": "string", "description": "A [dot path](/docs/configuration/field_paths) whereby object keys are found in SQS messages. Default: Records.*.s3.object.key" }, "max_messages": { "type": "integer", "description": "The maximum number of SQS messages to consume from each request. Default: 10" }, "url": { "type": "string", "description": "An optional SQS URL to connect to. When specified this queue will control which objects are downloaded. Default: " }, "wait_time_seconds": { "type": "integer", "description": "Whether to set the wait time. Enabling this activates long-polling. Valid values: 0 to 20. Default: 0" } } } } }, "aws_sqs": { "type": "object", "properties": { "credentials": { "type": "object", "description": "Optional manual configuration of AWS credentials to use. More information can be found [in this document](/docs/guides/cloud/aws).", "properties": { "expiry_window": { "type": "string", "description": "Allow the credentials to trigger refreshing prior to the credentials actually expiring. This is beneficial so race conditions with expiring credentials do not cause requests to fail. For example '10s' would refresh credentials ten seconds before expiration. Setting to a duration of `0` disables the expiry window. Default: " }, "from_ec2_role": { "type": "boolean", "description": "Use the credentials of a host EC2 machine configured to assume [an IAM role associated with the instance](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html). Default: false" }, "id": { "type": "string", "description": "The ID of credentials to use. Default: " }, "profile": { "type": "string", "description": "A profile from `~/.aws/credentials` to use. Default: " }, "role": { "type": "string", "description": "A role ARN to assume. Default: " }, "role_external_id": { "type": "string", "description": "An external ID to provide when assuming a role. Default: " }, "secret": { "type": "string", "description": "The secret for the credentials being used. Default: " }, "token": { "type": "string", "description": "The token for the credentials being used, required when using short term credentials. Default: " } } }, "custom_request_headers": { "type": "object", "description": "A map used to send custom HTTP headers alongside each SQS operation to AWS. Default: map[]" }, "delete_message": { "type": "boolean", "description": "Whether to delete the consumed message once it is acked. Disabling allows you to handle the deletion using a different mechanism. Default: true" }, "endpoint": { "type": "string", "description": "Allows you to specify a custom endpoint for the AWS API. Default: " }, "max_number_of_messages": { "type": "integer", "description": "The maximum number of messages to return on one poll. Valid values: 1 to 10. Default: 10" }, "region": { "type": "string", "description": "The AWS region to target. Default: " }, "reset_visibility": { "type": "boolean", "description": "Whether to set the visibility timeout of the consumed message to zero once it is nacked. Disabling honors the preset visibility timeout specified for the queue. Default: true" }, "update_visibility": { "type": "boolean", "description": "Whether to periodically refresh the visibility timeout of in-flight messages to prevent more-than-once delivery while still processing. Default: true" }, "url": { "type": "string", "description": "The SQS URL to consume from." }, "wait_time_seconds": { "type": "integer", "description": "Whether to set the wait time. Enabling this activates long-polling. Valid values: 0 to 20. Default: 0" } } }, "azure_blob_storage": { "type": "object", "properties": { "codec": { "type": "string", "description": "The way in which the bytes of a data source should be converted into discrete messages, codecs are useful for specifying how large files or continuous streams of data might be processed in small chunks rather than loading it all in memory. It's possible to consume lines using a custom delimiter with the `delim:x` codec, where x is the character sequence custom delimiter. Codecs can be chained with `/`, for example a gzip compressed CSV file can be consumed with the codec `gzip/csv`.", "examples": [ "lines", "delim:\t", "delim:foobar", "gzip/csv" ] }, "container": { "type": "string", "description": "The name of the container from which to download blobs." }, "delete_objects": { "type": "boolean", "description": "Whether to delete downloaded objects from the blob once they are processed. Default: false" }, "max_buffer": { "type": "integer", "description": " Default: 1000000" }, "prefix": { "type": "string", "description": "An optional path prefix, if set only objects with the prefix are consumed. Default: " }, "scanner": { "type": "object", "description": "The [scanner](/docs/components/scanners/about) by which the stream of bytes consumed will be broken out into individual messages. Scanners are useful for processing large sources of data without holding the entirety of it within memory. For example, the `csv` scanner allows you to process individual CSV rows without loading the entire CSV file in memory at once. Default: map[to_the_end:map[]]" }, "storage_access_key": { "type": "string", "description": "The storage account access key. This field is ignored if `storage_connection_string` is set. Default: " }, "storage_account": { "type": "string", "description": "The storage account to access. This field is ignored if `storage_connection_string` is set. Default: " }, "storage_connection_string": { "type": "string", "description": "A storage account connection string. This field is required if `storage_account` and `storage_access_key` / `storage_sas_token` are not set. Default: " }, "storage_sas_token": { "type": "string", "description": "The storage account SAS token. This field is ignored if `storage_connection_string` or `storage_access_key` are set. Default: " }, "targets_input": { "type": "object", "description": "EXPERIMENTAL: An optional source of download targets, configured as a [regular Bento input](/docs/components/inputs/about). Each message yielded by this input should be a single structured object containing a field `name`, which represents the blob to be downloaded.", "examples": [ { "mqtt": { "topics": [ "some-topic" ], "urls": [ "example.westeurope-1.ts.eventgrid.azure.net:8883" ] }, "processors": [ { "unarchive": { "format": "json_array" } }, { "mapping": "if this.eventType == \"Microsoft.Storage.BlobCreated\" {\n root.name = this.data.url.parse_url().path.trim_prefix(\"/foocontainer/\")\n} else {\n root = deleted()\n}" } ] } ] } } }, "azure_cosmosdb": { "type": "object", "properties": { "account_key": { "type": "string", "description": "Account key.", "examples": [ "C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==" ] }, "args_mapping": { "type": "string", "description": "A [Bloblang mapping](/docs/guides/bloblang/about) that, for each message, creates a list of arguments to use with the query.", "examples": [ "root = [\n { \"Name\": \"@name\", \"Value\": \"bento\" },\n]" ] }, "auto_replay_nacks": { "type": "boolean", "description": "Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation. Default: true" }, "batch_count": { "type": "integer", "description": "The maximum number of messages that should be accumulated into each batch. Use '-1' specify dynamic page size. Default: -1" }, "connection_string": { "type": "string", "description": "Connection string.", "examples": [ "AccountEndpoint=https://localhost:8081/;AccountKey=C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==;" ] }, "container": { "type": "string", "description": "Container.", "examples": [ "testcontainer" ] }, "database": { "type": "string", "description": "Database.", "examples": [ "testdb" ] }, "endpoint": { "type": "string", "description": "CosmosDB endpoint.", "examples": [ "https://localhost:8081" ] }, "partition_keys_map": { "type": "string", "description": "A [Bloblang mapping](/docs/guides/bloblang/about) which should evaluate to a single partition key value or an array of partition key values of type string, integer or boolean. Currently, hierarchical partition keys are not supported so only one value may be provided.", "examples": [ "root = \"blobfish\"", "root = 41", "root = true", "root = null", "root = now().ts_format(\"2006-01-02\")" ] }, "query": { "type": "string", "description": "The query to execute", "examples": [ "SELECT c.foo FROM testcontainer AS c WHERE c.bar = \"baz\" AND c.timestamp \u003c @timestamp" ] } } }, "azure_queue_storage": { "type": "object", "properties": { "dequeue_visibility_timeout": { "type": "string", "description": "The timeout duration until a dequeued message gets visible again, 30s by default Default: 30s" }, "max_in_flight": { "type": "integer", "description": "The maximum number of unprocessed messages to fetch at a given time. Default: 10" }, "queue_name": { "type": "string", "description": "The name of the source storage queue.", "examples": [ "foo_queue", "${! env(\"MESSAGE_TYPE\").lowercase() }" ] }, "storage_access_key": { "type": "string", "description": "The storage account access key. This field is ignored if `storage_connection_string` is set. Default: " }, "storage_account": { "type": "string", "description": "The storage account to access. This field is ignored if `storage_connection_string` is set. Default: " }, "storage_connection_string": { "type": "string", "description": "A storage account connection string. This field is required if `storage_account` and `storage_access_key` / `storage_sas_token` are not set. Default: " }, "storage_sas_token": { "type": "string", "description": " Default: " }, "track_properties": { "type": "boolean", "description": "If set to `true` the queue is polled on each read request for information such as the queue message lag. These properties are added to consumed messages as metadata, but will also have a negative performance impact. Default: false" } } }, "azure_service_bus_queue": { "type": "object", "properties": { "auto_ack": { "type": "boolean", "description": "Whether to automatically acknowledge messages as they are consumed rather than waiting for acknowledgments from downstream. This can improve throughput but at the cost of eliminating delivery guarantees. Default: false" }, "connection_string": { "type": "string", "description": "The Service Bus connection string. This can be obtained from the Azure portal. If not provided, namespace and default credentials will be used. Default: ", "examples": [ "Endpoint=sb://example.servicebus.windows.net/;SharedAccessKeyName=RootManageSharedAccessKey;SharedAccessKey=..." ] }, "max_in_flight": { "type": "integer", "description": "The maximum number of unprocessed messages to fetch at a given time. Default: 10" }, "nack_reject_patterns": { "type": "array", "description": "A list of regular expression patterns whereby if a message that has failed to be delivered by Bento has an error that matches it will be nacked (or sent to dead letter queue if configured). By default failed messages are nacked with requeue enabled. Default: []", "items": { "type": "string" }, "examples": [ [ "^reject me please:.+$" ] ] }, "namespace": { "type": "string", "description": "The Service Bus namespace. Required when connection_string is not provided. Default: ", "examples": [ "myservicebus.servicebus.windows.net" ] }, "queue": { "type": "string", "description": "The name of the Service Bus queue to consume from." }, "renew_lock": { "type": "boolean", "description": "Automatically renew message locks to prevent lock expiration during processing. Useful for long-running message processing. Default: true" } } }, "azure_table_storage": { "type": "object", "properties": { "filter": { "type": "string", "description": "OData filter expression. Is not set all rows are returned. Valid operators are `eq, ne, gt, lt, ge and le` Default: ", "examples": [ "PartitionKey eq 'foo' and RowKey gt '1000'" ] }, "page_size": { "type": "integer", "description": "Maximum number of records to return on each page. Default: 1000" }, "select": { "type": "string", "description": "Select expression using OData notation. Limits the columns on each record to just those requested. Default: ", "examples": [ "PartitionKey,RowKey,Foo,Bar,Timestamp" ] }, "storage_access_key": { "type": "string", "description": "The storage account access key. This field is ignored if `storage_connection_string` is set. Default: " }, "storage_account": { "type": "string", "description": "The storage account to access. This field is ignored if `storage_connection_string` is set. Default: " }, "storage_connection_string": { "type": "string", "description": "A storage account connection string. This field is required if `storage_account` and `storage_access_key` / `storage_sas_token` are not set. Default: " }, "storage_sas_token": { "type": "string", "description": "The storage account SAS token. This field is ignored if `storage_connection_string` or `storage_access_key` are set. Default: " }, "table_name": { "type": "string", "description": "The table to read messages from.", "examples": [ "Foo" ] } } }, "batched": { "type": "object", "properties": { "child": { "type": "object", "description": "The child input." }, "policy": { "type": "object", "description": "\nAllows you to configure a [batching policy](/docs/configuration/batching).", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] } } }, "beanstalkd": { "type": "object", "properties": { "address": { "type": "string", "description": "An address to connect to.", "examples": [ "127.0.0.1:11300" ] } } }, "broker": { "type": "object", "properties": { "batching": { "type": "object", "description": "\nAllows you to configure a [batching policy](/docs/configuration/batching).", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] }, "copies": { "type": "integer", "description": "Whatever is specified within `inputs` will be created this many times. Default: 1" }, "inputs": { "type": "array", "description": "A list of inputs to create.", "items": { "type": "object" } } } }, "cassandra": { "type": "object", "properties": { "addresses": { "type": "array", "description": "A list of Cassandra nodes to connect to. Multiple comma separated addresses can be specified on a single line.", "items": { "type": "string" }, "examples": [ [ "localhost:9042" ], [ "foo:9042", "bar:9042" ], [ "foo:9042,bar:9042" ] ] }, "auto_replay_nacks": { "type": "boolean", "description": "Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation. Default: true" }, "backoff": { "type": "object", "description": "Control time intervals between retry attempts.", "properties": { "initial_interval": { "type": "string", "description": "The initial period to wait between retry attempts. Default: 1s" }, "max_interval": { "type": "string", "description": "The maximum period to wait between retry attempts. Default: 5s" } } }, "disable_initial_host_lookup": { "type": "boolean", "description": "If enabled the driver will not attempt to get host info from the system.peers table. This can speed up queries but will mean that data_centre, rack and token information will not be available. Default: false" }, "max_retries": { "type": "integer", "description": "The maximum number of retries before giving up on a request. Default: 3" }, "password_authenticator": { "type": "object", "description": "Optional configuration of Cassandra authentication parameters.", "properties": { "enabled": { "type": "boolean", "description": "Whether to use password authentication Default: false" }, "password": { "type": "string", "description": "The password to authenticate with. Default: " }, "username": { "type": "string", "description": "The username to authenticate as. Default: " } } }, "query": { "type": "string", "description": "A query to execute." }, "timeout": { "type": "string", "description": "The client connection timeout. Default: 600ms" }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } } } }, "cockroachdb_changefeed": { "type": "object", "properties": { "auto_replay_nacks": { "type": "boolean", "description": "Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation. Default: true" }, "cursor_cache": { "type": "string", "description": "A [cache resource](https://warpstreamlabs.github.io/bento/docs/components/caches/about) to use for storing the current latest cursor that has been successfully delivered, this allows Bento to continue from that cursor upon restart, rather than consume the entire state of the table." }, "dsn": { "type": "string", "description": "A Data Source Name to identify the target database.", "examples": [ "postgres://user:password@example.com:26257/defaultdb?sslmode=require" ] }, "options": { "type": "array", "description": "A list of options to be included in the changefeed (WITH X, Y...).\n**NOTE: Both the CURSOR option and UPDATED will be ignored from these options when a `cursor_cache` is specified, as they are set explicitly by Bento in this case.**", "items": { "type": "string" }, "examples": [ [ "virtual_columns=\"omitted\"" ] ] }, "tables": { "type": "array", "description": "CSV of tables to be included in the changefeed", "items": { "type": "string" }, "examples": [ [ "table1", "table2" ] ] }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } } } }, "csv": { "type": "object", "properties": { "auto_replay_nacks": { "type": "boolean", "description": "Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation. Default: true" }, "batch_count": { "type": "integer", "description": "Optionally process records in batches. This can help to speed up the consumption of exceptionally large CSV files. When the end of the file is reached the remaining records are processed as a (potentially smaller) batch. Default: 1" }, "delete_on_finish": { "type": "boolean", "description": "Whether to delete input files from the disk once they are fully consumed. Default: false" }, "delimiter": { "type": "string", "description": "The delimiter to use for splitting values in each record. It must be a single character. Default: ," }, "expected_headers": { "type": "array", "description": "An optional list of expected headers in the header row. If provided, the scanner will check the file contents and emit an error if any expected headers don't match.", "items": { "type": "string" }, "examples": [ [ "first_name", "last_name", "age" ] ] }, "expected_number_of_fields": { "type": "integer", "description": "The number of expected fields in the csv file." }, "lazy_quotes": { "type": "boolean", "description": "If set to `true`, a quote may appear in an unquoted field and a non-doubled quote may appear in a quoted field. Default: false" }, "parse_header_row": { "type": "boolean", "description": "Whether to reference the first row as a header row. If set to true the output structure for messages will be an object where field keys are determined by the header row. Otherwise, each message will consist of an array of values from the corresponding CSV row. Default: true" }, "paths": { "type": "array", "description": "A list of file paths to read from. Each file will be read sequentially until the list is exhausted, at which point the input will close. Glob patterns are supported, including super globs (double star).", "items": { "type": "string" }, "examples": [ [ "/tmp/foo.csv", "/tmp/bar/*.csv", "/tmp/data/**/*.csv" ] ] } } }, "cypher": { "type": "object", "properties": { "auto_replay_nacks": { "type": "boolean", "description": "Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation. Default: true" }, "basic_auth": { "type": "object", "description": "Basic Authentication fields", "properties": { "password": { "type": "string", "description": "The password for basic auth. Default: " }, "realm": { "type": "string", "description": "The realm for basic auth. Default: " }, "user": { "type": "string", "description": "The username for basic auth. Default: " } } }, "database": { "type": "string", "description": "The name of the database to connect to.", "examples": [ "neo4j" ] }, "no_auth": { "type": "boolean", "description": "Set to true to connect without authentication. Default: false" }, "query": { "type": "string", "description": "The cypher query to execute.", "examples": [ "MATCH (n) RETURN n" ] }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "uri": { "type": "string", "description": "The URL of the database engine.", "examples": [ "bolt://localhost:7687" ] } } }, "discord": { "type": "object", "properties": { "auto_replay_nacks": { "type": "boolean", "description": "Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation. Default: true" }, "bot_token": { "type": "string", "description": "A bot token used for authentication." }, "cache": { "type": "string", "description": "A cache resource to use for performing unread message backfills, the ID of the last message received will be stored in this cache and used for subsequent requests." }, "cache_key": { "type": "string", "description": "The key identifier used when storing the ID of the last message received. Default: last_message_id" }, "channel_id": { "type": "string", "description": "A discord channel ID to consume messages from." }, "limit": { "type": "integer", "description": "The maximum number of messages to receive in a single request. Default: 100" }, "poll_period": { "type": "string", "description": "The length of time (as a duration string) to wait between each poll for backlogged messages. This field can be set empty, in which case requests are made at the limit set by the rate limit. This field also supports cron expressions. Default: 1m" }, "rate_limit": { "type": "string", "description": " Default: An optional rate limit resource to restrict API requests with." } } }, "dynamic": { "type": "object", "properties": { "inputs": { "type": "object", "description": "A map of inputs to statically create. Default: map[]" }, "prefix": { "type": "string", "description": "A path prefix for HTTP endpoints that are registered. Default: " } } }, "etcd": { "type": "object", "properties": { "auth": { "type": "object", "description": "Optional configuration of etcd authentication headers.", "properties": { "enabled": { "type": "boolean", "description": "Whether to use password authentication Default: false" }, "password": { "type": "string", "description": "The password to authenticate with. Default: " }, "username": { "type": "string", "description": "The username to authenticate as. Default: " } } }, "auto_replay_nacks": { "type": "boolean", "description": "Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation. Default: true" }, "auto_sync_interval": { "type": "string", "description": "The interval to update endpoints with its latest members. 0 disables auto-sync. By default auto-sync is disabled." }, "backoff_jitter_fraction": { "type": "number", "description": "The jitter fraction to randomize backoff wait time." }, "backoff_wait_between": { "type": "string", "description": "The wait time before retrying an RPC." }, "dial_timeout": { "type": "string", "description": "Timeout for failing to establish a connection. Default: 5s" }, "endpoints": { "type": "array", "description": "A set of URLs (schemes, hosts and ports only) that can be used to communicate with a logical etcd cluster. If multiple endpoints are provided, the Client will attempt to use them all in the event that one or more of them are unusable.", "items": { "type": "string" }, "examples": [ [ "etcd://:2379" ], [ "etcd://localhost:2379" ], [ "etcd://localhost:2379", "etcd://localhost:22379", "etcd://localhost:32379" ] ] }, "keep_alive_time": { "type": "string", "description": "Time after which client pings the server to see if transport is alive. Default: 5s" }, "keep_alive_timeout": { "type": "string", "description": "Time that the client waits for a response for the keep-alive probe. If the response is not received in this time, the connection is closed. Default: 1s" }, "key": { "type": "string", "description": "The key or prefix being watched. For prefix watching, options.with_prefix should be `true`" }, "max_call_recv_msg_size": { "type": "integer", "description": "The client-side response receive limit. If 0, it defaults to math.MaxInt32." }, "max_call_send_msg_size": { "type": "integer", "description": "The client-side request send limit in bytes. If 0, it defaults to 2.0 MiB (2 * 1024 * 1024)." }, "max_unary_retries": { "type": "integer", "description": "The maximum number of retries for unary RPCs." }, "options": { "type": "object", "description": "Collection of options to configure an etcd watcher.", "properties": { "with_created_notify": { "type": "boolean", "description": "Whether to send CREATED notify events to the watcher. Default: false" }, "with_delete_filter": { "type": "boolean", "description": "Whether to discard DELETE events from the watcher. Default: false" }, "with_prefix": { "type": "boolean", "description": "Whether to watch for events on a prefix. Default: false" }, "with_progress_notify": { "type": "boolean", "description": "Whether to send periodic progress updates every 10 minutes when there is no incoming events. Default: false" }, "with_put_filter": { "type": "boolean", "description": "Whether to discard PUT events from the watcher. Default: false" }, "with_range": { "type": "string", "description": "Will cause the watcher to return a range of lexicographically sorted keys to return in the form `[key, end)` where `end` is the passed parameter. Default: " } } }, "permit_without_stream": { "type": "boolean", "description": "When set, will allow client to send keepalive pings to server without any active streams (RPCs). Default: false" }, "reject_old_cluster": { "type": "boolean", "description": "When set, will refuse to create a client against an outdated cluster. Default: false" }, "request_timeout": { "type": "string", "description": "Timeout for a single request. This includes connection time, any redirects, and header wait time. Default: 1s" }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } } } }, "file": { "type": "object", "properties": { "auto_replay_nacks": { "type": "boolean", "description": "Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation. Default: true" }, "codec": { "type": "string", "description": "The way in which the bytes of a data source should be converted into discrete messages, codecs are useful for specifying how large files or continuous streams of data might be processed in small chunks rather than loading it all in memory. It's possible to consume lines using a custom delimiter with the `delim:x` codec, where x is the character sequence custom delimiter. Codecs can be chained with `/`, for example a gzip compressed CSV file can be consumed with the codec `gzip/csv`.", "examples": [ "lines", "delim:\t", "delim:foobar", "gzip/csv" ] }, "delete_on_finish": { "type": "boolean", "description": "Whether to delete input files from the disk once they are fully consumed. Default: false" }, "max_buffer": { "type": "integer", "description": " Default: 1000000" }, "paths": { "type": "array", "description": "A list of paths to consume sequentially. Glob patterns are supported, including super globs (double star).", "items": { "type": "string" } }, "scanner": { "type": "object", "description": "The [scanner](/docs/components/scanners/about) by which the stream of bytes consumed will be broken out into individual messages. Scanners are useful for processing large sources of data without holding the entirety of it within memory. For example, the `csv` scanner allows you to process individual CSV rows without loading the entire CSV file in memory at once. Default: map[lines:map[]]" } } }, "gcp_bigquery_select": { "type": "object", "properties": { "args_mapping": { "type": "string", "description": "An optional [Bloblang mapping](/docs/guides/bloblang/about) which should evaluate to an array of values matching in size to the number of placeholder arguments in the field `where`.", "examples": [ "root = [ \"article\", now().ts_format(\"2006-01-02\") ]" ] }, "auto_replay_nacks": { "type": "boolean", "description": "Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation. Default: true" }, "columns": { "type": "array", "description": "A list of columns to query.", "items": { "type": "string" } }, "job_labels": { "type": "object", "description": "A list of labels to add to the query job. Default: map[]" }, "prefix": { "type": "string", "description": "An optional prefix to prepend to the select query (before SELECT)." }, "priority": { "type": "string", "description": "The priority with which to schedule the query. Default: " }, "project": { "type": "string", "description": "GCP project where the query job will execute." }, "suffix": { "type": "string", "description": "An optional suffix to append to the select query." }, "table": { "type": "string", "description": "Fully-qualified BigQuery table name to query.", "examples": [ "bigquery-public-data.samples.shakespeare" ] }, "where": { "type": "string", "description": "An optional where clause to add. Placeholder arguments are populated with the `args_mapping` field. Placeholders should always be question marks (`?`).", "examples": [ "type = ? and created_at \u003e ?", "user_id = ?" ] } } }, "gcp_cloud_storage": { "type": "object", "properties": { "bucket": { "type": "string", "description": "The name of the bucket from which to download objects." }, "codec": { "type": "string", "description": "The way in which the bytes of a data source should be converted into discrete messages, codecs are useful for specifying how large files or continuous streams of data might be processed in small chunks rather than loading it all in memory. It's possible to consume lines using a custom delimiter with the `delim:x` codec, where x is the character sequence custom delimiter. Codecs can be chained with `/`, for example a gzip compressed CSV file can be consumed with the codec `gzip/csv`.", "examples": [ "lines", "delim:\t", "delim:foobar", "gzip/csv" ] }, "delete_objects": { "type": "boolean", "description": "Whether to delete downloaded objects from the bucket once they are processed. Default: false" }, "max_buffer": { "type": "integer", "description": " Default: 1000000" }, "prefix": { "type": "string", "description": "An optional path prefix, if set only objects with the prefix are consumed. Default: " }, "scanner": { "type": "object", "description": "The [scanner](/docs/components/scanners/about) by which the stream of bytes consumed will be broken out into individual messages. Scanners are useful for processing large sources of data without holding the entirety of it within memory. For example, the `csv` scanner allows you to process individual CSV rows without loading the entire CSV file in memory at once. Default: map[to_the_end:map[]]" } } }, "gcp_pubsub": { "type": "object", "properties": { "create_subscription": { "type": "object", "description": "Allows you to configure the input subscription and creates if it doesn't exist.", "properties": { "enabled": { "type": "boolean", "description": "Whether to configure subscription or not. Default: false" }, "topic": { "type": "string", "description": "Defines the topic that the subscription should be vinculated to. Default: " } } }, "endpoint": { "type": "string", "description": "An optional endpoint to override the default of `pubsub.googleapis.com:443`. This can be used to connect to a region specific pubsub endpoint. For a list of valid values check out [this document.](https://cloud.google.com/pubsub/docs/reference/service_apis_overview#list_of_regional_endpoints) Default: ", "examples": [ "us-central1-pubsub.googleapis.com:443", "us-west3-pubsub.googleapis.com:443" ] }, "max_outstanding_bytes": { "type": "integer", "description": "The maximum number of outstanding pending messages to be consumed measured in bytes. Default: 1e+09" }, "max_outstanding_messages": { "type": "integer", "description": "The maximum number of outstanding pending messages to be consumed at a given time. Default: 1000" }, "project": { "type": "string", "description": "The project ID of the target subscription." }, "subscription": { "type": "string", "description": "The target subscription ID." }, "sync": { "type": "boolean", "description": "Enable synchronous pull mode. Default: false" } } }, "gcp_spanner_cdc": { "type": "object", "properties": { "end_time": { "type": "string", "description": "An optional field to define the end time to read from the changestreams, timestamp format should conform to RFC3339", "examples": [ "2006-01-02T15:04:05Z07:00" ] }, "heartbeat_interval": { "type": "string", "description": "An optional field to configure the heartbeat interval for partitions. Default: 3s" }, "prefetch_count": { "type": "integer", "description": "The maximum number of messages to have pulled in. Default: 1024", "examples": [ 1024 ] }, "spanner_dsn": { "type": "string", "description": "The dsn for spanner from where to read the changestream.", "examples": [ "projects/{projectId}/instances/{instanceId}/databases/{databaseName}" ] }, "start_time": { "type": "string", "description": "An optional field to define the start point to read from the changestreams. If not set then the current time is used. The timestamp format should conform to RFC3339, for details on valid start times please see [this document](https://cloud.google.com/spanner/docs/change-streams#data-retention)", "examples": [ "2006-01-02T15:04:05Z07:00" ] }, "stream_name": { "type": "string", "description": "The name of the stream to track changes on." } } }, "generate": { "type": "object", "properties": { "auto_replay_nacks": { "type": "boolean", "description": "Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation. Default: true" }, "batch_size": { "type": "integer", "description": "The number of generated messages that should be accumulated into each batch flushed at the specified interval. Default: 1" }, "count": { "type": "integer", "description": "An optional number of messages to generate, if set above 0 the specified number of messages is generated and then the input will shut down. Default: 0" }, "interval": { "type": "string", "description": "The time interval at which messages should be generated, expressed either as a duration string or as a cron expression. If set to an empty string messages will be generated as fast as downstream services can process them. Cron expressions can specify a timezone by prefixing the expression with `TZ=\u003clocation name\u003e`, where the location name corresponds to a file within the IANA Time Zone database. Default: 1s", "examples": [ "5s", "1m", "1h", "@every 1s", "0,30 */2 * * * *", "TZ=Europe/London 30 3-6,20-23 * * *" ] }, "mapping": { "type": "string", "description": "A [bloblang](/docs/guides/bloblang/about) mapping to use for generating messages.", "examples": [ "root = \"hello world\"", "root = {\"test\":\"message\",\"id\":uuid_v4()}" ] } } }, "hdfs": { "type": "object", "properties": { "directory": { "type": "string", "description": "The directory to consume from." }, "hosts": { "type": "array", "description": "A list of target host addresses to connect to.", "items": { "type": "string" }, "examples": [ "localhost:9000" ] }, "user": { "type": "string", "description": "A user ID to connect as. Default: " } } }, "http_client": { "type": "object", "properties": { "auto_replay_nacks": { "type": "boolean", "description": "Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation. Default: true" }, "backoff_on": { "type": "integer", "description": "A list of status codes whereby the request should be considered to have failed and retries should be attempted, but the period between them should be increased gradually. Default: [429]", "items": { "type": "integer" } }, "basic_auth": { "type": "object", "description": "Allows you to specify basic authentication.", "properties": { "enabled": { "type": "boolean", "description": "Whether to use basic authentication in requests. Default: false" }, "password": { "type": "string", "description": "A password to authenticate with. Default: " }, "username": { "type": "string", "description": "A username to authenticate as. Default: " } } }, "drop_empty_bodies": { "type": "boolean", "description": "Whether empty payloads received from the target server should be dropped. Default: true" }, "drop_on": { "type": "integer", "description": "A list of status codes whereby the request should be considered to have failed but retries should not be attempted. This is useful for preventing wasted retries for requests that will never succeed. Note that with these status codes the _request_ is dropped, but _message_ that caused the request will not be dropped. Default: []", "items": { "type": "integer" } }, "dump_request_log_level": { "type": "string", "description": "EXPERIMENTAL: Optionally set a level at which the request and response payload of each request made will be logged. Default: " }, "extract_headers": { "type": "object", "description": "Specify which response headers should be added to resulting messages as metadata. Header keys are lowercased before matching, so ensure that your patterns target lowercased versions of the header keys that you expect.", "properties": { "include_patterns": { "type": "array", "description": "Provide a list of explicit metadata key regular expression (re2) patterns to match against. Default: []", "items": { "type": "string" }, "examples": [ [ ".*" ], [ "_timestamp_unix$" ] ] }, "include_prefixes": { "type": "array", "description": "Provide a list of explicit metadata key prefixes to match against. Default: []", "items": { "type": "string" }, "examples": [ [ "foo_", "bar_" ], [ "kafka_" ], [ "content-" ] ] } } }, "headers": { "type": "object", "description": "A map of headers to add to the request. Default: map[]", "examples": [ { "Content-Type": "application/octet-stream", "traceparent": "${! tracing_span().traceparent }" } ] }, "jwt": { "type": "object", "description": "BETA: Allows you to specify JWT authentication.", "properties": { "claims": { "type": "object", "description": "A value used to identify the claims that issued the JWT. Default: map[]" }, "enabled": { "type": "boolean", "description": "Whether to use JWT authentication in requests. Default: false" }, "headers": { "type": "object", "description": "Add optional key/value headers to the JWT. Default: map[]" }, "private_key_file": { "type": "string", "description": "A file with the PEM encoded via PKCS1 or PKCS8 as private key. Default: " }, "signing_method": { "type": "string", "description": "A method used to sign the token such as RS256, RS384, RS512 or EdDSA. Default: " } } }, "max_retry_backoff": { "type": "string", "description": "The maximum period to wait between failed requests. Default: 300s" }, "metadata": { "type": "object", "description": "Specify optional matching rules to determine which metadata keys should be added to the HTTP request as headers.", "properties": { "include_patterns": { "type": "array", "description": "Provide a list of explicit metadata key regular expression (re2) patterns to match against. Default: []", "items": { "type": "string" }, "examples": [ [ ".*" ], [ "_timestamp_unix$" ] ] }, "include_prefixes": { "type": "array", "description": "Provide a list of explicit metadata key prefixes to match against. Default: []", "items": { "type": "string" }, "examples": [ [ "foo_", "bar_" ], [ "kafka_" ], [ "content-" ] ] } } }, "oauth": { "type": "object", "description": "Allows you to specify open authentication via OAuth version 1.", "properties": { "access_token": { "type": "string", "description": "A value used to gain access to the protected resources on behalf of the user. Default: " }, "access_token_secret": { "type": "string", "description": "A secret provided in order to establish ownership of a given access token. Default: " }, "consumer_key": { "type": "string", "description": "A value used to identify the client to the service provider. Default: " }, "consumer_secret": { "type": "string", "description": "A secret used to establish ownership of the consumer key. Default: " }, "enabled": { "type": "boolean", "description": "Whether to use OAuth version 1 in requests. Default: false" } } }, "oauth2": { "type": "object", "description": "Allows you to specify open authentication via OAuth version 2 using the client credentials token flow.", "properties": { "client_key": { "type": "string", "description": "A value used to identify the client to the token provider. Default: " }, "client_secret": { "type": "string", "description": "A secret used to establish ownership of the client key. Default: " }, "enabled": { "type": "boolean", "description": "Whether to use OAuth version 2 in requests. Default: false" }, "endpoint_params": { "type": "object", "description": "A list of optional endpoint parameters, values should be arrays of strings. Default: map[]", "examples": [ { "bar": [ "woof" ], "foo": [ "meow", "quack" ] } ] }, "scopes": { "type": "array", "description": "A list of optional requested permissions. Default: []", "items": { "type": "string" } }, "token_url": { "type": "string", "description": "The URL of the token provider. Default: " } } }, "payload": { "type": "string", "description": "An optional payload to deliver for each request." }, "proxy_url": { "type": "string", "description": "An optional HTTP proxy URL." }, "rate_limit": { "type": "string", "description": "An optional [rate limit](/docs/components/rate_limits/about) to throttle requests by." }, "retries": { "type": "integer", "description": "The maximum number of retry attempts to make. Default: 3" }, "retry_period": { "type": "string", "description": "The base period to wait between failed requests. Default: 1s" }, "stream": { "type": "object", "description": "Allows you to set streaming mode, where requests are kept open and messages are processed line-by-line.", "properties": { "codec": { "type": "string", "description": "The way in which the bytes of a data source should be converted into discrete messages, codecs are useful for specifying how large files or continuous streams of data might be processed in small chunks rather than loading it all in memory. It's possible to consume lines using a custom delimiter with the `delim:x` codec, where x is the character sequence custom delimiter. Codecs can be chained with `/`, for example a gzip compressed CSV file can be consumed with the codec `gzip/csv`.", "examples": [ "lines", "delim:\t", "delim:foobar", "gzip/csv" ] }, "enabled": { "type": "boolean", "description": "Enables streaming mode. Default: false" }, "max_buffer": { "type": "integer", "description": " Default: 1000000" }, "reconnect": { "type": "boolean", "description": "Sets whether to re-establish the connection once it is lost. Default: true" }, "scanner": { "type": "object", "description": "The [scanner](/docs/components/scanners/about) by which the stream of bytes consumed will be broken out into individual messages. Scanners are useful for processing large sources of data without holding the entirety of it within memory. For example, the `csv` scanner allows you to process individual CSV rows without loading the entire CSV file in memory at once. Default: map[lines:map[]]" } } }, "successful_on": { "type": "integer", "description": "A list of status codes whereby the attempt should be considered successful, this is useful for dropping requests that return non-2XX codes indicating that the message has been dealt with, such as a 303 See Other or a 409 Conflict. All 2XX codes are considered successful unless they are present within `backoff_on` or `drop_on`, regardless of this field. Default: []", "items": { "type": "integer" } }, "timeout": { "type": "string", "description": "A static timeout to apply to requests. Default: 5s" }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "transport": { "type": "object", "description": "Custom transport options.", "properties": { "dial_context": { "type": "object", "description": "Settings for the dialer used to create new connections.", "properties": { "keep_alive": { "type": "string", "description": "Keep-alive period for active network connections used by the dialer. Default: 30s" }, "timeout": { "type": "string", "description": "Timeout for establishing new network connections. Default: 30s" } } }, "expect_continue_timeout": { "type": "string", "description": "Time to wait for a server's first response headers after sending request headers when 'Expect: 100-continue' is used. Zero means send body immediately. Default: 1s" }, "force_http2": { "type": "boolean", "description": "If true, the transport will attempt to use HTTP/2. Default: true" }, "idle_connection_timeout": { "type": "string", "description": "Maximum time an idle keep-alive connection remains open before closing itself. Default: 90s" }, "max_idle_connections": { "type": "integer", "description": "Maximum number of idle keep-alive connections. Zero = unlimited. Default: 100" }, "tls_handshake_timeout": { "type": "string", "description": "Maximum time allowed for TLS handshake to complete. Default: 10s" } } }, "url": { "type": "string", "description": "The URL to connect to." }, "verb": { "type": "string", "description": "A verb to connect with Default: GET", "examples": [ "POST", "GET", "DELETE" ] } } }, "http_server": { "type": "object", "properties": { "address": { "type": "string", "description": "An alternative address to host from. If left empty the service wide address is used. Default: " }, "allowed_verbs": { "type": "array", "description": "An array of verbs that are allowed for the `path` endpoint. Default: [POST]", "items": { "type": "string" } }, "cert_file": { "type": "string", "description": "Enable TLS by specifying a certificate and key file. Only valid with a custom `address`. Default: " }, "cors": { "type": "object", "description": "Adds Cross-Origin Resource Sharing headers. Only valid with a custom `address`.", "properties": { "allowed_headers": { "type": "array", "description": "Appends additional headers to the list of default allowed headers: Accept, Accept-Language, Content-Language \u0026 Origin. These default headers are therefore always allowed. Default: []", "items": { "type": "string" } }, "allowed_methods": { "type": "array", "description": "Used to explicitly set allowed methods in the Access-Control-Allow-Methods header. Default: [GET HEAD POST PUT PATCH DELETE]", "items": { "type": "string" } }, "allowed_origins": { "type": "array", "description": "An explicit list of origins that are allowed for CORS requests. Default: []", "items": { "type": "string" } }, "enabled": { "type": "boolean", "description": "Whether to allow CORS requests. Default: false" } } }, "key_file": { "type": "string", "description": "Enable TLS by specifying a certificate and key file. Only valid with a custom `address`. Default: " }, "path": { "type": "string", "description": "The endpoint path to listen for POST requests. Default: /post" }, "rate_limit": { "type": "string", "description": "An optional [rate limit](/docs/components/rate_limits/about) to throttle requests by. Default: " }, "sync_response": { "type": "object", "description": "Customise messages returned via [synchronous responses](/docs/guides/sync_responses).", "properties": { "headers": { "type": "object", "description": "Specify headers to return with synchronous responses. Default: map[Content-Type:application/octet-stream]" }, "metadata_headers": { "type": "object", "description": "Specify criteria for which metadata values are added to the response as headers.", "properties": { "include_patterns": { "type": "array", "description": "Provide a list of explicit metadata key regular expression (re2) patterns to match against. Default: []", "items": { "type": "string" }, "examples": [ [ ".*" ], [ "_timestamp_unix$" ] ] }, "include_prefixes": { "type": "array", "description": "Provide a list of explicit metadata key prefixes to match against. Default: []", "items": { "type": "string" }, "examples": [ [ "foo_", "bar_" ], [ "kafka_" ], [ "content-" ] ] } } }, "status": { "type": "string", "description": "Specify the status code to return with synchronous responses. This is a string value, which allows you to customize it based on resulting payloads and their metadata. Default: 200", "examples": [ "${! json(\"status\") }", "${! metadata(\"status\").string() }" ] } } }, "timeout": { "type": "string", "description": "Timeout for requests. If a consumed messages takes longer than this to be delivered the connection is closed, but the message may still be delivered. Default: 5s" }, "ws_path": { "type": "string", "description": "The endpoint path to create websocket connections from. Default: /post/ws" }, "ws_rate_limit_message": { "type": "string", "description": "An optional message to delivery to websocket connections that are rate limited. Default: " }, "ws_welcome_message": { "type": "string", "description": "An optional message to deliver to fresh websocket connections. Default: " } } }, "inproc": { "type": "string", "description": " Default: " }, "kafka": { "type": "object", "properties": { "addresses": { "type": "array", "description": "A list of broker addresses to connect to. If an item of the list contains commas it will be expanded into multiple addresses.", "items": { "type": "string" }, "examples": [ [ "localhost:9092" ], [ "localhost:9041,localhost:9042" ], [ "localhost:9041", "localhost:9042" ] ] }, "auto_replay_nacks": { "type": "boolean", "description": "Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation. Default: true" }, "batching": { "type": "object", "description": "\nAllows you to configure a [batching policy](/docs/configuration/batching).", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] }, "checkpoint_limit": { "type": "integer", "description": "The maximum number of messages of the same topic and partition that can be processed at a given time. Increasing this limit enables parallel processing and batching at the output level to work on individual partitions. Any given offset will not be committed unless all messages under that offset are delivered in order to preserve at least once delivery guarantees. Default: 1024" }, "client_id": { "type": "string", "description": "An identifier for the client connection. Default: bento" }, "commit_period": { "type": "string", "description": "The period of time between each commit of the current partition offsets. Offsets are always committed during shutdown. Default: 1s" }, "consumer_group": { "type": "string", "description": "An identifier for the consumer group of the connection. This field can be explicitly made empty in order to disable stored offsets for the consumed topic partitions. Default: " }, "extract_tracing_map": { "type": "string", "description": "EXPERIMENTAL: A [Bloblang mapping](/docs/guides/bloblang/about) that attempts to extract an object containing tracing propagation information, which will then be used as the root tracing span for the message. The specification of the extracted fields must match the format used by the service wide tracer.", "examples": [ "root = @", "root = this.meta.span" ] }, "fetch_buffer_cap": { "type": "integer", "description": "The maximum number of unprocessed messages to fetch at a given time. Default: 256" }, "group": { "type": "object", "description": "Tuning parameters for consumer group synchronization.", "properties": { "heartbeat_interval": { "type": "string", "description": "A period in which heartbeats should be sent out. Default: 3s" }, "rebalance_timeout": { "type": "string", "description": "A period after which rebalancing is abandoned if unresolved. Default: 60s" }, "session_timeout": { "type": "string", "description": "A period after which a consumer of the group is kicked after no heartbeats. Default: 10s" } } }, "max_processing_period": { "type": "string", "description": "A maximum estimate for the time taken to process a message, this is used for tuning consumer group synchronization. Default: 100ms" }, "multi_header": { "type": "boolean", "description": "Decode headers into lists to allow handling of multiple values with the same key Default: false" }, "new_root_span_with_link": { "type": "boolean", "description": "EXPERIMENTAL: Starts a new root span with link to parent." }, "rack_id": { "type": "string", "description": "A rack identifier for this client. Default: " }, "sasl": { "type": "object", "description": "Enables SASL authentication.", "properties": { "access_token": { "type": "string", "description": "A static OAUTHBEARER access token Default: " }, "aws": { "type": "object", "description": "Contains AWS specific fields for when the `mechanism` is set to `AWS_MSK_IAM`.", "properties": { "credentials": { "type": "object", "description": "Optional manual configuration of AWS credentials to use. More information can be found [in this document](/docs/guides/cloud/aws).", "properties": { "expiry_window": { "type": "string", "description": "Allow the credentials to trigger refreshing prior to the credentials actually expiring. This is beneficial so race conditions with expiring credentials do not cause requests to fail. For example '10s' would refresh credentials ten seconds before expiration. Setting to a duration of `0` disables the expiry window. Default: " }, "from_ec2_role": { "type": "boolean", "description": "Use the credentials of a host EC2 machine configured to assume [an IAM role associated with the instance](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html). Default: false" }, "id": { "type": "string", "description": "The ID of credentials to use. Default: " }, "profile": { "type": "string", "description": "A profile from `~/.aws/credentials` to use. Default: " }, "role": { "type": "string", "description": "A role ARN to assume. Default: " }, "role_external_id": { "type": "string", "description": "An external ID to provide when assuming a role. Default: " }, "secret": { "type": "string", "description": "The secret for the credentials being used. Default: " }, "token": { "type": "string", "description": "The token for the credentials being used, required when using short term credentials. Default: " } } }, "endpoint": { "type": "string", "description": "Allows you to specify a custom endpoint for the AWS API. Default: " }, "region": { "type": "string", "description": "The AWS region to target. Default: " } } }, "mechanism": { "type": "string", "description": "The SASL authentication mechanism, if left empty SASL authentication is not used. Default: none" }, "password": { "type": "string", "description": "A PLAIN password. It is recommended that you use environment variables to populate this field. Default: ", "examples": [ "${PASSWORD}" ] }, "token_cache": { "type": "string", "description": "Instead of using a static `access_token` allows you to query a [`cache`](/docs/components/caches/about) resource to fetch OAUTHBEARER tokens from Default: " }, "token_key": { "type": "string", "description": "Required when using a `token_cache`, the key to query the cache with for tokens. Default: " }, "user": { "type": "string", "description": "A PLAIN username. It is recommended that you use environment variables to populate this field. Default: ", "examples": [ "${USER}" ] } } }, "start_from_oldest": { "type": "boolean", "description": "Determines whether to consume from the oldest available offset, otherwise messages are consumed from the latest offset. The setting is applied when creating a new consumer group or the saved offset no longer exists. Default: true" }, "target_version": { "type": "string", "description": "The version of the Kafka protocol to use. This limits the capabilities used by the client and should ideally match the version of your brokers. Defaults to the oldest supported stable version.", "examples": [ "2.1.0", "3.1.0" ] }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "topics": { "type": "array", "description": "A list of topics to consume from. Multiple comma separated topics can be listed in a single element. Partitions are automatically distributed across consumers of a topic. Alternatively, it's possible to specify explicit partitions to consume from with a colon after the topic name, e.g. `foo:0` would consume the partition 0 of the topic foo. This syntax supports ranges, e.g. `foo:0-10` would consume partitions 0 through to 10 inclusive.", "items": { "type": "string" }, "examples": [ [ "foo", "bar" ], [ "foo,bar" ], [ "foo:0", "bar:1", "bar:3" ], [ "foo:0,bar:1,bar:3" ], [ "foo:0-5" ] ] } } }, "kafka_franz": { "type": "object", "properties": { "auto_offset_reset": { "type": "string", "description": "Determines which offset to automatically consume from, matching Kafka's `auto.offset.reset` property. When specified, this takes precedence over `start_from_oldest`." }, "auto_replay_nacks": { "type": "boolean", "description": "Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation. Default: true" }, "batching": { "type": "object", "description": "Allows you to configure a [batching policy](/docs/configuration/batching) that applies to individual topic partitions in order to batch messages together before flushing them for processing. Batching can be beneficial for performance as well as useful for windowed processing, and doing so this way preserves the ordering of topic partitions.", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] }, "checkpoint_limit": { "type": "integer", "description": ":::caution \n\t\t\tSetting this `checkpoint_limit: 1`_will not_ enforce 'strict ordered' processing of records. Use the [kafka input processor](/docs/components/inputs/kafka/) for 'strict ordered' processing.\n:::\n\t\t\t\n\t\t\tDetermines how many messages of the same partition can be processed in parallel before applying back pressure. When a message of a given offset is delivered to the output the offset is only allowed to be committed when all messages of prior offsets have also been delivered, this ensures at-least-once delivery guarantees. However, this mechanism also increases the likelihood of duplicates in the event of crashes or server faults, reducing the checkpoint limit will mitigate this. Default: 1024" }, "client_id": { "type": "string", "description": "An identifier for the client connection. Default: bento" }, "commit_period": { "type": "string", "description": "The period of time between each commit of the current partition offsets. Offsets are always committed during shutdown. Default: 5s" }, "consumer_group": { "type": "string", "description": "An optional consumer group to consume as. When specified the partitions of specified topics are automatically distributed across consumers sharing a consumer group, and partition offsets are automatically committed and resumed under this name. Consumer groups are not supported when specifying explicit partitions to consume from in the `topics` field." }, "fetch_max_bytes": { "type": "string", "description": "This sets the maximum amount of bytes a broker will try to send during a fetch. Note that brokers may not obey this limit if it has records larger than this limit. Also note that this client sends a fetch to each broker concurrently, meaning the client will buffer up to `\u003cbrokers * max bytes\u003e` worth of memory. Equivalent to Kafka's `fetch.max.bytes` option. Default: 50MiB" }, "fetch_max_partition_bytes": { "type": "string", "description": "Sets the maximum amount of bytes that will be consumed for a single partition in a fetch request. Note that if a single batch is larger than this number, that batch will still be returned so the client can make progress. Equivalent to Kafka's `max.partition.fetch.bytes` option. Default: 1MiB" }, "fetch_max_wait": { "type": "string", "description": "This sets the maximum amount of time a broker will wait for a fetch response to hit the minimum number of required bytes before returning, overriding the default 5s. Default: 5s" }, "group_balancers": { "type": "array", "description": "Balancers sets the group balancers to use for dividing topic partitions among group members. This option is equivalent to Kafka's `partition.assignment.strategies` option. Default: [cooperative_sticky]", "items": { "type": "string" } }, "metadata_max_age": { "type": "string", "description": "This sets the maximum age for the client's cached metadata, to allow detection of new topics, partitions, etc. Default: 5m" }, "multi_header": { "type": "boolean", "description": "Decode headers into lists to allow handling of multiple values with the same key Default: false" }, "preferring_lag": { "type": "integer", "description": "\nThis allows you to re-order partitions before they are fetched, given each partition's current lag.\n\nBy default, the client rotates partitions fetched by one after every fetch request. Kafka answers fetch requests in the order that partitions are requested, filling the fetch response until`fetch_max_bytes` and `fetch_max_partition_bytes` are hit. All partitions eventually rotate to the front, ensuring no partition is starved.\n\nWith this option, you can return topic order and per-topic partition ordering. These orders will sort to the front (first by topic, then by partition). Any topic or partitions that you do not return are added to the end, preserving their original ordering." }, "rack_id": { "type": "string", "description": "A rack identifier for this client. Default: " }, "rate_limit": { "type": "string", "description": "An optional [`rate_limit`](/docs/components/rate_limits/about) to throttle invocations by. Default: " }, "reconnect_on_unknown_topic_or_partition": { "type": "boolean", "description": "Determines whether to close the client and force a reconnect after seeing an UNKNOWN_TOPIC_OR_PARTITION or UNKNOWN_TOPIC_ID error. Default: false" }, "regexp_topics": { "type": "boolean", "description": "Whether listed topics should be interpreted as regular expression patterns for matching multiple topics. When topics are specified with explicit partitions this field must remain set to `false`. Default: false" }, "sasl": { "type": "array", "description": "Specify one or more methods of SASL authentication. SASL is tried in order; if the broker supports the first mechanism, all connections will use that mechanism. If the first mechanism fails, the client will pick the first supported mechanism. If the broker does not support any client mechanisms, connections will fail.", "items": { "type": "object" }, "examples": [ [ { "mechanism": "SCRAM-SHA-512", "password": "bar", "username": "foo" } ] ] }, "seed_brokers": { "type": "array", "description": "A list of broker addresses to connect to in order to establish connections. If an item of the list contains commas it will be expanded into multiple addresses.", "items": { "type": "string" }, "examples": [ [ "localhost:9092" ], [ "foo:9092", "bar:9092" ], [ "foo:9092,bar:9092" ] ] }, "start_from_oldest": { "type": "boolean", "description": "Determines whether to consume from the oldest available offset, otherwise messages are consumed from the latest offset. The setting is applied when creating a new consumer group or the saved offset no longer exists. Default: true" }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "topics": { "type": "array", "description": "\nA list of topics to consume from. Multiple comma separated topics can be listed in a single element. When a `consumer_group` is specified partitions are automatically distributed across consumers of a topic, otherwise all partitions are consumed.\n\nAlternatively, it's possible to specify explicit partitions to consume from with a colon after the topic name, e.g. `foo:0` would consume the partition 0 of the topic foo. This syntax supports ranges, e.g. `foo:0-10` would consume partitions 0 through to 10 inclusive.\n\nFinally, it's also possible to specify an explicit offset to consume from by adding another colon after the partition, e.g. `foo:0:10` would consume the partition 0 of the topic foo starting from the offset 10. If the offset is not present (or remains unspecified) then the field `start_from_oldest` determines which offset to start from.", "items": { "type": "string" }, "examples": [ [ "foo", "bar" ], [ "things.*" ], [ "foo,bar" ], [ "foo:0", "bar:1", "bar:3" ], [ "foo:0,bar:1,bar:3" ], [ "foo:0-5" ] ] } } }, "kubernetes_watch": { "type": "object", "properties": { "api_server": { "type": "string", "description": "Kubernetes API server URL. Only used when providing explicit credentials. Default: " }, "auto_auth": { "type": "boolean", "description": "Automatically detect authentication method. Tries in-cluster config first, then falls back to kubeconfig. Default: true" }, "backoff": { "type": "object", "description": "Control time intervals between retry attempts.", "properties": { "initial_interval": { "type": "string", "description": "The initial period to wait between retry attempts. Default: 1s" }, "max_elapsed_time": { "type": "string", "description": "The maximum period to wait before retry attempts are abandoned. If zero then no limit is used. Default: 0s" }, "max_interval": { "type": "string", "description": "The maximum period to wait between retry attempts. Default: 60s" } } }, "ca_file": { "type": "string", "description": "Path to CA certificate file for verifying API server. Default: " }, "client_burst": { "type": "integer", "description": "Burst limit for Kubernetes API client. 0 uses the client-go default. Default: 0" }, "client_qps": { "type": "number", "description": "QPS limit for Kubernetes API client. 0 uses the client-go default. Default: 0" }, "context": { "type": "string", "description": "Kubernetes context to use from kubeconfig. If empty, uses the current context. Default: " }, "custom_resource": { "type": "object", "description": "Custom Resource Definition to watch. Use this for CRDs instead of 'resource'.", "properties": { "group": { "type": "string", "description": "API group for the custom resource (e.g., 'stable.example.com'). Default: " }, "resource": { "type": "string", "description": "Plural name of the custom resource (e.g., 'crontabs'). Default: " }, "version": { "type": "string", "description": "API version for the custom resource (e.g., 'v1', 'v1beta1'). Default: v1" } } }, "event_types": { "type": "array", "description": "Watch event types to include. Valid values are `ADDED`, `MODIFIED`, and `DELETED`. Default: [ADDED MODIFIED DELETED]", "items": { "type": "string" }, "examples": [ [ "ADDED", "DELETED" ], [ "MODIFIED" ] ] }, "field_selector": { "type": "object", "description": "Kubernetes field selector to filter resources. Default: map[]", "examples": [ { "status.phase": "Running" }, { "metadata.name": "my-pod" } ] }, "include_initial_list": { "type": "boolean", "description": "Emit ADDED events for all existing resources when starting. Default: true" }, "insecure_skip_verify": { "type": "boolean", "description": "Skip TLS certificate verification. Not recommended for production. Default: false" }, "kubeconfig": { "type": "string", "description": "Path to kubeconfig file. If empty, uses $KUBECONFIG (if set) or the default kubeconfig location (~/.kube/config). Default: " }, "kubeconfig_yaml": { "type": "string", "description": "Kubeconfig content as a YAML string. Use this to embed kubeconfig directly in your config (e.g., from a secret or environment variable) instead of referencing a file path. Default: " }, "label_selector": { "type": "object", "description": "Kubernetes label selector to filter resources. Default: map[]", "examples": [ { "app": "myapp" }, { "app": "myapp", "env": "prod" } ] }, "max_retries": { "type": "integer", "description": "The maximum number of retries before giving up on the request. If set to zero there is no discrete limit. Default: 0" }, "namespaces": { "type": "array", "description": "Namespaces to watch. Empty list means all namespaces. Default: []", "items": { "type": "string" }, "examples": [ [ "default" ], [ "production", "staging" ] ] }, "request_timeout": { "type": "string", "description": "Timeout for Kubernetes API requests such as list calls. Use \"0s\" to disable. Default: 30s" }, "resource": { "type": "string", "description": "Standard Kubernetes resource type to watch. Default: ", "examples": [ "pods", "deployments", "configmaps" ] }, "token": { "type": "string", "description": "Bearer token for authentication. Can be a service account token. Default: " }, "token_file": { "type": "string", "description": "Path to file containing bearer token. Default: " } } }, "mongodb": { "type": "object", "properties": { "auto_replay_nacks": { "type": "boolean", "description": "Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation. Default: true" }, "batch_size": { "type": "integer", "description": "A explicit number of documents to batch up before flushing them for processing. Must be greater than `0`. Operations: `find`, `aggregate`", "examples": [ 1000 ] }, "collection": { "type": "string", "description": "The collection to select from." }, "database": { "type": "string", "description": "The name of the target MongoDB database." }, "json_marshal_mode": { "type": "string", "description": "The json_marshal_mode setting is optional and controls the format of the output message. Default: canonical" }, "limit": { "type": "integer", "description": "An explicit maximum number of documents to return. Operations: `find`" }, "operation": { "type": "string", "description": "The mongodb operation to perform. Default: find" }, "password": { "type": "string", "description": "The password to connect to the database. Default: " }, "query": { "type": "string", "description": "Bloblang expression describing MongoDB query.", "examples": [ "\n root.from = {\"$lte\": timestamp_unix()}\n root.to = {\"$gte\": timestamp_unix()}\n" ] }, "sort": { "type": "object", "description": "An object specifying fields to sort by, and the respective sort order (`1` ascending, `-1` descending). Note: The driver currently appears to support only one sorting key. Operations: `find`", "examples": [ { "name": 1 }, { "age": -1 } ] }, "url": { "type": "string", "description": "The URL of the target MongoDB server.", "examples": [ "mongodb://localhost:27017" ] }, "username": { "type": "string", "description": "The username to connect to the database. Default: " } } }, "mqtt": { "type": "object", "properties": { "auto_replay_nacks": { "type": "boolean", "description": "Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation. Default: true" }, "clean_session": { "type": "boolean", "description": "Set whether the connection is non-persistent. Default: true" }, "client_id": { "type": "string", "description": "An identifier for the client connection. Default: " }, "connect_timeout": { "type": "string", "description": "The maximum amount of time to wait in order to establish a connection before the attempt is abandoned. Default: 30s", "examples": [ "1s", "500ms" ] }, "dynamic_client_id_suffix": { "type": "string", "description": "Append a dynamically generated suffix to the specified `client_id` on each run of the pipeline. This can be useful when clustering Bento producers." }, "keepalive": { "type": "integer", "description": "Max seconds of inactivity before a keepalive message is sent. Default: 30" }, "password": { "type": "string", "description": "A password to connect with. Default: " }, "qos": { "type": "integer", "description": "The level of delivery guarantee to enforce. Has options 0, 1, 2. Default: 1" }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "topics": { "type": "array", "description": "A list of topics to consume from.", "items": { "type": "string" } }, "urls": { "type": "array", "description": "A list of URLs to connect to. If an item of the list contains commas it will be expanded into multiple URLs.", "items": { "type": "string" }, "examples": [ [ "tcp://localhost:1883" ] ] }, "user": { "type": "string", "description": "A username to connect with. Default: " }, "will": { "type": "object", "description": "Set last will message in case of Bento failure", "properties": { "enabled": { "type": "boolean", "description": "Whether to enable last will messages. Default: false" }, "payload": { "type": "string", "description": "Set payload for last will message. Default: " }, "qos": { "type": "integer", "description": "Set QoS for last will message. Valid values are: 0, 1, 2. Default: 0" }, "retained": { "type": "boolean", "description": "Set retained for last will message. Default: false" }, "topic": { "type": "string", "description": "Set topic for last will message. Default: " } } } } }, "nanomsg": { "type": "object", "properties": { "auto_replay_nacks": { "type": "boolean", "description": "Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation. Default: true" }, "bind": { "type": "boolean", "description": "Whether the URLs provided should be connected to, or bound as. Default: true" }, "poll_timeout": { "type": "string", "description": "The period to wait until a poll is abandoned and reattempted. Default: 5s" }, "socket_type": { "type": "string", "description": "The socket type to use. Default: PULL" }, "sub_filters": { "type": "array", "description": "A list of subscription topic filters to use when consuming from a SUB socket. Specifying a single sub_filter of `''` will subscribe to everything. Default: []", "items": { "type": "string" } }, "urls": { "type": "array", "description": "A list of URLs to connect to (or as). If an item of the list contains commas it will be expanded into multiple URLs.", "items": { "type": "string" } } } }, "nats": { "type": "object", "properties": { "auth": { "type": "object", "description": "Optional configuration of NATS authentication parameters.", "properties": { "nkey_file": { "type": "string", "description": "An optional file containing a NKey seed.", "examples": [ "./seed.nk" ] }, "user_credentials_file": { "type": "string", "description": "An optional file containing user credentials which consist of an user JWT and corresponding NKey seed.", "examples": [ "./user.creds" ] }, "user_jwt": { "type": "string", "description": "An optional plain text user JWT (given along with the corresponding user NKey Seed)." }, "user_nkey_seed": { "type": "string", "description": "An optional plain text user NKey Seed (given along with the corresponding user JWT)." } } }, "auto_replay_nacks": { "type": "boolean", "description": "Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation. Default: true" }, "extract_tracing_map": { "type": "string", "description": "EXPERIMENTAL: A [Bloblang mapping](/docs/guides/bloblang/about) that attempts to extract an object containing tracing propagation information, which will then be used as the root tracing span for the message. The specification of the extracted fields must match the format used by the service wide tracer.", "examples": [ "root = @", "root = this.meta.span" ] }, "nak_delay": { "type": "string", "description": "An optional delay duration on redelivering a message when negatively acknowledged.", "examples": [ "1m" ] }, "new_root_span_with_link": { "type": "boolean", "description": "EXPERIMENTAL: Starts a new root span with link to parent." }, "prefetch_count": { "type": "integer", "description": "The maximum number of messages to pull at a time. Default: 524288" }, "queue": { "type": "string", "description": "An optional queue group to consume as." }, "subject": { "type": "string", "description": "A subject to consume from. Supports wildcards for consuming multiple subjects. Either a subject or stream must be specified.", "examples": [ "foo.bar.baz", "foo.*.baz", "foo.bar.*", "foo.\u003e" ] }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "urls": { "type": "array", "description": "A list of URLs to connect to. If an item of the list contains commas it will be expanded into multiple URLs.", "items": { "type": "string" }, "examples": [ [ "nats://127.0.0.1:4222" ], [ "nats://username:password@127.0.0.1:4222" ] ] } } }, "nats_jetstream": { "type": "object", "properties": { "ack_wait": { "type": "string", "description": "The maximum amount of time NATS server should wait for an ack from consumer. Default: 30s", "examples": [ "100ms", "5m" ] }, "auth": { "type": "object", "description": "Optional configuration of NATS authentication parameters.", "properties": { "nkey_file": { "type": "string", "description": "An optional file containing a NKey seed.", "examples": [ "./seed.nk" ] }, "user_credentials_file": { "type": "string", "description": "An optional file containing user credentials which consist of an user JWT and corresponding NKey seed.", "examples": [ "./user.creds" ] }, "user_jwt": { "type": "string", "description": "An optional plain text user JWT (given along with the corresponding user NKey Seed)." }, "user_nkey_seed": { "type": "string", "description": "An optional plain text user NKey Seed (given along with the corresponding user JWT)." } } }, "bind": { "type": "boolean", "description": "Indicates that the subscription should use an existing consumer." }, "deliver": { "type": "string", "description": "Determines which messages to deliver when consuming without a durable subscriber. Default: all" }, "durable": { "type": "string", "description": "Preserve the state of your consumer under a durable name." }, "extract_tracing_map": { "type": "string", "description": "EXPERIMENTAL: A [Bloblang mapping](/docs/guides/bloblang/about) that attempts to extract an object containing tracing propagation information, which will then be used as the root tracing span for the message. The specification of the extracted fields must match the format used by the service wide tracer.", "examples": [ "root = @", "root = this.meta.span" ] }, "max_ack_pending": { "type": "integer", "description": "The maximum number of outstanding acks to be allowed before consuming is halted. Default: 1024" }, "new_root_span_with_link": { "type": "boolean", "description": "EXPERIMENTAL: Starts a new root span with link to parent." }, "queue": { "type": "string", "description": "An optional queue group to consume as." }, "stream": { "type": "string", "description": "A stream to consume from. Either a subject or stream must be specified." }, "subject": { "type": "string", "description": "A subject to consume from. Supports wildcards for consuming multiple subjects. Either a subject or stream must be specified.", "examples": [ "foo.bar.baz", "foo.*.baz", "foo.bar.*", "foo.\u003e" ] }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "urls": { "type": "array", "description": "A list of URLs to connect to. If an item of the list contains commas it will be expanded into multiple URLs.", "items": { "type": "string" }, "examples": [ [ "nats://127.0.0.1:4222" ], [ "nats://username:password@127.0.0.1:4222" ] ] } } }, "nats_kv": { "type": "object", "properties": { "auth": { "type": "object", "description": "Optional configuration of NATS authentication parameters.", "properties": { "nkey_file": { "type": "string", "description": "An optional file containing a NKey seed.", "examples": [ "./seed.nk" ] }, "user_credentials_file": { "type": "string", "description": "An optional file containing user credentials which consist of an user JWT and corresponding NKey seed.", "examples": [ "./user.creds" ] }, "user_jwt": { "type": "string", "description": "An optional plain text user JWT (given along with the corresponding user NKey Seed)." }, "user_nkey_seed": { "type": "string", "description": "An optional plain text user NKey Seed (given along with the corresponding user JWT)." } } }, "auto_replay_nacks": { "type": "boolean", "description": "Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation. Default: true" }, "bucket": { "type": "string", "description": "The name of the KV bucket.", "examples": [ "my_kv_bucket" ] }, "ignore_deletes": { "type": "boolean", "description": "Do not send delete markers as messages. Default: false" }, "include_history": { "type": "boolean", "description": "Include all the history per key, not just the last one. Default: false" }, "key": { "type": "string", "description": "Key to watch for updates, can include wildcards. Default: \u003e", "examples": [ "foo.bar.baz", "foo.*.baz", "foo.bar.*", "foo.\u003e" ] }, "meta_only": { "type": "boolean", "description": "Retrieve only the metadata of the entry Default: false" }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "urls": { "type": "array", "description": "A list of URLs to connect to. If an item of the list contains commas it will be expanded into multiple URLs.", "items": { "type": "string" }, "examples": [ [ "nats://127.0.0.1:4222" ], [ "nats://username:password@127.0.0.1:4222" ] ] } } }, "nats_object_store": { "type": "object", "properties": { "auth": { "type": "object", "description": "Optional configuration of NATS authentication parameters.", "properties": { "nkey_file": { "type": "string", "description": "An optional file containing a NKey seed.", "examples": [ "./seed.nk" ] }, "user_credentials_file": { "type": "string", "description": "An optional file containing user credentials which consist of an user JWT and corresponding NKey seed.", "examples": [ "./user.creds" ] }, "user_jwt": { "type": "string", "description": "An optional plain text user JWT (given along with the corresponding user NKey Seed)." }, "user_nkey_seed": { "type": "string", "description": "An optional plain text user NKey Seed (given along with the corresponding user JWT)." } } }, "auto_replay_nacks": { "type": "boolean", "description": "Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation. Default: true" }, "bucket": { "type": "string", "description": "The name of the object store bucket.", "examples": [ "my_bucket" ] }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "urls": { "type": "array", "description": "A list of URLs to connect to. If an item of the list contains commas it will be expanded into multiple URLs.", "items": { "type": "string" }, "examples": [ [ "nats://127.0.0.1:4222" ], [ "nats://username:password@127.0.0.1:4222" ] ] } } }, "nats_stream": { "type": "object", "properties": { "ack_wait": { "type": "string", "description": "An optional duration to specify at which a message that is yet to be acked will be automatically retried. Default: 30s" }, "auth": { "type": "object", "description": "Optional configuration of NATS authentication parameters.", "properties": { "nkey_file": { "type": "string", "description": "An optional file containing a NKey seed.", "examples": [ "./seed.nk" ] }, "user_credentials_file": { "type": "string", "description": "An optional file containing user credentials which consist of an user JWT and corresponding NKey seed.", "examples": [ "./user.creds" ] }, "user_jwt": { "type": "string", "description": "An optional plain text user JWT (given along with the corresponding user NKey Seed)." }, "user_nkey_seed": { "type": "string", "description": "An optional plain text user NKey Seed (given along with the corresponding user JWT)." } } }, "client_id": { "type": "string", "description": "A client ID to connect as. Default: " }, "cluster_id": { "type": "string", "description": "The ID of the cluster to consume from." }, "durable_name": { "type": "string", "description": "Preserve the state of your consumer under a durable name. Default: " }, "extract_tracing_map": { "type": "string", "description": "EXPERIMENTAL: A [Bloblang mapping](/docs/guides/bloblang/about) that attempts to extract an object containing tracing propagation information, which will then be used as the root tracing span for the message. The specification of the extracted fields must match the format used by the service wide tracer.", "examples": [ "root = @", "root = this.meta.span" ] }, "max_inflight": { "type": "integer", "description": "The maximum number of unprocessed messages to fetch at a given time. Default: 1024" }, "new_root_span_with_link": { "type": "boolean", "description": "EXPERIMENTAL: Starts a new root span with link to parent." }, "queue": { "type": "string", "description": "The queue to consume from. Default: " }, "start_from_oldest": { "type": "boolean", "description": "If a position is not found for a queue, determines whether to consume from the oldest available message, otherwise messages are consumed from the latest. Default: true" }, "subject": { "type": "string", "description": "A subject to consume from. Default: " }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "unsubscribe_on_close": { "type": "boolean", "description": "Whether the subscription should be destroyed when this client disconnects. Default: false" }, "urls": { "type": "array", "description": "A list of URLs to connect to. If an item of the list contains commas it will be expanded into multiple URLs.", "items": { "type": "string" }, "examples": [ [ "nats://127.0.0.1:4222" ], [ "nats://username:password@127.0.0.1:4222" ] ] } } }, "nsq": { "type": "object", "properties": { "auth_secret": { "type": "string", "description": "An optional secret for NSQ authentication (requires nsqd 0.2.29+)." }, "channel": { "type": "string", "description": "The channel to consume from." }, "lookupd_http_addresses": { "type": "array", "description": "A list of nsqlookupd addresses to connect to.", "items": { "type": "string" } }, "max_attempts": { "type": "integer", "description": "The maximum number of attempts to successfully consume a messages. Default: 5" }, "max_in_flight": { "type": "integer", "description": "The maximum number of pending messages to consume at any given time. Default: 100" }, "nsqd_tcp_addresses": { "type": "array", "description": "A list of nsqd addresses to connect to.", "items": { "type": "string" } }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "topic": { "type": "string", "description": "The topic to consume from." }, "user_agent": { "type": "string", "description": "A user agent to assume when connecting." } } }, "parquet": { "type": "object", "properties": { "auto_replay_nacks": { "type": "boolean", "description": "Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation. Default: true" }, "batch_count": { "type": "integer", "description": "Optionally process records in batches. This can help to speed up the consumption of exceptionally large files. When the end of the file is reached the remaining records are processed as a (potentially smaller) batch. Default: 1" }, "paths": { "type": "array", "description": "A list of file paths to read from. Each file will be read sequentially until the list is exhausted, at which point the input will close. Glob patterns are supported, including super globs (double star).", "items": { "type": "string" }, "examples": [ "/tmp/foo.parquet", "/tmp/bar/*.parquet", "/tmp/data/**/*.parquet" ] }, "strict_schema": { "type": "boolean", "description": "Whether to enforce strict Parquet schema validation. When set to false, allows reading files with non-standard schema structures (such as non-standard LIST formats). Disabling strict mode may reduce validation but increases compatibility. Default: true" } } }, "pulsar": { "type": "object", "properties": { "auth": { "type": "object", "description": "Optional configuration of Pulsar authentication methods.", "properties": { "oauth2": { "type": "object", "description": "Parameters for Pulsar OAuth2 authentication.", "properties": { "audience": { "type": "string", "description": "OAuth2 audience. Default: " }, "enabled": { "type": "boolean", "description": "Whether OAuth2 is enabled. Default: false" }, "issuer_url": { "type": "string", "description": "OAuth2 issuer URL. Default: " }, "private_key_file": { "type": "string", "description": "The path to a file containing a private key. Default: " } } }, "token": { "type": "object", "description": "Parameters for Pulsar Token authentication.", "properties": { "enabled": { "type": "boolean", "description": "Whether Token Auth is enabled. Default: false" }, "token": { "type": "string", "description": "Actual base64 encoded token. Default: " } } } } }, "subscription_name": { "type": "string", "description": "Specify the subscription name for this consumer." }, "subscription_type": { "type": "string", "description": "Specify the subscription type for this consumer.\n\n\u003e NOTE: Using a `key_shared` subscription type will __allow out-of-order delivery__ since nack-ing messages sets non-zero nack delivery delay - this can potentially cause consumers to stall. See [Pulsar documentation](https://pulsar.apache.org/docs/en/2.8.1/concepts-messaging/#negative-acknowledgement) and [this Github issue](https://github.com/apache/pulsar/issues/12208) for more details. Default: shared" }, "tls": { "type": "object", "description": "Specify the path to a custom CA certificate to trust broker TLS service.", "properties": { "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] } } }, "topics": { "type": "array", "description": "A list of topics to subscribe to. This or topics_pattern must be set.", "items": { "type": "string" } }, "topics_pattern": { "type": "string", "description": "A regular expression matching the topics to subscribe to. This or topics must be set." }, "url": { "type": "string", "description": "A URL to connect to.", "examples": [ "pulsar://localhost:6650", "pulsar://pulsar.us-west.example.com:6650", "pulsar+ssl://pulsar.us-west.example.com:6651" ] } } }, "read_until": { "type": "object", "properties": { "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether the input should now be closed.", "examples": [ "this.type == \"foo\"", "count(\"messages\") \u003e= 100" ] }, "idle_timeout": { "type": "string", "description": "The maximum amount of time without receiving new messages after which the input is closed.", "examples": [ "5s" ] }, "input": { "type": "object", "description": "The child input to consume from." }, "restart_input": { "type": "boolean", "description": "Whether the input should be reopened if it closes itself before the condition has resolved to true. Default: false" } } }, "redis_list": { "type": "object", "properties": { "auto_replay_nacks": { "type": "boolean", "description": "Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation. Default: true" }, "command": { "type": "string", "description": "The command used to pop elements from the Redis list Default: blpop" }, "key": { "type": "string", "description": "The key of a list to read from." }, "kind": { "type": "string", "description": "Specifies a simple, cluster-aware, or failover-aware redis client. Default: simple" }, "master": { "type": "string", "description": "Name of the redis master when `kind` is `failover` Default: ", "examples": [ "mymaster" ] }, "max_in_flight": { "type": "integer", "description": "Optionally sets a limit on the number of messages that can be flowing through a Bento stream pending acknowledgment from the input at any given time. Once a message has been either acknowledged or rejected (nacked) it is no longer considered pending. If the input produces logical batches then each batch is considered a single count against the maximum. **WARNING**: Batching policies at the output level will stall if this field limits the number of messages below the batching threshold. Zero (default) or lower implies no limit. Default: 0" }, "timeout": { "type": "string", "description": "The length of time to poll for new messages before reattempting. Default: 5s" }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.\n\n**Troubleshooting**\n\nSome cloud hosted instances of Redis (such as Azure Cache) might need some hand holding in order to establish stable connections. Unfortunately, it is often the case that TLS issues will manifest as generic error messages such as \"i/o timeout\". If you're using TLS and are seeing connectivity problems consider setting `enable_renegotiation` to `true`, and ensuring that the server supports at least TLS version 1.2.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "url": { "type": "string", "description": "The URL of the target Redis server. Database is optional and is supplied as the URL path.", "examples": [ "redis://:6397", "redis://localhost:6379", "redis://foousername:foopassword@redisplace:6379", "redis://:foopassword@redisplace:6379", "redis://localhost:6379/1", "redis://localhost:6379/1,redis://localhost:6380/1" ] } } }, "redis_pubsub": { "type": "object", "properties": { "auto_replay_nacks": { "type": "boolean", "description": "Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation. Default: true" }, "channels": { "type": "array", "description": "A list of channels to consume from.", "items": { "type": "string" } }, "kind": { "type": "string", "description": "Specifies a simple, cluster-aware, or failover-aware redis client. Default: simple" }, "master": { "type": "string", "description": "Name of the redis master when `kind` is `failover` Default: ", "examples": [ "mymaster" ] }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.\n\n**Troubleshooting**\n\nSome cloud hosted instances of Redis (such as Azure Cache) might need some hand holding in order to establish stable connections. Unfortunately, it is often the case that TLS issues will manifest as generic error messages such as \"i/o timeout\". If you're using TLS and are seeing connectivity problems consider setting `enable_renegotiation` to `true`, and ensuring that the server supports at least TLS version 1.2.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "url": { "type": "string", "description": "The URL of the target Redis server. Database is optional and is supplied as the URL path.", "examples": [ "redis://:6397", "redis://localhost:6379", "redis://foousername:foopassword@redisplace:6379", "redis://:foopassword@redisplace:6379", "redis://localhost:6379/1", "redis://localhost:6379/1,redis://localhost:6380/1" ] }, "use_patterns": { "type": "boolean", "description": "Whether to use the PSUBSCRIBE command, allowing for glob-style patterns within target channel names. Default: false" } } }, "redis_scan": { "type": "object", "properties": { "auto_replay_nacks": { "type": "boolean", "description": "Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation. Default: true" }, "kind": { "type": "string", "description": "Specifies a simple, cluster-aware, or failover-aware redis client. Default: simple" }, "master": { "type": "string", "description": "Name of the redis master when `kind` is `failover` Default: ", "examples": [ "mymaster" ] }, "match": { "type": "string", "description": "Iterates only elements matching the optional glob-style pattern. By default, it matches all elements. Default: ", "examples": [ "*", "1*", "foo*", "foo", "*4*" ] }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.\n\n**Troubleshooting**\n\nSome cloud hosted instances of Redis (such as Azure Cache) might need some hand holding in order to establish stable connections. Unfortunately, it is often the case that TLS issues will manifest as generic error messages such as \"i/o timeout\". If you're using TLS and are seeing connectivity problems consider setting `enable_renegotiation` to `true`, and ensuring that the server supports at least TLS version 1.2.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "url": { "type": "string", "description": "The URL of the target Redis server. Database is optional and is supplied as the URL path.", "examples": [ "redis://:6397", "redis://localhost:6379", "redis://foousername:foopassword@redisplace:6379", "redis://:foopassword@redisplace:6379", "redis://localhost:6379/1", "redis://localhost:6379/1,redis://localhost:6380/1" ] } } }, "redis_streams": { "type": "object", "properties": { "auto_replay_nacks": { "type": "boolean", "description": "Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation. Default: true" }, "body_key": { "type": "string", "description": "The field key to extract the raw message from. All other keys will be stored in the message as metadata. Default: body" }, "client_id": { "type": "string", "description": "An identifier for the client connection. Default: " }, "commit_period": { "type": "string", "description": "The period of time between each commit of the current offset. Offsets are always committed during shutdown. Default: 1s" }, "consumer_group": { "type": "string", "description": "An identifier for the consumer group of the stream. Default: " }, "create_streams": { "type": "boolean", "description": "Create subscribed streams if they do not exist (MKSTREAM option). Default: true" }, "kind": { "type": "string", "description": "Specifies a simple, cluster-aware, or failover-aware redis client. Default: simple" }, "limit": { "type": "integer", "description": "The maximum number of messages to consume from a single request. Default: 10" }, "master": { "type": "string", "description": "Name of the redis master when `kind` is `failover` Default: ", "examples": [ "mymaster" ] }, "start_from_oldest": { "type": "boolean", "description": "If an offset is not found for a stream, determines whether to consume from the oldest available offset, otherwise messages are consumed from the latest offset. Default: true" }, "streams": { "type": "array", "description": "A list of streams to consume from.", "items": { "type": "string" } }, "timeout": { "type": "string", "description": "The length of time to poll for new messages before reattempting. Default: 1s" }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.\n\n**Troubleshooting**\n\nSome cloud hosted instances of Redis (such as Azure Cache) might need some hand holding in order to establish stable connections. Unfortunately, it is often the case that TLS issues will manifest as generic error messages such as \"i/o timeout\". If you're using TLS and are seeing connectivity problems consider setting `enable_renegotiation` to `true`, and ensuring that the server supports at least TLS version 1.2.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "url": { "type": "string", "description": "The URL of the target Redis server. Database is optional and is supplied as the URL path.", "examples": [ "redis://:6397", "redis://localhost:6379", "redis://foousername:foopassword@redisplace:6379", "redis://:foopassword@redisplace:6379", "redis://localhost:6379/1", "redis://localhost:6379/1,redis://localhost:6380/1" ] } } }, "resource": { "type": "string", "description": " Default: " }, "s2": { "type": "object", "properties": { "access_token": { "type": "string", "description": "Access token for S2 account" }, "backoff_duration": { "type": "string", "description": "Interval to backoff for before reconnecting to a stream Default: 100ms" }, "basin": { "type": "string", "description": "Basin name" }, "cache": { "type": "string", "description": "Cache resource label for storing sequence number" }, "max_in_flight": { "type": "integer", "description": "Optionally sets a limit on the number of messages that can be flowing through a Bento stream pending acknowledgment from the input at any given time. Once a message has been either acknowledged or rejected (nacked) it is no longer considered pending. If the input produces logical batches then each batch is considered a single count against the maximum. **WARNING**: Batching policies at the output level will stall if this field limits the number of messages below the batching threshold. Zero (default) or lower implies no limit. Default: 0" }, "start_seq_num": { "type": "string", "description": "Start consuming the stream from either the earliest or the latest sequence number Default: earliest" }, "streams": { "description": "Streams prefix or list of streams to subscribe to" }, "update_streams_interval": { "type": "string", "description": "Interval after which the streams list should update dynamically Default: 1m" } } }, "sequence": { "type": "object", "properties": { "inputs": { "type": "array", "description": "An array of inputs to read from sequentially.", "items": { "type": "object" } }, "sharded_join": { "type": "object", "description": "EXPERIMENTAL: Provides a way to perform outer joins of arbitrarily structured and unordered data resulting from the input sequence, even when the overall size of the data surpasses the memory available on the machine.\n\nWhen configured the sequence of inputs will be consumed one or more times according to the number of iterations, and when more than one iteration is specified each iteration will process an entirely different set of messages by sharding them by the ID field. Increasing the number of iterations reduces the memory consumption at the cost of needing to fully parse the data each time.\n\nEach message must be structured (JSON or otherwise processed into a structured form) and the fields will be aggregated with those of other messages sharing the ID. At the end of each iteration the joined messages are flushed downstream before the next iteration begins, hence keeping memory usage limited.", "properties": { "id_path": { "type": "string", "description": "A [dot path](/docs/configuration/field_paths) that points to a common field within messages of each fragmented data set and can be used to join them. Messages that are not structured or are missing this field will be dropped. This field must be set in order to enable joins. Default: " }, "iterations": { "type": "integer", "description": "The total number of iterations (shards), increasing this number will increase the overall time taken to process the data, but reduces the memory used in the process. The real memory usage required is significantly higher than the real size of the data and therefore the number of iterations should be at least an order of magnitude higher than the available memory divided by the overall size of the dataset. Default: 1" }, "merge_strategy": { "type": "string", "description": "The chosen strategy to use when a data join would otherwise result in a collision of field values. The strategy `array` means non-array colliding values are placed into an array and colliding arrays are merged. The strategy `replace` replaces old values with new values. The strategy `keep` keeps the old value. Default: array" }, "type": { "type": "string", "description": "The type of join to perform. A `full-outer` ensures that all identifiers seen in any of the input sequences are sent, and is performed by consuming all input sequences before flushing the joined results. An `outer` join consumes all input sequences but only writes data joined from the last input in the sequence, similar to a left or right outer join. With an `outer` join if an identifier appears multiple times within the final sequence input it will be flushed each time it appears. `full-outter` and `outter` have been deprecated in favour of `full-outer` and `outer`. Default: none" } } } } }, "sftp": { "type": "object", "properties": { "address": { "type": "string", "description": "The address of the server to connect to." }, "auto_replay_nacks": { "type": "boolean", "description": "Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation. Default: true" }, "codec": { "type": "string", "description": "The way in which the bytes of a data source should be converted into discrete messages, codecs are useful for specifying how large files or continuous streams of data might be processed in small chunks rather than loading it all in memory. It's possible to consume lines using a custom delimiter with the `delim:x` codec, where x is the character sequence custom delimiter. Codecs can be chained with `/`, for example a gzip compressed CSV file can be consumed with the codec `gzip/csv`.", "examples": [ "lines", "delim:\t", "delim:foobar", "gzip/csv" ] }, "credentials": { "type": "object", "description": "The credentials to use to log into the target server.", "properties": { "password": { "type": "string", "description": "The password for the username to connect to the SFTP server. Default: " }, "private_key_file": { "type": "string", "description": "The private key for the username to connect to the SFTP server. Default: " }, "private_key_pass": { "type": "string", "description": "Optional passphrase for private key. Default: " }, "username": { "type": "string", "description": "The username to connect to the SFTP server. Default: " } } }, "delete_on_finish": { "type": "boolean", "description": "Whether to delete files from the server once they are processed. Default: false" }, "max_buffer": { "type": "integer", "description": " Default: 1000000" }, "paths": { "type": "array", "description": "A list of paths to consume sequentially. Glob patterns are supported.", "items": { "type": "string" } }, "scanner": { "type": "object", "description": "The [scanner](/docs/components/scanners/about) by which the stream of bytes consumed will be broken out into individual messages. Scanners are useful for processing large sources of data without holding the entirety of it within memory. For example, the `csv` scanner allows you to process individual CSV rows without loading the entire CSV file in memory at once. Default: map[to_the_end:map[]]" }, "watcher": { "type": "object", "description": "An experimental mode whereby the input will periodically scan the target paths for new files and consume them, when all files are consumed the input will continue polling for new files.", "properties": { "cache": { "type": "string", "description": "A [cache resource](/docs/components/caches/about) for storing the paths of files already consumed. Default: " }, "enabled": { "type": "boolean", "description": "Whether file watching is enabled. Default: false" }, "minimum_age": { "type": "string", "description": "The minimum period of time since a file was last updated before attempting to consume it. Increasing this period decreases the likelihood that a file will be consumed whilst it is still being written to. Default: 1s", "examples": [ "10s", "1m", "10m" ] }, "poll_interval": { "type": "string", "description": "The interval between each attempt to scan the target paths for new files. Default: 1s", "examples": [ "100ms", "1s" ] } } } } }, "socket": { "type": "object", "properties": { "address": { "type": "string", "description": "The address to connect to.", "examples": [ "/tmp/bento.sock", "127.0.0.1:6000" ] }, "auto_replay_nacks": { "type": "boolean", "description": "Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation. Default: true" }, "codec": { "type": "string", "description": "The way in which the bytes of a data source should be converted into discrete messages, codecs are useful for specifying how large files or continuous streams of data might be processed in small chunks rather than loading it all in memory. It's possible to consume lines using a custom delimiter with the `delim:x` codec, where x is the character sequence custom delimiter. Codecs can be chained with `/`, for example a gzip compressed CSV file can be consumed with the codec `gzip/csv`.", "examples": [ "lines", "delim:\t", "delim:foobar", "gzip/csv" ] }, "max_buffer": { "type": "integer", "description": " Default: 1000000" }, "network": { "type": "string", "description": "A network type to assume (unix|tcp)." }, "scanner": { "type": "object", "description": "The [scanner](/docs/components/scanners/about) by which the stream of bytes consumed will be broken out into individual messages. Scanners are useful for processing large sources of data without holding the entirety of it within memory. For example, the `csv` scanner allows you to process individual CSV rows without loading the entire CSV file in memory at once. Default: map[lines:map[]]" } } }, "socket_server": { "type": "object", "properties": { "address": { "type": "string", "description": "The address to listen from.", "examples": [ "/tmp/bento.sock", "0.0.0.0:6000" ] }, "address_cache": { "type": "string", "description": "An optional [`cache`](/docs/components/caches/about) within which this input should write it's bound address once known. The key of the cache item containing the address will be the label of the component suffixed with `_address` (e.g. `foo_address`), or `socket_server_address` when a label has not been provided. This is useful in situations where the address is dynamically allocated by the server (`127.0.0.1:0`) and you want to store the allocated address somewhere for reference by other systems and components." }, "auto_replay_nacks": { "type": "boolean", "description": "Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation. Default: true" }, "codec": { "type": "string", "description": "The way in which the bytes of a data source should be converted into discrete messages, codecs are useful for specifying how large files or continuous streams of data might be processed in small chunks rather than loading it all in memory. It's possible to consume lines using a custom delimiter with the `delim:x` codec, where x is the character sequence custom delimiter. Codecs can be chained with `/`, for example a gzip compressed CSV file can be consumed with the codec `gzip/csv`.", "examples": [ "lines", "delim:\t", "delim:foobar", "gzip/csv" ] }, "max_buffer": { "type": "integer", "description": " Default: 1000000" }, "network": { "type": "string", "description": "A network type to accept." }, "scanner": { "type": "object", "description": "The [scanner](/docs/components/scanners/about) by which the stream of bytes consumed will be broken out into individual messages. Scanners are useful for processing large sources of data without holding the entirety of it within memory. For example, the `csv` scanner allows you to process individual CSV rows without loading the entire CSV file in memory at once. Default: map[lines:map[]]" }, "tls": { "type": "object", "description": "TLS specific configuration, valid when the `network` is set to `tls`.", "properties": { "cert_file": { "type": "string", "description": "PEM encoded certificate for use with TLS." }, "key_file": { "type": "string", "description": "PEM encoded private key for use with TLS." }, "self_signed": { "type": "boolean", "description": "Whether to generate self signed certificates. Default: false" } } } } }, "sql_raw": { "type": "object", "properties": { "args_mapping": { "type": "string", "description": "A [Bloblang mapping](/docs/guides/bloblang/about) which should evaluate to an array of values matching in size to the number of columns specified.", "examples": [ "root = [ this.cat.meow, this.doc.woofs[0] ]", "root = [ metadata(\"user.id\").string() ]" ] }, "auto_replay_nacks": { "type": "boolean", "description": "Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation. Default: true" }, "azure": { "type": "object", "description": "Optional Fields that can be set to use Azure based authentication for Azure Postgres SQL", "properties": { "entra_enabled": { "type": "boolean", "description": "An optional field used to generate an entra token to connect to 'Azure Database for PostgreSQL flexible server', This will create a new connection string with the host, user and database from the DSN field - you may need to URL encode the dsn! The [Default Azure Credential Chain](https://learn.microsoft.com/en-gb/azure/developer/go/sdk/authentication/authentication-overview#defaultazurecredential) is used from the Azure SDK. Default: false" }, "token_request_options": { "type": "object", "properties": { "claims": { "type": "string", "description": "Set additional claims for the token. Default: " }, "enable_cae": { "type": "boolean", "description": "Indicates whether to enable Continuous Access Evaluation (CAE) for the requested token Default: false" }, "scopes": { "type": "array", "description": "Scopes contains the list of permission scopes required for the token. Default: [https://ossrdbms-aad.database.windows.net/.default]", "items": { "type": "string" } }, "tenant_id": { "type": "string", "description": "tenant_id identifies the tenant from which to request the token. azure credentials authenticate in their configured default tenants when this field isn't set. Default: " } } } } }, "conn_max_idle": { "type": "integer", "description": "An optional maximum number of connections in the idle connection pool. If conn_max_open is greater than 0 but less than the new conn_max_idle, then the new conn_max_idle will be reduced to match the conn_max_open limit. If `value \u003c= 0`, no idle connections are retained. The default max idle connections is currently 2. This may change in a future release. Default: 2" }, "conn_max_idle_time": { "type": "string", "description": "An optional maximum amount of time a connection may be idle. Expired connections may be closed lazily before reuse. If `value \u003c= 0`, connections are not closed due to a connections idle time." }, "conn_max_life_time": { "type": "string", "description": "An optional maximum amount of time a connection may be reused. Expired connections may be closed lazily before reuse. If `value \u003c= 0`, connections are not closed due to a connections age." }, "conn_max_open": { "type": "integer", "description": "An optional maximum number of open connections to the database. If conn_max_idle is greater than 0 and the new conn_max_open is less than conn_max_idle, then conn_max_idle will be reduced to match the new conn_max_open limit. If `value \u003c= 0`, then there is no limit on the number of open connections. The default is 0 (unlimited)." }, "credentials": { "type": "object", "description": "Optional manual configuration of AWS credentials to use. More information can be found [in this document](/docs/guides/cloud/aws).", "properties": { "expiry_window": { "type": "string", "description": "Allow the credentials to trigger refreshing prior to the credentials actually expiring. This is beneficial so race conditions with expiring credentials do not cause requests to fail. For example '10s' would refresh credentials ten seconds before expiration. Setting to a duration of `0` disables the expiry window. Default: " }, "from_ec2_role": { "type": "boolean", "description": "Use the credentials of a host EC2 machine configured to assume [an IAM role associated with the instance](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html). Default: false" }, "id": { "type": "string", "description": "The ID of credentials to use. Default: " }, "profile": { "type": "string", "description": "A profile from `~/.aws/credentials` to use. Default: " }, "role": { "type": "string", "description": "A role ARN to assume. Default: " }, "role_external_id": { "type": "string", "description": "An external ID to provide when assuming a role. Default: " }, "secret": { "type": "string", "description": "The secret for the credentials being used. Default: " }, "token": { "type": "string", "description": "The token for the credentials being used, required when using short term credentials. Default: " } } }, "driver": { "type": "string", "description": "A database [driver](#drivers) to use." }, "dsn": { "type": "string", "description": "A Data Source Name to identify the target database.\n\n#### Drivers\n\nThe following is a list of supported drivers, their placeholder style, and their respective DSN formats:\n\n| Driver | Data Source Name Format |\n|---|---|\n| `clickhouse` | [`clickhouse://[username[:password]@][netloc][:port]/dbname[?param1=value1\u0026...\u0026paramN=valueN]`](https://github.com/ClickHouse/clickhouse-go#dsn) |\n| `mysql` | `[username[:password]@][protocol[(address)]]/dbname[?param1=value1\u0026...\u0026paramN=valueN]` |\n| `postgres` | `postgres://[user[:password]@][netloc][:port][/dbname][?param1=value1\u0026...]` |\n| `mssql` | `sqlserver://[user[:password]@][netloc][:port][?database=dbname\u0026param1=value1\u0026...]` |\n| `sqlite` | `file:/path/to/filename.db[?param\u0026=value1\u0026...]` |\n| `oracle` | `oracle://[username[:password]@][netloc][:port]/service_name?server=server2\u0026server=server3` |\n| `snowflake` | `username[:password]@account_identifier/dbname/schemaname[?param1=value\u0026...\u0026paramN=valueN]` |\n| `spanner` | `projects/[project]/instances/[instance]/databases/dbname` |\n| `trino` | [`http[s]://user[:pass]@host[:port][?parameters]`](https://github.com/trinodb/trino-go-client#dsn-data-source-name) |\n| `gocosmos` | [`AccountEndpoint=\u003ccosmosdb-endpoint\u003e;AccountKey=\u003ccosmosdb-account-key\u003e[;TimeoutMs=\u003ctimeout-in-ms\u003e][;Version=\u003ccosmosdb-api-version\u003e][;DefaultDb/Db=\u003cdb-name\u003e][;AutoId=\u003ctrue/false\u003e][;InsecureSkipVerify=\u003ctrue/false\u003e]`](https://pkg.go.dev/github.com/microsoft/gocosmos#readme-example-usage) |\n\nPlease note that the `postgres` driver enforces SSL by default, you can override this with the parameter `sslmode=disable` if required.\n\nThe `snowflake` driver supports multiple DSN formats. Please consult [the docs](https://pkg.go.dev/github.com/snowflakedb/gosnowflake#hdr-Connection_String) for more details. For [key pair authentication](https://docs.snowflake.com/en/user-guide/key-pair-auth.html#configuring-key-pair-authentication), the DSN has the following format: `\u003csnowflake_user\u003e@\u003csnowflake_account\u003e/\u003cdb_name\u003e/\u003cschema_name\u003e?warehouse=\u003cwarehouse\u003e\u0026role=\u003crole\u003e\u0026authenticator=snowflake_jwt\u0026privateKey=\u003cbase64_url_encoded_private_key\u003e`, where the value for the `privateKey` parameter can be constructed from an unencrypted RSA private key file `rsa_key.p8` using `openssl enc -d -base64 -in rsa_key.p8 | basenc --base64url -w0` (you can use `gbasenc` insted of `basenc` on OSX if you install `coreutils` via Homebrew). If you have a password-encrypted private key, you can decrypt it using `openssl pkcs8 -in rsa_key_encrypted.p8 -out rsa_key.p8`. Also, make sure fields such as the username are URL-encoded.\n\nThe [`gocosmos`](https://pkg.go.dev/github.com/microsoft/gocosmos) driver is still experimental, but it has support for [hierarchical partition keys](https://learn.microsoft.com/en-us/azure/cosmos-db/hierarchical-partition-keys) as well as [cross-partition queries](https://learn.microsoft.com/en-us/azure/cosmos-db/nosql/how-to-query-container#cross-partition-query). Please refer to the [SQL notes](https://github.com/microsoft/gocosmos/blob/main/SQL.md) for details.", "examples": [ "clickhouse://username:password@host1:9000,host2:9000/database?dial_timeout=200ms\u0026max_execution_time=60", "foouser:foopassword@tcp(localhost:3306)/foodb", "postgres://foouser:foopass@localhost:5432/foodb?sslmode=disable", "oracle://foouser:foopass@localhost:1521/service_name" ] }, "endpoint": { "type": "string", "description": "Allows you to specify a custom endpoint for the AWS API. Default: " }, "iam_enabled": { "type": "boolean", "description": "An optional field used to generate an IAM authentication token to connect to an Amazon Relational Database (RDS) DB instance. This will overwrite the Password in the DSN with the generated token only if the drivers are `mysql` or `postgres`. Default: false" }, "init_files": { "type": "array", "description": "\nAn optional list of file paths containing SQL statements to execute immediately upon the first connection to the target database. This is a useful way to initialise tables before processing data. Glob patterns are supported, including super globs (double star).\n\nCare should be taken to ensure that the statements are idempotent, and therefore would not cause issues when run multiple times after service restarts. If both `init_statement` and `init_files` are specified the `init_statement` is executed _after_ the `init_files`.\n\nIf a statement fails for any reason a warning log will be emitted but the operation of this component will not be stopped.\n", "items": { "type": "string" }, "examples": [ [ "./init/*.sql" ], [ "./foo.sql", "./bar.sql" ] ] }, "init_statement": { "type": "string", "description": "\nAn optional SQL statement to execute immediately upon the first connection to the target database. This is a useful way to initialise tables before processing data. Care should be taken to ensure that the statement is idempotent, and therefore would not cause issues when run multiple times after service restarts.\n\nIf both `init_statement` and `init_files` are specified the `init_statement` is executed _after_ the `init_files`.\n\nIf the statement fails for any reason a warning log will be emitted but the operation of this component will not be stopped.\n", "examples": [ "\nCREATE TABLE IF NOT EXISTS some_table (\n foo varchar(50) not null,\n bar integer,\n baz varchar(50),\n primary key (foo)\n) WITHOUT ROWID;\n" ] }, "init_verify_conn": { "type": "boolean", "description": "Whether to verify the database connection on startup by performing a simple ping, by default this is disabled. Default: false" }, "query": { "type": "string", "description": "The query to execute. The style of placeholder to use depends on the driver, some drivers require question marks (`?`) whereas others expect incrementing dollar signs (`$1`, `$2`, and so on) or colons (`:1`, `:2` and so on). The style to use is outlined in this table:\n\n| Driver | Placeholder Style |\n|---|---|\n| `clickhouse` | Dollar sign |\n| `mysql` | Question mark |\n| `postgres` | Dollar sign |\n| `mssql` | Question mark |\n| `sqlite` | Question mark |\n| `oracle` | Colon |\n| `snowflake` | Question mark |\n| `spanner` | Question mark |\n| `trino` | Question mark |\n| `gocosmos` | Colon |\n", "examples": [ "SELECT * FROM footable WHERE user_id = $1;" ] }, "region": { "type": "string", "description": "The AWS region to target. Default: " }, "secret_name": { "type": "string", "description": "An optional field that can be used to get the Username + Password from AWS Secrets Manager. This will overwrite the Username + Password in the DSN with the values from the Secret only if the driver is set to `postgres`." } } }, "sql_select": { "type": "object", "properties": { "args_mapping": { "type": "string", "description": "An optional [Bloblang mapping](/docs/guides/bloblang/about) which should evaluate to an array of values matching in size to the number of placeholder arguments in the field `where`.", "examples": [ "root = [ \"article\", now().ts_format(\"2006-01-02\") ]" ] }, "auto_replay_nacks": { "type": "boolean", "description": "Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation. Default: true" }, "azure": { "type": "object", "description": "Optional Fields that can be set to use Azure based authentication for Azure Postgres SQL", "properties": { "entra_enabled": { "type": "boolean", "description": "An optional field used to generate an entra token to connect to 'Azure Database for PostgreSQL flexible server', This will create a new connection string with the host, user and database from the DSN field - you may need to URL encode the dsn! The [Default Azure Credential Chain](https://learn.microsoft.com/en-gb/azure/developer/go/sdk/authentication/authentication-overview#defaultazurecredential) is used from the Azure SDK. Default: false" }, "token_request_options": { "type": "object", "properties": { "claims": { "type": "string", "description": "Set additional claims for the token. Default: " }, "enable_cae": { "type": "boolean", "description": "Indicates whether to enable Continuous Access Evaluation (CAE) for the requested token Default: false" }, "scopes": { "type": "array", "description": "Scopes contains the list of permission scopes required for the token. Default: [https://ossrdbms-aad.database.windows.net/.default]", "items": { "type": "string" } }, "tenant_id": { "type": "string", "description": "tenant_id identifies the tenant from which to request the token. azure credentials authenticate in their configured default tenants when this field isn't set. Default: " } } } } }, "columns": { "type": "array", "description": "A list of columns to select.", "items": { "type": "string" }, "examples": [ [ "*" ], [ "foo", "bar", "baz" ] ] }, "conn_max_idle": { "type": "integer", "description": "An optional maximum number of connections in the idle connection pool. If conn_max_open is greater than 0 but less than the new conn_max_idle, then the new conn_max_idle will be reduced to match the conn_max_open limit. If `value \u003c= 0`, no idle connections are retained. The default max idle connections is currently 2. This may change in a future release. Default: 2" }, "conn_max_idle_time": { "type": "string", "description": "An optional maximum amount of time a connection may be idle. Expired connections may be closed lazily before reuse. If `value \u003c= 0`, connections are not closed due to a connections idle time." }, "conn_max_life_time": { "type": "string", "description": "An optional maximum amount of time a connection may be reused. Expired connections may be closed lazily before reuse. If `value \u003c= 0`, connections are not closed due to a connections age." }, "conn_max_open": { "type": "integer", "description": "An optional maximum number of open connections to the database. If conn_max_idle is greater than 0 and the new conn_max_open is less than conn_max_idle, then conn_max_idle will be reduced to match the new conn_max_open limit. If `value \u003c= 0`, then there is no limit on the number of open connections. The default is 0 (unlimited)." }, "credentials": { "type": "object", "description": "Optional manual configuration of AWS credentials to use. More information can be found [in this document](/docs/guides/cloud/aws).", "properties": { "expiry_window": { "type": "string", "description": "Allow the credentials to trigger refreshing prior to the credentials actually expiring. This is beneficial so race conditions with expiring credentials do not cause requests to fail. For example '10s' would refresh credentials ten seconds before expiration. Setting to a duration of `0` disables the expiry window. Default: " }, "from_ec2_role": { "type": "boolean", "description": "Use the credentials of a host EC2 machine configured to assume [an IAM role associated with the instance](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html). Default: false" }, "id": { "type": "string", "description": "The ID of credentials to use. Default: " }, "profile": { "type": "string", "description": "A profile from `~/.aws/credentials` to use. Default: " }, "role": { "type": "string", "description": "A role ARN to assume. Default: " }, "role_external_id": { "type": "string", "description": "An external ID to provide when assuming a role. Default: " }, "secret": { "type": "string", "description": "The secret for the credentials being used. Default: " }, "token": { "type": "string", "description": "The token for the credentials being used, required when using short term credentials. Default: " } } }, "driver": { "type": "string", "description": "A database [driver](#drivers) to use." }, "dsn": { "type": "string", "description": "A Data Source Name to identify the target database.\n\n#### Drivers\n\nThe following is a list of supported drivers, their placeholder style, and their respective DSN formats:\n\n| Driver | Data Source Name Format |\n|---|---|\n| `clickhouse` | [`clickhouse://[username[:password]@][netloc][:port]/dbname[?param1=value1\u0026...\u0026paramN=valueN]`](https://github.com/ClickHouse/clickhouse-go#dsn) |\n| `mysql` | `[username[:password]@][protocol[(address)]]/dbname[?param1=value1\u0026...\u0026paramN=valueN]` |\n| `postgres` | `postgres://[user[:password]@][netloc][:port][/dbname][?param1=value1\u0026...]` |\n| `mssql` | `sqlserver://[user[:password]@][netloc][:port][?database=dbname\u0026param1=value1\u0026...]` |\n| `sqlite` | `file:/path/to/filename.db[?param\u0026=value1\u0026...]` |\n| `oracle` | `oracle://[username[:password]@][netloc][:port]/service_name?server=server2\u0026server=server3` |\n| `snowflake` | `username[:password]@account_identifier/dbname/schemaname[?param1=value\u0026...\u0026paramN=valueN]` |\n| `spanner` | `projects/[project]/instances/[instance]/databases/dbname` |\n| `trino` | [`http[s]://user[:pass]@host[:port][?parameters]`](https://github.com/trinodb/trino-go-client#dsn-data-source-name) |\n| `gocosmos` | [`AccountEndpoint=\u003ccosmosdb-endpoint\u003e;AccountKey=\u003ccosmosdb-account-key\u003e[;TimeoutMs=\u003ctimeout-in-ms\u003e][;Version=\u003ccosmosdb-api-version\u003e][;DefaultDb/Db=\u003cdb-name\u003e][;AutoId=\u003ctrue/false\u003e][;InsecureSkipVerify=\u003ctrue/false\u003e]`](https://pkg.go.dev/github.com/microsoft/gocosmos#readme-example-usage) |\n\nPlease note that the `postgres` driver enforces SSL by default, you can override this with the parameter `sslmode=disable` if required.\n\nThe `snowflake` driver supports multiple DSN formats. Please consult [the docs](https://pkg.go.dev/github.com/snowflakedb/gosnowflake#hdr-Connection_String) for more details. For [key pair authentication](https://docs.snowflake.com/en/user-guide/key-pair-auth.html#configuring-key-pair-authentication), the DSN has the following format: `\u003csnowflake_user\u003e@\u003csnowflake_account\u003e/\u003cdb_name\u003e/\u003cschema_name\u003e?warehouse=\u003cwarehouse\u003e\u0026role=\u003crole\u003e\u0026authenticator=snowflake_jwt\u0026privateKey=\u003cbase64_url_encoded_private_key\u003e`, where the value for the `privateKey` parameter can be constructed from an unencrypted RSA private key file `rsa_key.p8` using `openssl enc -d -base64 -in rsa_key.p8 | basenc --base64url -w0` (you can use `gbasenc` insted of `basenc` on OSX if you install `coreutils` via Homebrew). If you have a password-encrypted private key, you can decrypt it using `openssl pkcs8 -in rsa_key_encrypted.p8 -out rsa_key.p8`. Also, make sure fields such as the username are URL-encoded.\n\nThe [`gocosmos`](https://pkg.go.dev/github.com/microsoft/gocosmos) driver is still experimental, but it has support for [hierarchical partition keys](https://learn.microsoft.com/en-us/azure/cosmos-db/hierarchical-partition-keys) as well as [cross-partition queries](https://learn.microsoft.com/en-us/azure/cosmos-db/nosql/how-to-query-container#cross-partition-query). Please refer to the [SQL notes](https://github.com/microsoft/gocosmos/blob/main/SQL.md) for details.", "examples": [ "clickhouse://username:password@host1:9000,host2:9000/database?dial_timeout=200ms\u0026max_execution_time=60", "foouser:foopassword@tcp(localhost:3306)/foodb", "postgres://foouser:foopass@localhost:5432/foodb?sslmode=disable", "oracle://foouser:foopass@localhost:1521/service_name" ] }, "endpoint": { "type": "string", "description": "Allows you to specify a custom endpoint for the AWS API. Default: " }, "iam_enabled": { "type": "boolean", "description": "An optional field used to generate an IAM authentication token to connect to an Amazon Relational Database (RDS) DB instance. This will overwrite the Password in the DSN with the generated token only if the drivers are `mysql` or `postgres`. Default: false" }, "init_files": { "type": "array", "description": "\nAn optional list of file paths containing SQL statements to execute immediately upon the first connection to the target database. This is a useful way to initialise tables before processing data. Glob patterns are supported, including super globs (double star).\n\nCare should be taken to ensure that the statements are idempotent, and therefore would not cause issues when run multiple times after service restarts. If both `init_statement` and `init_files` are specified the `init_statement` is executed _after_ the `init_files`.\n\nIf a statement fails for any reason a warning log will be emitted but the operation of this component will not be stopped.\n", "items": { "type": "string" }, "examples": [ [ "./init/*.sql" ], [ "./foo.sql", "./bar.sql" ] ] }, "init_statement": { "type": "string", "description": "\nAn optional SQL statement to execute immediately upon the first connection to the target database. This is a useful way to initialise tables before processing data. Care should be taken to ensure that the statement is idempotent, and therefore would not cause issues when run multiple times after service restarts.\n\nIf both `init_statement` and `init_files` are specified the `init_statement` is executed _after_ the `init_files`.\n\nIf the statement fails for any reason a warning log will be emitted but the operation of this component will not be stopped.\n", "examples": [ "\nCREATE TABLE IF NOT EXISTS some_table (\n foo varchar(50) not null,\n bar integer,\n baz varchar(50),\n primary key (foo)\n) WITHOUT ROWID;\n" ] }, "init_verify_conn": { "type": "boolean", "description": "Whether to verify the database connection on startup by performing a simple ping, by default this is disabled. Default: false" }, "prefix": { "type": "string", "description": "An optional prefix to prepend to the select query (before SELECT)." }, "region": { "type": "string", "description": "The AWS region to target. Default: " }, "secret_name": { "type": "string", "description": "An optional field that can be used to get the Username + Password from AWS Secrets Manager. This will overwrite the Username + Password in the DSN with the values from the Secret only if the driver is set to `postgres`." }, "suffix": { "type": "string", "description": "An optional suffix to append to the select query." }, "table": { "type": "string", "description": "The table to select from.", "examples": [ "foo" ] }, "where": { "type": "string", "description": "An optional where clause to add. Placeholder arguments are populated with the `args_mapping` field. Placeholders should always be question marks, and will automatically be converted to dollar syntax when the postgres or clickhouse drivers are used.", "examples": [ "type = ? and created_at \u003e ?", "user_id = ?" ] } } }, "stdin": { "type": "object", "properties": { "auto_replay_nacks": { "type": "boolean", "description": "Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation. Default: true" }, "codec": { "type": "string", "description": "The way in which the bytes of a data source should be converted into discrete messages, codecs are useful for specifying how large files or continuous streams of data might be processed in small chunks rather than loading it all in memory. It's possible to consume lines using a custom delimiter with the `delim:x` codec, where x is the character sequence custom delimiter. Codecs can be chained with `/`, for example a gzip compressed CSV file can be consumed with the codec `gzip/csv`.", "examples": [ "lines", "delim:\t", "delim:foobar", "gzip/csv" ] }, "max_buffer": { "type": "integer", "description": " Default: 1000000" }, "scanner": { "type": "object", "description": "The [scanner](/docs/components/scanners/about) by which the stream of bytes consumed will be broken out into individual messages. Scanners are useful for processing large sources of data without holding the entirety of it within memory. For example, the `csv` scanner allows you to process individual CSV rows without loading the entire CSV file in memory at once. Default: map[lines:map[]]" } } }, "subprocess": { "type": "object", "properties": { "args": { "type": "array", "description": "A list of arguments to provide the command. Default: []", "items": { "type": "string" } }, "codec": { "type": "string", "description": "The way in which messages should be consumed from the subprocess. Default: lines" }, "max_buffer": { "type": "integer", "description": "The maximum expected size of an individual message. Default: 65536" }, "name": { "type": "string", "description": "The command to execute as a subprocess.", "examples": [ "cat", "sed", "awk" ] }, "restart_on_exit": { "type": "boolean", "description": "Whether the command should be re-executed each time the subprocess ends. Default: false" } } }, "twitter_search": { "type": "object", "properties": { "api_key": { "type": "string", "description": "An API key for OAuth 2.0 authentication. It is recommended that you populate this field using [environment variables](/docs/configuration/interpolation)." }, "api_secret": { "type": "string", "description": "An API secret for OAuth 2.0 authentication. It is recommended that you populate this field using [environment variables](/docs/configuration/interpolation)." }, "backfill_period": { "type": "string", "description": "A duration string indicating the maximum age of tweets to acquire when starting a search. Default: 5m" }, "cache": { "type": "string", "description": "A cache resource to use for request pagination." }, "cache_key": { "type": "string", "description": "The key identifier used when storing the ID of the last tweet received. Default: last_tweet_id" }, "poll_period": { "type": "string", "description": "The length of time (as a duration string) to wait between each search request. This field can be set empty, in which case requests are made at the limit set by the rate limit. This field also supports cron expressions. Default: 1m" }, "query": { "type": "string", "description": "A search expression to use." }, "rate_limit": { "type": "string", "description": "An optional rate limit resource to restrict API requests with. Default: " }, "tweet_fields": { "type": "array", "description": "An optional list of additional fields to obtain for each tweet, by default only the fields `id` and `text` are returned. For more info refer to the [twitter API docs.](https://developer.twitter.com/en/docs/twitter-api/fields) Default: []", "items": { "type": "string" } } } }, "websocket": { "type": "object", "properties": { "auto_replay_nacks": { "type": "boolean", "description": "Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation. Default: true" }, "basic_auth": { "type": "object", "description": "Allows you to specify basic authentication.", "properties": { "enabled": { "type": "boolean", "description": "Whether to use basic authentication in requests. Default: false" }, "password": { "type": "string", "description": "A password to authenticate with. Default: " }, "username": { "type": "string", "description": "A username to authenticate as. Default: " } } }, "connection": { "type": "object", "description": "Customise how websocket connection attempts are made.", "properties": { "max_retries": { "type": "integer", "description": "An optional limit to the number of consecutive retry attempts that will be made before abandoning the connection altogether and gracefully terminating the input. When all inputs terminate in this way the service (or stream) will shut down. If set to zero connections will never be reattempted upon a failure. If set below zero this field is ignored (effectively unset).", "examples": [ -1, 10 ] } } }, "headers": { "type": "object", "description": "A map of custom headers to add to the websocket handshake. Default: map[]", "examples": [ { "Sec-WebSocket-Protocol": "graphql-ws", "User-Agent": "${! uuid_v4() }", "X-Client-ID": "${CLIENT_ID}" } ] }, "jwt": { "type": "object", "description": "BETA: Allows you to specify JWT authentication.", "properties": { "claims": { "type": "object", "description": "A value used to identify the claims that issued the JWT. Default: map[]" }, "enabled": { "type": "boolean", "description": "Whether to use JWT authentication in requests. Default: false" }, "headers": { "type": "object", "description": "Add optional key/value headers to the JWT. Default: map[]" }, "private_key_file": { "type": "string", "description": "A file with the PEM encoded via PKCS1 or PKCS8 as private key. Default: " }, "signing_method": { "type": "string", "description": "A method used to sign the token such as RS256, RS384, RS512 or EdDSA. Default: " } } }, "oauth": { "type": "object", "description": "Allows you to specify open authentication via OAuth version 1.", "properties": { "access_token": { "type": "string", "description": "A value used to gain access to the protected resources on behalf of the user. Default: " }, "access_token_secret": { "type": "string", "description": "A secret provided in order to establish ownership of a given access token. Default: " }, "consumer_key": { "type": "string", "description": "A value used to identify the client to the service provider. Default: " }, "consumer_secret": { "type": "string", "description": "A secret used to establish ownership of the consumer key. Default: " }, "enabled": { "type": "boolean", "description": "Whether to use OAuth version 1 in requests. Default: false" } } }, "open_message": { "type": "string", "description": "An optional message to send to the server upon connection." }, "open_message_type": { "type": "string", "description": "An optional flag to indicate the data type of open_message. Default: binary" }, "open_messages": { "type": "array", "description": "An optional list of messages to send to the server upon connection. This field replaces `open_message`, which will be removed in a future version.", "items": { "type": "string" } }, "proxy_url": { "type": "string", "description": "An optional HTTP proxy URL." }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "url": { "type": "string", "description": "The URL to connect to.", "examples": [ "ws://localhost:4195/get/ws" ] } } }, "zmq4n": { "type": "object", "properties": { "bind": { "type": "boolean", "description": "Whether to bind to the specified URLs (otherwise they are connected to). Default: false" }, "high_water_mark": { "type": "integer", "description": "The message high water mark to use. (experimental i go-zeromq) Default: 0" }, "poll_timeout": { "type": "string", "description": "The poll timeout to use. Default: 5s" }, "socket_type": { "type": "string", "description": "The socket type to connect as." }, "sub_filters": { "type": "array", "description": "A list of subscription topic filters to use when consuming from a SUB socket. Specifying a single sub_filter of `''` will subscribe to everything. Default: []", "items": { "type": "string" } }, "urls": { "type": "array", "description": "A list of URLs to connect to. If an item of the list contains commas it will be expanded into multiple URLs.", "items": { "type": "string" }, "examples": [ [ "tcp://localhost:5555" ] ] } } } }, "items": { "type": "object" } }, "logger": { "type": "object", "description": "Describes how operational logs should be emitted.", "properties": { "add_timestamp": { "type": "boolean", "description": "Whether to include timestamps in logs. Default: false" }, "file": { "type": "object", "description": "Experimental: Specify fields for optionally writing logs to a file.", "properties": { "path": { "type": "string", "description": "The file path to write logs to, if the file does not exist it will be created. Leave this field empty or unset to disable file based logging. Default: " }, "rotate": { "type": "boolean", "description": "Whether to rotate log files automatically. Default: false" }, "rotate_max_age_days": { "type": "integer", "description": "The maximum number of days to retain old log files based on the timestamp encoded in their filename, after which they are deleted. Setting to zero disables this mechanism. Default: 0" } } }, "format": { "type": "string", "description": "Set the format of emitted logs. Default: logfmt" }, "level": { "type": "string", "description": "Set the minimum severity level for emitting logs. Default: INFO" }, "level_name": { "type": "string", "description": "The name of the level field added to logs when the `format` is `json`. Default: level" }, "log_all_errors": { "type": "boolean", "description": "Experimental: When set, the logger will promote any log containing an `error` type to the `ERROR` level. Default: false" }, "message_name": { "type": "string", "description": "The name of the message field added to logs when the `format` is `json`. Default: msg" }, "static_fields": { "type": "object", "description": "A map of key/value pairs to add to each structured log. Default: map[@service:bento]" }, "timestamp_name": { "type": "string", "description": "The name of the timestamp field added to logs when `add_timestamp` is set to `true` and the `format` is `json`. Default: time" } } }, "metrics": { "type": "object", "description": "A mechanism for exporting metrics. Default: map[mapping: prometheus:map[]]", "properties": { "aws_cloudwatch": { "type": "object", "properties": { "credentials": { "type": "object", "description": "Optional manual configuration of AWS credentials to use. More information can be found [in this document](/docs/guides/cloud/aws).", "properties": { "expiry_window": { "type": "string", "description": "Allow the credentials to trigger refreshing prior to the credentials actually expiring. This is beneficial so race conditions with expiring credentials do not cause requests to fail. For example '10s' would refresh credentials ten seconds before expiration. Setting to a duration of `0` disables the expiry window. Default: " }, "from_ec2_role": { "type": "boolean", "description": "Use the credentials of a host EC2 machine configured to assume [an IAM role associated with the instance](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html). Default: false" }, "id": { "type": "string", "description": "The ID of credentials to use. Default: " }, "profile": { "type": "string", "description": "A profile from `~/.aws/credentials` to use. Default: " }, "role": { "type": "string", "description": "A role ARN to assume. Default: " }, "role_external_id": { "type": "string", "description": "An external ID to provide when assuming a role. Default: " }, "secret": { "type": "string", "description": "The secret for the credentials being used. Default: " }, "token": { "type": "string", "description": "The token for the credentials being used, required when using short term credentials. Default: " } } }, "endpoint": { "type": "string", "description": "Allows you to specify a custom endpoint for the AWS API. Default: " }, "flush_period": { "type": "string", "description": "The period of time between PutMetricData requests. Default: 100ms" }, "namespace": { "type": "string", "description": "The namespace used to distinguish metrics from other services. Default: Bento" }, "region": { "type": "string", "description": "The AWS region to target. Default: " } } }, "influxdb": { "type": "object", "properties": { "db": { "type": "string", "description": "The name of the database to use." }, "include": { "type": "object", "description": "Optional additional metrics to collect, enabling these metrics may have some performance implications as it acquires a global semaphore and does `stoptheworld()`.", "properties": { "debug_gc": { "type": "string", "description": "A duration string indicating how often to poll and collect GC metrics. Leave empty to disable this metric. Default: ", "examples": [ "1m" ] }, "runtime": { "type": "string", "description": "A duration string indicating how often to poll and collect runtime metrics. Leave empty to disable this metric Default: ", "examples": [ "1m" ] } } }, "interval": { "type": "string", "description": "A duration string indicating how often metrics should be flushed. Default: 1m" }, "password": { "type": "string", "description": "A password (when applicable). Default: " }, "ping_interval": { "type": "string", "description": "A duration string indicating how often to ping the host. Default: 20s" }, "precision": { "type": "string", "description": "[ns|us|ms|s] timestamp precision passed to write api. Default: s" }, "retention_policy": { "type": "string", "description": "Sets the retention policy for each write." }, "tags": { "type": "object", "description": "Global tags added to each metric. Default: map[]", "examples": [ { "hostname": "localhost", "zone": "danger" } ] }, "timeout": { "type": "string", "description": "How long to wait for response for both ping and writing metrics. Default: 5s" }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "url": { "type": "string", "description": "A URL of the format `[https|http|udp]://host:port` to the InfluxDB host." }, "username": { "type": "string", "description": "A username (when applicable). Default: " }, "write_consistency": { "type": "string", "description": "[any|one|quorum|all] sets write consistency when available." } } }, "json_api": { "type": "object", "description": " Default: map[]" }, "logger": { "type": "object", "properties": { "flush_metrics": { "type": "boolean", "description": "Whether counters and timing metrics should be reset to 0 each time metrics are printed. Default: false" }, "push_interval": { "type": "string", "description": "An optional period of time to continuously print all metrics." } } }, "none": { "type": "object", "description": " Default: map[]" }, "prometheus": { "type": "object", "properties": { "add_go_metrics": { "type": "boolean", "description": "Whether to export Go runtime metrics such as GC pauses in addition to Bento metrics. Default: false" }, "add_process_metrics": { "type": "boolean", "description": "Whether to export process metrics such as CPU and memory usage in addition to Bento metrics. Default: false" }, "file_output_path": { "type": "string", "description": "An optional file path to write all prometheus metrics on service shutdown. Default: " }, "histogram_buckets": { "type": "number", "description": "Timing metrics histogram buckets (in seconds). If left empty defaults to DefBuckets (https://pkg.go.dev/github.com/prometheus/client_golang/prometheus#pkg-variables). Applicable when `use_histogram_timing` is set to `true`. Default: []", "items": { "type": "number" } }, "push_basic_auth": { "type": "object", "description": "The Basic Authentication credentials.", "properties": { "password": { "type": "string", "description": "The Basic Authentication password. Default: " }, "username": { "type": "string", "description": "The Basic Authentication username. Default: " } } }, "push_interval": { "type": "string", "description": "The period of time between each push when sending metrics to a Push Gateway." }, "push_job_name": { "type": "string", "description": "An identifier for push jobs. Default: bento_push" }, "push_url": { "type": "string", "description": "An optional [Push Gateway URL](#push-gateway) to push metrics to." }, "summary_quantiles_objectives": { "type": "array", "description": "A list of timing metrics summary buckets (as quantiles). Applicable when `use_histogram_timing` is set to `false`. Default: [map[error:0.05 quantile:0.5] map[error:0.01 quantile:0.9] map[error:0.001 quantile:0.99]]", "items": { "type": "object" }, "examples": [ [ { "error": 0.05, "quantile": 0.5 }, { "error": 0.01, "quantile": 0.9 }, { "error": 0.001, "quantile": 0.99 } ] ] }, "use_default_registry": { "type": "boolean", "description": "Whether to use the same Prometheus registry as the main process or create a new one. Most useful when using the StreamBuilder API and you want the stream metrics to be scraped from the same port as the metrics of the parent process. Default: false" }, "use_histogram_timing": { "type": "boolean", "description": "Whether to export timing metrics as a histogram, if `false` a summary is used instead. When exporting histogram timings the delta values are converted from nanoseconds into seconds in order to better fit within bucket definitions. For more information on histograms and summaries refer to: https://prometheus.io/docs/practices/histograms/. Default: false" } } }, "statsd": { "type": "object", "properties": { "address": { "type": "string", "description": "The address to send metrics to." }, "flush_period": { "type": "string", "description": "The time interval between metrics flushes. Default: 100ms" }, "tag_format": { "type": "string", "description": "Metrics tagging is supported in a variety of formats. Default: none" } } } } }, "output": { "type": "object", "description": "An output to sink messages to. Default: map[stdout:map[]]", "properties": { "amqp_0_9": { "type": "object", "properties": { "app_id": { "type": "string", "description": "Set the application ID of each message with a dynamic interpolated expression. Default: " }, "content_encoding": { "type": "string", "description": "The content encoding attribute to set for each message. Default: " }, "content_type": { "type": "string", "description": "The content type attribute to set for each message. Default: application/octet-stream" }, "correlation_id": { "type": "string", "description": "Set the correlation ID of each message with a dynamic interpolated expression. Default: " }, "exchange": { "type": "string", "description": "An AMQP exchange to publish to." }, "exchange_declare": { "type": "object", "description": "Optionally declare the target exchange (passive).", "properties": { "durable": { "type": "boolean", "description": "Whether the exchange should be durable. Default: true" }, "enabled": { "type": "boolean", "description": "Whether to declare the exchange. Default: false" }, "type": { "type": "string", "description": "The type of the exchange. Default: direct" } } }, "expiration": { "type": "string", "description": "Set the per-message TTL Default: " }, "immediate": { "type": "boolean", "description": "Whether to set the immediate flag on published messages. When set if there are no ready consumers of a queue then the message is dropped instead of waiting. Default: false" }, "key": { "type": "string", "description": "The binding key to set for each message. Default: " }, "mandatory": { "type": "boolean", "description": "Whether to set the mandatory flag on published messages. When set if a published message is routed to zero queues it is returned. Default: false" }, "max_in_flight": { "type": "integer", "description": "The maximum number of messages to have in flight at a given time. Increase this to improve throughput. Default: 64" }, "message_id": { "type": "string", "description": "Set the message ID of each message with a dynamic interpolated expression. Default: " }, "metadata": { "type": "object", "description": "Specify criteria for which metadata values are attached to messages as headers.", "properties": { "exclude_prefixes": { "type": "array", "description": "Provide a list of explicit metadata key prefixes to be excluded when adding metadata to sent messages. Default: []", "items": { "type": "string" } } } }, "persistent": { "type": "boolean", "description": "Whether message delivery should be persistent (transient by default). Default: false" }, "priority": { "type": "string", "description": "Set the priority of each message with a dynamic interpolated expression. Default: ", "examples": [ "0", "${! metadata(\"amqp_priority\") }", "${! json(\"doc.priority\") }" ] }, "reply_to": { "type": "string", "description": "Carries response queue name - set with a dynamic interpolated expression. Default: " }, "timeout": { "type": "string", "description": "The maximum period to wait before abandoning it and reattempting. If not set, wait indefinitely. Default: " }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "type": { "type": "string", "description": "The type property to set for each message. Default: " }, "urls": { "type": "array", "description": "A list of URLs to connect to. The first URL to successfully establish a connection will be used until the connection is closed. If an item of the list contains commas it will be expanded into multiple URLs.", "items": { "type": "string" }, "examples": [ [ "amqp://guest:guest@127.0.0.1:5672/" ], [ "amqp://127.0.0.1:5672/,amqp://127.0.0.2:5672/" ], [ "amqp://127.0.0.1:5672/", "amqp://127.0.0.2:5672/" ] ] }, "user_id": { "type": "string", "description": "Set the user ID to the name of the publisher. If this property is set by a publisher, its value must be equal to the name of the user used to open the connection. Default: " } } }, "amqp_1": { "type": "object", "properties": { "application_properties_map": { "type": "string", "description": "An optional Bloblang mapping that can be defined in order to set the `application-properties` on output messages." }, "max_in_flight": { "type": "integer", "description": "The maximum number of messages to have in flight at a given time. Increase this to improve throughput. Default: 64" }, "metadata": { "type": "object", "description": "Specify criteria for which metadata values are attached to messages as headers.", "properties": { "exclude_prefixes": { "type": "array", "description": "Provide a list of explicit metadata key prefixes to be excluded when adding metadata to sent messages. Default: []", "items": { "type": "string" } } } }, "sasl": { "type": "object", "description": "Enables SASL authentication.", "properties": { "mechanism": { "type": "string", "description": "The SASL authentication mechanism to use. Default: none" }, "password": { "type": "string", "description": "A SASL plain text password. It is recommended that you use environment variables to populate this field. Default: ", "examples": [ "${PASSWORD}" ] }, "user": { "type": "string", "description": "A SASL plain text username. It is recommended that you use environment variables to populate this field. Default: ", "examples": [ "${USER}" ] } } }, "target_address": { "type": "string", "description": "The target address to write to.", "examples": [ "/foo", "queue:/bar", "topic:/baz" ] }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "url": { "type": "string", "description": "A URL to connect to.", "examples": [ "amqp://localhost:5672/", "amqps://guest:guest@localhost:5672/" ] }, "urls": { "type": "array", "description": "A list of URLs to connect to. The first URL to successfully establish a connection will be used until the connection is closed. If an item of the list contains commas it will be expanded into multiple URLs.", "items": { "type": "string" }, "examples": [ [ "amqp://guest:guest@127.0.0.1:5672/" ], [ "amqp://127.0.0.1:5672/,amqp://127.0.0.2:5672/" ], [ "amqp://127.0.0.1:5672/", "amqp://127.0.0.2:5672/" ] ] } } }, "aws_dynamodb": { "type": "object", "properties": { "backoff": { "type": "object", "description": "Control time intervals between retry attempts.", "properties": { "initial_interval": { "type": "string", "description": "The initial period to wait between retry attempts. Default: 1s" }, "max_elapsed_time": { "type": "string", "description": "The maximum period to wait before retry attempts are abandoned. If zero then no limit is used. Default: 30s" }, "max_interval": { "type": "string", "description": "The maximum period to wait between retry attempts. Default: 5s" } } }, "batching": { "type": "object", "description": "\nAllows you to configure a [batching policy](/docs/configuration/batching).", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] }, "credentials": { "type": "object", "description": "Optional manual configuration of AWS credentials to use. More information can be found [in this document](/docs/guides/cloud/aws).", "properties": { "expiry_window": { "type": "string", "description": "Allow the credentials to trigger refreshing prior to the credentials actually expiring. This is beneficial so race conditions with expiring credentials do not cause requests to fail. For example '10s' would refresh credentials ten seconds before expiration. Setting to a duration of `0` disables the expiry window. Default: " }, "from_ec2_role": { "type": "boolean", "description": "Use the credentials of a host EC2 machine configured to assume [an IAM role associated with the instance](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html). Default: false" }, "id": { "type": "string", "description": "The ID of credentials to use. Default: " }, "profile": { "type": "string", "description": "A profile from `~/.aws/credentials` to use. Default: " }, "role": { "type": "string", "description": "A role ARN to assume. Default: " }, "role_external_id": { "type": "string", "description": "An external ID to provide when assuming a role. Default: " }, "secret": { "type": "string", "description": "The secret for the credentials being used. Default: " }, "token": { "type": "string", "description": "The token for the credentials being used, required when using short term credentials. Default: " } } }, "delete": { "type": "object", "description": "Optional config fields that enable creating Delete requests from messages. If the bloblang mapping provided in `delete.condition` resolves to true, a delete request for the corresponding partition key will be made.", "properties": { "condition": { "type": "string", "description": "A bloblang mapping that should return a bool, that will determine if the message will be used to create a Delete rather than Put Default: ", "examples": [ "root = this.isDelete == \"true\"" ] }, "partition_key": { "type": "string", "description": "The partition key for DeleteItem requests. Required when `delete.condition` is true. The value of the key will be resolved from either `string_columns or json_map_columns` Default: " }, "sort_key": { "type": "string", "description": "The sort key for DeleteItem requests. The value of the key will be resolved from either `string_columns or json_map_columns` Default: " } } }, "endpoint": { "type": "string", "description": "Allows you to specify a custom endpoint for the AWS API. Default: " }, "json_map_columns": { "type": "object", "description": "A map of column keys to [field paths](/docs/configuration/field_paths) pointing to value data within messages. Default: map[]", "examples": [ { "user": "path.to.user", "whole_document": "." }, { "": "." } ] }, "max_in_flight": { "type": "integer", "description": "The maximum number of messages to have in flight at a given time. Increase this to improve throughput. Default: 64" }, "max_retries": { "type": "integer", "description": "The maximum number of retries before giving up on the request. If set to zero there is no discrete limit. Default: 3" }, "region": { "type": "string", "description": "The AWS region to target. Default: " }, "string_columns": { "type": "object", "description": "A map of column keys to string values to store. Default: map[]", "examples": [ { "full_content": "${!content()}", "id": "${!json(\"id\")}", "title": "${!json(\"body.title\")}", "topic": "${!meta(\"kafka_topic\")}" } ] }, "table": { "type": "string", "description": "The table to store messages in." }, "ttl": { "type": "string", "description": "An optional TTL to set for items, calculated from the moment the message is sent. Default: " }, "ttl_key": { "type": "string", "description": "The column key to place the TTL value within. Default: " } } }, "aws_kinesis": { "type": "object", "properties": { "backoff": { "type": "object", "description": "Control time intervals between retry attempts.", "properties": { "initial_interval": { "type": "string", "description": "The initial period to wait between retry attempts. Default: 1s" }, "max_elapsed_time": { "type": "string", "description": "The maximum period to wait before retry attempts are abandoned. If zero then no limit is used. Default: 30s" }, "max_interval": { "type": "string", "description": "The maximum period to wait between retry attempts. Default: 5s" } } }, "batching": { "type": "object", "description": "\nAllows you to configure a [batching policy](/docs/configuration/batching).", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] }, "credentials": { "type": "object", "description": "Optional manual configuration of AWS credentials to use. More information can be found [in this document](/docs/guides/cloud/aws).", "properties": { "expiry_window": { "type": "string", "description": "Allow the credentials to trigger refreshing prior to the credentials actually expiring. This is beneficial so race conditions with expiring credentials do not cause requests to fail. For example '10s' would refresh credentials ten seconds before expiration. Setting to a duration of `0` disables the expiry window. Default: " }, "from_ec2_role": { "type": "boolean", "description": "Use the credentials of a host EC2 machine configured to assume [an IAM role associated with the instance](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html). Default: false" }, "id": { "type": "string", "description": "The ID of credentials to use. Default: " }, "profile": { "type": "string", "description": "A profile from `~/.aws/credentials` to use. Default: " }, "role": { "type": "string", "description": "A role ARN to assume. Default: " }, "role_external_id": { "type": "string", "description": "An external ID to provide when assuming a role. Default: " }, "secret": { "type": "string", "description": "The secret for the credentials being used. Default: " }, "token": { "type": "string", "description": "The token for the credentials being used, required when using short term credentials. Default: " } } }, "endpoint": { "type": "string", "description": "Allows you to specify a custom endpoint for the AWS API. Default: " }, "hash_key": { "type": "string", "description": "A optional hash key for partitioning messages." }, "max_in_flight": { "type": "integer", "description": "The maximum number of parallel message batches to have in flight at any given time. Default: 64" }, "max_retries": { "type": "integer", "description": "The maximum number of retries before giving up on the request. If set to zero there is no discrete limit. Default: 0" }, "partition_key": { "type": "string", "description": "A required key for partitioning messages." }, "region": { "type": "string", "description": "The AWS region to target. Default: " }, "stream": { "type": "string", "description": "The stream to publish messages to. Streams can either be specified by their name or full ARN.", "examples": [ "foo", "arn:aws:kinesis:*:111122223333:stream/my-stream" ] } } }, "aws_kinesis_firehose": { "type": "object", "properties": { "backoff": { "type": "object", "description": "Control time intervals between retry attempts.", "properties": { "initial_interval": { "type": "string", "description": "The initial period to wait between retry attempts. Default: 1s" }, "max_elapsed_time": { "type": "string", "description": "The maximum period to wait before retry attempts are abandoned. If zero then no limit is used. Default: 30s" }, "max_interval": { "type": "string", "description": "The maximum period to wait between retry attempts. Default: 5s" } } }, "batching": { "type": "object", "description": "\nAllows you to configure a [batching policy](/docs/configuration/batching).", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] }, "credentials": { "type": "object", "description": "Optional manual configuration of AWS credentials to use. More information can be found [in this document](/docs/guides/cloud/aws).", "properties": { "expiry_window": { "type": "string", "description": "Allow the credentials to trigger refreshing prior to the credentials actually expiring. This is beneficial so race conditions with expiring credentials do not cause requests to fail. For example '10s' would refresh credentials ten seconds before expiration. Setting to a duration of `0` disables the expiry window. Default: " }, "from_ec2_role": { "type": "boolean", "description": "Use the credentials of a host EC2 machine configured to assume [an IAM role associated with the instance](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html). Default: false" }, "id": { "type": "string", "description": "The ID of credentials to use. Default: " }, "profile": { "type": "string", "description": "A profile from `~/.aws/credentials` to use. Default: " }, "role": { "type": "string", "description": "A role ARN to assume. Default: " }, "role_external_id": { "type": "string", "description": "An external ID to provide when assuming a role. Default: " }, "secret": { "type": "string", "description": "The secret for the credentials being used. Default: " }, "token": { "type": "string", "description": "The token for the credentials being used, required when using short term credentials. Default: " } } }, "endpoint": { "type": "string", "description": "Allows you to specify a custom endpoint for the AWS API. Default: " }, "max_in_flight": { "type": "integer", "description": "The maximum number of messages to have in flight at a given time. Increase this to improve throughput. Default: 64" }, "max_retries": { "type": "integer", "description": "The maximum number of retries before giving up on the request. If set to zero there is no discrete limit. Default: 0" }, "region": { "type": "string", "description": "The AWS region to target. Default: " }, "stream": { "type": "string", "description": "The stream to publish messages to." } } }, "aws_s3": { "type": "object", "properties": { "batching": { "type": "object", "description": "\nAllows you to configure a [batching policy](/docs/configuration/batching).", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] }, "bucket": { "type": "string", "description": "The bucket to upload messages to." }, "cache_control": { "type": "string", "description": "The cache control to set for each object. Default: " }, "content_disposition": { "type": "string", "description": "The content disposition to set for each object. Default: " }, "content_encoding": { "type": "string", "description": "An optional content encoding to set for each object. Default: " }, "content_language": { "type": "string", "description": "The content language to set for each object. Default: " }, "content_type": { "type": "string", "description": "The content type to set for each object. Default: application/octet-stream" }, "credentials": { "type": "object", "description": "Optional manual configuration of AWS credentials to use. More information can be found [in this document](/docs/guides/cloud/aws).", "properties": { "expiry_window": { "type": "string", "description": "Allow the credentials to trigger refreshing prior to the credentials actually expiring. This is beneficial so race conditions with expiring credentials do not cause requests to fail. For example '10s' would refresh credentials ten seconds before expiration. Setting to a duration of `0` disables the expiry window. Default: " }, "from_ec2_role": { "type": "boolean", "description": "Use the credentials of a host EC2 machine configured to assume [an IAM role associated with the instance](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html). Default: false" }, "id": { "type": "string", "description": "The ID of credentials to use. Default: " }, "profile": { "type": "string", "description": "A profile from `~/.aws/credentials` to use. Default: " }, "role": { "type": "string", "description": "A role ARN to assume. Default: " }, "role_external_id": { "type": "string", "description": "An external ID to provide when assuming a role. Default: " }, "secret": { "type": "string", "description": "The secret for the credentials being used. Default: " }, "token": { "type": "string", "description": "The token for the credentials being used, required when using short term credentials. Default: " } } }, "endpoint": { "type": "string", "description": "Allows you to specify a custom endpoint for the AWS API. Default: " }, "force_path_style_urls": { "type": "boolean", "description": "Forces the client API to use path style URLs, which helps when connecting to custom endpoints. Default: false" }, "kms_key_id": { "type": "string", "description": "An optional server side encryption key. Default: " }, "max_in_flight": { "type": "integer", "description": "The maximum number of messages to have in flight at a given time. Increase this to improve throughput. Default: 64" }, "metadata": { "type": "object", "description": "Specify criteria for which metadata values are attached to objects as headers.", "properties": { "exclude_prefixes": { "type": "array", "description": "Provide a list of explicit metadata key prefixes to be excluded when adding metadata to sent messages. Default: []", "items": { "type": "string" } } } }, "path": { "type": "string", "description": "The path of each message to upload. Default: ${!count(\"files\")}-${!timestamp_unix_nano()}.txt", "examples": [ "${!count(\"files\")}-${!timestamp_unix_nano()}.txt", "${!metadata(\"kafka_key\")}.json", "${!json(\"doc.namespace\")}/${!json(\"doc.id\")}.json" ] }, "region": { "type": "string", "description": "The AWS region to target. Default: " }, "server_side_encryption": { "type": "string", "description": "An optional server side encryption algorithm. Default: " }, "storage_class": { "type": "string", "description": "The storage class to set for each object. Default: STANDARD" }, "tags": { "type": "object", "description": "Key/value pairs to store with the object as tags. Default: map[]", "examples": [ { "Key1": "Value1", "Timestamp": "${!metadata(\"Timestamp\")}" } ] }, "timeout": { "type": "string", "description": "The maximum period to wait on an upload before abandoning it and reattempting. Default: 5s" }, "website_redirect_location": { "type": "string", "description": "The website redirect location to set for each object. Default: " } } }, "aws_sns": { "type": "object", "properties": { "credentials": { "type": "object", "description": "Optional manual configuration of AWS credentials to use. More information can be found [in this document](/docs/guides/cloud/aws).", "properties": { "expiry_window": { "type": "string", "description": "Allow the credentials to trigger refreshing prior to the credentials actually expiring. This is beneficial so race conditions with expiring credentials do not cause requests to fail. For example '10s' would refresh credentials ten seconds before expiration. Setting to a duration of `0` disables the expiry window. Default: " }, "from_ec2_role": { "type": "boolean", "description": "Use the credentials of a host EC2 machine configured to assume [an IAM role associated with the instance](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html). Default: false" }, "id": { "type": "string", "description": "The ID of credentials to use. Default: " }, "profile": { "type": "string", "description": "A profile from `~/.aws/credentials` to use. Default: " }, "role": { "type": "string", "description": "A role ARN to assume. Default: " }, "role_external_id": { "type": "string", "description": "An external ID to provide when assuming a role. Default: " }, "secret": { "type": "string", "description": "The secret for the credentials being used. Default: " }, "token": { "type": "string", "description": "The token for the credentials being used, required when using short term credentials. Default: " } } }, "endpoint": { "type": "string", "description": "Allows you to specify a custom endpoint for the AWS API. Default: " }, "max_in_flight": { "type": "integer", "description": "The maximum number of messages to have in flight at a given time. Increase this to improve throughput. Default: 64" }, "message_deduplication_id": { "type": "string", "description": "An optional deduplication ID to set for messages." }, "message_group_id": { "type": "string", "description": "An optional group ID to set for messages." }, "metadata": { "type": "object", "description": "Specify criteria for which metadata values are sent as headers.", "properties": { "exclude_prefixes": { "type": "array", "description": "Provide a list of explicit metadata key prefixes to be excluded when adding metadata to sent messages. Default: []", "items": { "type": "string" } } } }, "region": { "type": "string", "description": "The AWS region to target. Default: " }, "timeout": { "type": "string", "description": "The maximum period to wait on an upload before abandoning it and reattempting. Default: 5s" }, "topic_arn": { "type": "string", "description": "The topic to publish to." } } }, "aws_sqs": { "type": "object", "properties": { "backoff": { "type": "object", "description": "Control time intervals between retry attempts.", "properties": { "initial_interval": { "type": "string", "description": "The initial period to wait between retry attempts. Default: 1s" }, "max_elapsed_time": { "type": "string", "description": "The maximum period to wait before retry attempts are abandoned. If zero then no limit is used. Default: 30s" }, "max_interval": { "type": "string", "description": "The maximum period to wait between retry attempts. Default: 5s" } } }, "batching": { "type": "object", "description": "\nAllows you to configure a [batching policy](/docs/configuration/batching).", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] }, "credentials": { "type": "object", "description": "Optional manual configuration of AWS credentials to use. More information can be found [in this document](/docs/guides/cloud/aws).", "properties": { "expiry_window": { "type": "string", "description": "Allow the credentials to trigger refreshing prior to the credentials actually expiring. This is beneficial so race conditions with expiring credentials do not cause requests to fail. For example '10s' would refresh credentials ten seconds before expiration. Setting to a duration of `0` disables the expiry window. Default: " }, "from_ec2_role": { "type": "boolean", "description": "Use the credentials of a host EC2 machine configured to assume [an IAM role associated with the instance](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html). Default: false" }, "id": { "type": "string", "description": "The ID of credentials to use. Default: " }, "profile": { "type": "string", "description": "A profile from `~/.aws/credentials` to use. Default: " }, "role": { "type": "string", "description": "A role ARN to assume. Default: " }, "role_external_id": { "type": "string", "description": "An external ID to provide when assuming a role. Default: " }, "secret": { "type": "string", "description": "The secret for the credentials being used. Default: " }, "token": { "type": "string", "description": "The token for the credentials being used, required when using short term credentials. Default: " } } }, "delay_seconds": { "type": "string", "description": "An optional delay time in seconds for message. Value between 0 and 900" }, "endpoint": { "type": "string", "description": "Allows you to specify a custom endpoint for the AWS API. Default: " }, "max_in_flight": { "type": "integer", "description": "The maximum number of parallel message batches to have in flight at any given time. Default: 64" }, "max_retries": { "type": "integer", "description": "The maximum number of retries before giving up on the request. If set to zero there is no discrete limit. Default: 0" }, "message_deduplication_id": { "type": "string", "description": "An optional deduplication ID to set for messages." }, "message_group_id": { "type": "string", "description": "An optional group ID to set for messages." }, "metadata": { "type": "object", "description": "Specify criteria for which metadata values are sent as headers.", "properties": { "exclude_prefixes": { "type": "array", "description": "Provide a list of explicit metadata key prefixes to be excluded when adding metadata to sent messages. Default: []", "items": { "type": "string" } } } }, "region": { "type": "string", "description": "The AWS region to target. Default: " }, "url": { "type": "string", "description": "The URL of the target SQS queue." } } }, "azure_blob_storage": { "type": "object", "properties": { "blob_type": { "type": "string", "description": "Block and Append blobs are comprised of blocks, and each blob can support up to 50,000 blocks. The default value is `+\"`BLOCK`\"+`.` Default: BLOCK" }, "container": { "type": "string", "description": "The container for uploading the messages to.", "examples": [ "messages-${!timestamp(\"2006\")}" ] }, "max_in_flight": { "type": "integer", "description": "The maximum number of messages to have in flight at a given time. Increase this to improve throughput. Default: 64" }, "path": { "type": "string", "description": "The path of each message to upload. Default: ${!count(\"files\")}-${!timestamp_unix_nano()}.txt", "examples": [ "${!count(\"files\")}-${!timestamp_unix_nano()}.json", "${!metadata(\"kafka_key\")}.json", "${!json(\"doc.namespace\")}/${!json(\"doc.id\")}.json" ] }, "public_access_level": { "type": "string", "description": "The container's public access level. The default value is `PRIVATE`. Default: PRIVATE" }, "storage_access_key": { "type": "string", "description": "The storage account access key. This field is ignored if `storage_connection_string` is set. Default: " }, "storage_account": { "type": "string", "description": "The storage account to access. This field is ignored if `storage_connection_string` is set. Default: " }, "storage_connection_string": { "type": "string", "description": "A storage account connection string. This field is required if `storage_account` and `storage_access_key` / `storage_sas_token` are not set. Default: " }, "storage_sas_token": { "type": "string", "description": "The storage account SAS token. This field is ignored if `storage_connection_string` or `storage_access_key` are set. Default: " } } }, "azure_cosmosdb": { "type": "object", "properties": { "account_key": { "type": "string", "description": "Account key.", "examples": [ "C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==" ] }, "auto_id": { "type": "boolean", "description": "Automatically set the item `id` field to a random UUID v4. If the `id` field is already set, then it will not be overwritten. Setting this to `false` can improve performance, since the messages will not have to be parsed. Default: true" }, "batching": { "type": "object", "description": "\nAllows you to configure a [batching policy](/docs/configuration/batching).", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] }, "connection_string": { "type": "string", "description": "Connection string.", "examples": [ "AccountEndpoint=https://localhost:8081/;AccountKey=C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==;" ] }, "container": { "type": "string", "description": "Container.", "examples": [ "testcontainer" ] }, "database": { "type": "string", "description": "Database.", "examples": [ "testdb" ] }, "endpoint": { "type": "string", "description": "CosmosDB endpoint.", "examples": [ "https://localhost:8081" ] }, "item_id": { "type": "string", "description": "ID of item to replace or delete. Only used by the Replace and Delete operations", "examples": [ "${! json(\"id\") }" ] }, "max_in_flight": { "type": "integer", "description": "The maximum number of messages to have in flight at a given time. Increase this to improve throughput. Default: 64" }, "operation": { "type": "string", "description": "Operation. Default: Create" }, "partition_keys_map": { "type": "string", "description": "A [Bloblang mapping](/docs/guides/bloblang/about) which should evaluate to a single partition key value or an array of partition key values of type string, integer or boolean. Currently, hierarchical partition keys are not supported so only one value may be provided.", "examples": [ "root = \"blobfish\"", "root = 41", "root = true", "root = null", "root = json(\"blobfish\").depth" ] }, "patch_condition": { "type": "string", "description": "Patch operation condition.", "examples": [ "from c where not is_defined(c.blobfish)" ] }, "patch_operations": { "type": "array", "description": "Patch operations to be performed when `operation: Patch` .", "items": { "type": "object" } } } }, "azure_queue_storage": { "type": "object", "properties": { "batching": { "type": "object", "description": "\nAllows you to configure a [batching policy](/docs/configuration/batching).", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] }, "max_in_flight": { "type": "integer", "description": "The maximum number of parallel message batches to have in flight at any given time. Default: 64" }, "queue_name": { "type": "string", "description": "The name of the target Queue Storage queue." }, "storage_access_key": { "type": "string", "description": "The storage account access key. This field is ignored if `storage_connection_string` is set. Default: " }, "storage_account": { "type": "string", "description": "The storage account to access. This field is ignored if `storage_connection_string` is set. Default: " }, "storage_connection_string": { "type": "string", "description": "A storage account connection string. This field is required if `storage_account` and `storage_access_key` / `storage_sas_token` are not set. Default: " }, "storage_sas_token": { "type": "string", "description": "The storage account SAS token. This field is ignored if `storage_connection_string` or `storage_access_key` are set. Default: " }, "ttl": { "type": "string", "description": "The TTL of each individual message as a duration string. Defaults to 0, meaning no retention period is set Default: ", "examples": [ "60s", "5m", "36h" ] } } }, "azure_table_storage": { "type": "object", "properties": { "batching": { "type": "object", "description": "\nAllows you to configure a [batching policy](/docs/configuration/batching).", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] }, "insert_type": { "type": "string", "description": "Type of insert operation. Valid options are `INSERT`, `INSERT_MERGE` and `INSERT_REPLACE` Default: ", "examples": [ "${! json(\"operation\") }", "${! metadata(\"operation\") }", "INSERT" ] }, "max_in_flight": { "type": "integer", "description": "The maximum number of parallel message batches to have in flight at any given time. Default: 64" }, "partition_key": { "type": "string", "description": "The partition key. Default: ", "examples": [ "${! json(\"date\") }" ] }, "properties": { "type": "object", "description": "A map of properties to store into the table. Default: map[]" }, "row_key": { "type": "string", "description": "The row key. Default: ", "examples": [ "${! json(\"device\")}-${!uuid_v4() }" ] }, "storage_access_key": { "type": "string", "description": "The storage account access key. This field is ignored if `storage_connection_string` is set. Default: " }, "storage_account": { "type": "string", "description": "The storage account to access. This field is ignored if `storage_connection_string` is set. Default: " }, "storage_connection_string": { "type": "string", "description": "A storage account connection string. This field is required if `storage_account` and `storage_access_key` / `storage_sas_token` are not set. Default: " }, "storage_sas_token": { "type": "string", "description": "The storage account SAS token. This field is ignored if `storage_connection_string` or `storage_access_key` are set. Default: " }, "table_name": { "type": "string", "description": "The table to store messages into.", "examples": [ "${! metadata(\"kafka_topic\") }", "${! json(\"table\") }" ] }, "timeout": { "type": "string", "description": "The maximum period to wait on an upload before abandoning it and reattempting. Default: 5s" }, "transaction_type": { "type": "string", "description": "Type of transaction operation. Default: INSERT", "examples": [ "${! json(\"operation\") }", "${! metadata(\"operation\") }", "INSERT" ] } } }, "beanstalkd": { "type": "object", "properties": { "address": { "type": "string", "description": "An address to connect to.", "examples": [ "127.0.0.1:11300" ] }, "max_in_flight": { "type": "integer", "description": "The maximum number of messages to have in flight at a given time. Increase to improve throughput. Default: 64" } } }, "broker": { "type": "object", "properties": { "batching": { "type": "object", "description": "\nAllows you to configure a [batching policy](/docs/configuration/batching).", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] }, "copies": { "type": "integer", "description": "The number of copies of each configured output to spawn. Default: 1" }, "outputs": { "type": "array", "description": "A list of child outputs to broker. Each item can be either a complete inline output configuration or a reference to an output resource.", "items": { "type": "object" } }, "pattern": { "type": "string", "description": "The brokering pattern to use. Default: fan_out" } } }, "cache": { "type": "object", "properties": { "key": { "type": "string", "description": "The key to store messages by, function interpolation should be used in order to derive a unique key for each message. Default: ${!count(\"items\")}-${!timestamp_unix_nano()}", "examples": [ "${!count(\"items\")}-${!timestamp_unix_nano()}", "${!json(\"doc.id\")}", "${!metadata(\"kafka_key\")}" ] }, "max_in_flight": { "type": "integer", "description": "The maximum number of messages to have in flight at a given time. Increase this to improve throughput. Default: 64" }, "target": { "type": "string", "description": "The target cache to store messages in." }, "ttl": { "type": "string", "description": "The TTL of each individual item as a duration string. After this period an item will be eligible for removal during the next compaction. Not all caches support per-key TTLs, and those that do not will fall back to their generally configured TTL setting.", "examples": [ "60s", "5m", "36h" ] } } }, "cassandra": { "type": "object", "properties": { "addresses": { "type": "array", "description": "A list of Cassandra nodes to connect to. Multiple comma separated addresses can be specified on a single line.", "items": { "type": "string" }, "examples": [ [ "localhost:9042" ], [ "foo:9042", "bar:9042" ], [ "foo:9042,bar:9042" ] ] }, "args_mapping": { "type": "string", "description": "A [Bloblang mapping](/docs/guides/bloblang/about) that can be used to provide arguments to Cassandra queries. The result of the query must be an array containing a matching number of elements to the query arguments." }, "backoff": { "type": "object", "description": "Control time intervals between retry attempts.", "properties": { "initial_interval": { "type": "string", "description": "The initial period to wait between retry attempts. Default: 1s" }, "max_interval": { "type": "string", "description": "The maximum period to wait between retry attempts. Default: 5s" } } }, "batching": { "type": "object", "description": "\nAllows you to configure a [batching policy](/docs/configuration/batching).", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] }, "consistency": { "type": "string", "description": "The consistency level to use. Default: QUORUM" }, "disable_initial_host_lookup": { "type": "boolean", "description": "If enabled the driver will not attempt to get host info from the system.peers table. This can speed up queries but will mean that data_centre, rack and token information will not be available. Default: false" }, "logged_batch": { "type": "boolean", "description": "If enabled the driver will perform a logged batch. Disabling this prompts unlogged batches to be used instead, which are less efficient but necessary for alternative storages that do not support logged batches. Default: true" }, "max_in_flight": { "type": "integer", "description": "The maximum number of messages to have in flight at a given time. Increase this to improve throughput. Default: 64" }, "max_retries": { "type": "integer", "description": "The maximum number of retries before giving up on a request. Default: 3" }, "password_authenticator": { "type": "object", "description": "Optional configuration of Cassandra authentication parameters.", "properties": { "enabled": { "type": "boolean", "description": "Whether to use password authentication Default: false" }, "password": { "type": "string", "description": "The password to authenticate with. Default: " }, "username": { "type": "string", "description": "The username to authenticate as. Default: " } } }, "query": { "type": "string", "description": "A query to execute for each message." }, "timeout": { "type": "string", "description": "The client connection timeout. Default: 600ms" }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } } } }, "couchbase": { "type": "object", "properties": { "batching": { "type": "object", "description": "\nAllows you to configure a [batching policy](/docs/configuration/batching).", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] }, "bucket": { "type": "string", "description": "Couchbase bucket." }, "cas_enabled": { "type": "boolean", "description": "Enable CAS validation. Default: true" }, "collection": { "type": "string", "description": "Bucket collection. Default: _default" }, "content": { "type": "string", "description": "Document content." }, "id": { "type": "string", "description": "Document id.", "examples": [ "${! json(\"id\") }" ] }, "max_in_flight": { "type": "integer", "description": "The maximum number of messages to have in flight at a given time. Increase this to improve throughput. Default: 64" }, "operation": { "type": "string", "description": "Couchbase operation to perform. Default: upsert" }, "password": { "type": "string", "description": "Password to connect to the cluster." }, "timeout": { "type": "string", "description": "Operation timeout. Default: 15s" }, "transcoder": { "type": "string", "description": "Couchbase transcoder to use. Default: legacy" }, "ttl": { "type": "string", "description": "An optional TTL to set for items." }, "url": { "type": "string", "description": "Couchbase connection string.", "examples": [ "couchbase://localhost:11210" ] }, "username": { "type": "string", "description": "Username to connect to the cluster." } } }, "cypher": { "type": "object", "properties": { "basic_auth": { "type": "object", "description": "Basic Authentication fields", "properties": { "password": { "type": "string", "description": "The password for basic auth. Default: " }, "realm": { "type": "string", "description": "The realm for basic auth. Default: " }, "user": { "type": "string", "description": "The username for basic auth. Default: " } } }, "batching": { "type": "object", "description": "\nAllows you to configure a [batching policy](/docs/configuration/batching).", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] }, "database": { "type": "string", "description": "The name of the database to connect to.", "examples": [ "neo4j" ] }, "max_in_flight": { "type": "integer", "description": "The maximum number of messages to have in flight at a given time. Increase this to improve throughput. Default: 64" }, "no_auth": { "type": "boolean", "description": "Set to true to connect without authentication. Default: false" }, "query": { "type": "string", "description": "The cypher query to execute.", "examples": [ "CREATE (p:Person {name: $name}) RETURN p" ] }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "uri": { "type": "string", "description": "The URL of the database engine.", "examples": [ "bolt://localhost:7687" ] }, "values": { "type": "object", "description": "A map of strings -\u003e bloblang interpolations that form the values of the references in the query i.e. $name. Default: map[]", "examples": [ { "name": "${! json(\"name\") }" } ] } } }, "discord": { "type": "object", "properties": { "bot_token": { "type": "string", "description": "A bot token used for authentication." }, "channel_id": { "type": "string", "description": "A discord channel ID to write messages to." }, "rate_limit": { "type": "string", "description": " Default: An optional rate limit resource to restrict API requests with." } } }, "drop": { "type": "object", "description": " Default: map[]" }, "drop_on": { "type": "object", "properties": { "back_pressure": { "type": "string", "description": "An optional duration string that determines the maximum length of time to wait for a given message to be accepted by the child output before the message should be dropped instead. The most common reason for an output to block is when waiting for a lost connection to be re-established. Once a message has been dropped due to back pressure all subsequent messages are dropped immediately until the output is ready to process them again. Note that if `error` is set to `false` and this field is specified then messages dropped due to back pressure will return an error response (are nacked or reattempted).", "examples": [ "30s", "1m" ] }, "error": { "type": "boolean", "description": "Whether messages should be dropped when the child output returns an error of any type. For example, this could be when an `http_client` output gets a 4XX response code. In order to instead drop only on specific error patterns use the `error_matches` field instead. Default: false" }, "error_patterns": { "type": "array", "description": "A list of regular expressions (re2) where if the child output returns an error that matches any part of any of these patterns the message will be dropped.", "items": { "type": "string" }, "examples": [ [ "and that was really bad$" ], [ "roughly [0-9]+ issues occurred" ] ] }, "output": { "type": "object", "description": "A child output to wrap with this drop mechanism." } } }, "dynamic": { "type": "object", "properties": { "outputs": { "type": "object", "description": "A map of outputs to statically create. Default: map[]" }, "prefix": { "type": "string", "description": "A path prefix for HTTP endpoints that are registered. Default: " } } }, "elasticsearch": { "type": "object", "properties": { "action": { "type": "string", "description": "The action to take on the document. This field must resolve to one of the following action types: `create`, `index`, `update`, `upsert` or `delete`. Default: index" }, "aws": { "type": "object", "description": "Enables and customises connectivity to Amazon Elastic Service.", "properties": { "credentials": { "type": "object", "description": "Optional manual configuration of AWS credentials to use. More information can be found [in this document](/docs/guides/cloud/aws).", "properties": { "expiry_window": { "type": "string", "description": "Allow the credentials to trigger refreshing prior to the credentials actually expiring. This is beneficial so race conditions with expiring credentials do not cause requests to fail. For example '10s' would refresh credentials ten seconds before expiration. Setting to a duration of `0` disables the expiry window. Default: " }, "from_ec2_role": { "type": "boolean", "description": "Use the credentials of a host EC2 machine configured to assume [an IAM role associated with the instance](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html). Default: false" }, "id": { "type": "string", "description": "The ID of credentials to use. Default: " }, "profile": { "type": "string", "description": "A profile from `~/.aws/credentials` to use. Default: " }, "role": { "type": "string", "description": "A role ARN to assume. Default: " }, "role_external_id": { "type": "string", "description": "An external ID to provide when assuming a role. Default: " }, "secret": { "type": "string", "description": "The secret for the credentials being used. Default: " }, "token": { "type": "string", "description": "The token for the credentials being used, required when using short term credentials. Default: " } } }, "enabled": { "type": "boolean", "description": "Whether to connect to Amazon Elastic Service. Default: false" }, "endpoint": { "type": "string", "description": "Allows you to specify a custom endpoint for the AWS API. Default: " }, "region": { "type": "string", "description": "The AWS region to target. Default: " } } }, "backoff": { "type": "object", "description": "Control time intervals between retry attempts.", "properties": { "initial_interval": { "type": "string", "description": "The initial period to wait between retry attempts. Default: 1s" }, "max_elapsed_time": { "type": "string", "description": "The maximum period to wait before retry attempts are abandoned. If zero then no limit is used. Default: 30s" }, "max_interval": { "type": "string", "description": "The maximum period to wait between retry attempts. Default: 5s" } } }, "basic_auth": { "type": "object", "description": "Allows you to specify basic authentication.", "properties": { "enabled": { "type": "boolean", "description": "Whether to use basic authentication in requests. Default: false" }, "password": { "type": "string", "description": "A password to authenticate with. Default: " }, "username": { "type": "string", "description": "A username to authenticate as. Default: " } } }, "batching": { "type": "object", "description": "\nAllows you to configure a [batching policy](/docs/configuration/batching).", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] }, "gzip_compression": { "type": "boolean", "description": "Enable gzip compression on the request side. Default: false" }, "healthcheck": { "type": "boolean", "description": "Whether to enable healthchecks. Default: true" }, "id": { "type": "string", "description": "The ID for indexed messages. Interpolation should be used in order to create a unique ID for each message. Default: ${!count(\"elastic_ids\")}-${!timestamp_unix()}" }, "index": { "type": "string", "description": "The index to place messages." }, "max_in_flight": { "type": "integer", "description": "The maximum number of messages to have in flight at a given time. Increase this to improve throughput. Default: 64" }, "max_retries": { "type": "integer", "description": "The maximum number of retries before giving up on the request. If set to zero there is no discrete limit. Default: 0" }, "pipeline": { "type": "string", "description": "An optional pipeline id to preprocess incoming documents. Default: " }, "routing": { "type": "string", "description": "The routing key to use for the document. Default: " }, "sniff": { "type": "boolean", "description": "Prompts Bento to sniff for brokers to connect to when establishing a connection. Default: true" }, "timeout": { "type": "string", "description": "The maximum time to wait before abandoning a request (and trying again). Default: 5s" }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "type": { "type": "string", "description": "The document mapping type. This field is required for versions of elasticsearch earlier than 6.0.0, but are invalid for versions 7.0.0 or later. Default: " }, "urls": { "type": "array", "description": "A list of URLs to connect to. If an item of the list contains commas it will be expanded into multiple URLs.", "items": { "type": "string" }, "examples": [ [ "http://localhost:9200" ] ] } } }, "elasticsearch_v2": { "type": "object", "properties": { "action": { "type": "string", "description": "The action to take on the document. This field must resolve to one of the following action types: `create`, `index`, `update`, `upsert` or `delete`. Default: index" }, "backoff": { "type": "object", "description": "Control time intervals between retry attempts.", "properties": { "initial_interval": { "type": "string", "description": "The initial period to wait between retry attempts. Default: 1s" }, "max_elapsed_time": { "type": "string", "description": "The maximum period to wait before retry attempts are abandoned. If zero then no limit is used. Default: 30s" }, "max_interval": { "type": "string", "description": "The maximum period to wait between retry attempts. Default: 5s" } } }, "basic_auth": { "type": "object", "description": "Allows you to specify basic authentication.", "properties": { "enabled": { "type": "boolean", "description": "Whether to use basic authentication in requests. Default: false" }, "password": { "type": "string", "description": "A password to authenticate with. Default: " }, "username": { "type": "string", "description": "A username to authenticate as. Default: " } } }, "batching": { "type": "object", "description": "\nAllows you to configure a [batching policy](/docs/configuration/batching).", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] }, "compress_request_body": { "type": "boolean", "description": "Enable gzip compression on the request side. Default: false" }, "discover_nodes_interval": { "type": "string", "description": "Discover nodes periodically. Default: 0s" }, "discover_nodes_on_start": { "type": "boolean", "description": "Discover nodes when initializing the client. Default: false" }, "id": { "type": "string", "description": "The ID for indexed messages. Interpolation should be used in order to create a unique ID for each message. Default: ${!count(\"elastic_ids\")}-${!timestamp_unix()}" }, "index": { "type": "string", "description": "The index to place messages." }, "max_in_flight": { "type": "integer", "description": "The maximum number of messages to have in flight at a given time. Increase this to improve throughput. Default: 64" }, "max_retries": { "type": "integer", "description": "The maximum number of retries before giving up on the request. If set to zero there is no discrete limit. Default: 0" }, "pipeline": { "type": "string", "description": "An optional pipeline id to preprocess incoming documents. Default: " }, "retry_on_status": { "type": "integer", "description": "HTTP Status codes that should be retried. Default: [502 503 504]", "items": { "type": "integer" } }, "routing": { "type": "string", "description": "The routing key to use for the document. Default: " }, "timeout": { "type": "string", "description": "The maximum time to wait before abandoning a request (and trying again). Default: 5s" }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "urls": { "type": "array", "description": "A list of URLs to connect to. If an item of the list contains commas it will be expanded into multiple URLs.", "items": { "type": "string" }, "examples": [ [ "http://localhost:9200" ] ] } } }, "fallback": { "type": "array", "description": " Default: []", "items": { "type": "object" } }, "file": { "type": "object", "properties": { "codec": { "type": "string", "description": "The way in which the bytes of messages should be written out into the output data stream. It's possible to write lines using a custom delimiter with the `delim:x` codec, where x is the character sequence custom delimiter. Default: lines", "examples": [ "lines", "delim:\t", "delim:foobar" ] }, "path": { "type": "string", "description": "The file to write to, if the file does not yet exist it will be created.", "examples": [ "/tmp/data.txt", "/tmp/${! timestamp_unix() }.txt", "/tmp/${! json(\"document.id\") }.json" ] } } }, "gcp_bigquery": { "type": "object", "properties": { "auto_detect": { "type": "boolean", "description": "Indicates if we should automatically infer the options and schema for CSV and JSON sources. If the table doesn't exist and this field is set to `false` the output may not be able to insert data and will throw insertion error. Be careful using this field since it delegates to the GCP BigQuery service the schema detection and values like `\"no\"` may be treated as booleans for the CSV format. Default: false" }, "batching": { "type": "object", "description": "\nAllows you to configure a [batching policy](/docs/configuration/batching).", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] }, "create_disposition": { "type": "string", "description": "Specifies the circumstances under which destination table will be created. If CREATE_IF_NEEDED is used the GCP BigQuery will create the table if it does not already exist and tables are created atomically on successful completion of a job. The CREATE_NEVER option ensures the table must already exist and will not be automatically created. Default: CREATE_IF_NEEDED" }, "csv": { "type": "object", "description": "Specify how CSV data should be interpretted.", "properties": { "allow_jagged_rows": { "type": "boolean", "description": "Causes missing trailing optional columns to be tolerated when reading CSV data. Missing values are treated as nulls. Default: false" }, "allow_quoted_newlines": { "type": "boolean", "description": "Sets whether quoted data sections containing newlines are allowed when reading CSV data. Default: false" }, "encoding": { "type": "string", "description": "Encoding is the character encoding of data to be read. Default: UTF-8" }, "field_delimiter": { "type": "string", "description": "The separator for fields in a CSV file, used when reading or exporting data. Default: ," }, "header": { "type": "array", "description": "A list of values to use as header for each batch of messages. If not specified the first line of each message will be used as header. Default: []", "items": { "type": "string" } }, "skip_leading_rows": { "type": "integer", "description": "The number of rows at the top of a CSV file that BigQuery will skip when reading data. The default value is 1 since Bento will add the specified header in the first line of each batch sent to BigQuery. Default: 1" } } }, "dataset": { "type": "string", "description": "The BigQuery Dataset ID." }, "format": { "type": "string", "description": "The format of each incoming message. Default: NEWLINE_DELIMITED_JSON" }, "ignore_unknown_values": { "type": "boolean", "description": "Causes values not matching the schema to be tolerated. Unknown values are ignored. For CSV this ignores extra values at the end of a line. For JSON this ignores named values that do not match any column name. If this field is set to false (the default value), records containing unknown values are treated as bad records. The max_bad_records field can be used to customize how bad records are handled. Default: false" }, "job_labels": { "type": "object", "description": "A list of labels to add to the load job. Default: map[]" }, "max_bad_records": { "type": "integer", "description": "The maximum number of bad records that will be ignored when reading data. Default: 0" }, "max_in_flight": { "type": "integer", "description": "The maximum number of message batches to have in flight at a given time. Increase this to improve throughput. Default: 64" }, "project": { "type": "string", "description": "The project ID of the dataset to insert data to. If not set, it will be inferred from the credentials or read from the GOOGLE_CLOUD_PROJECT environment variable. Default: " }, "table": { "type": "string", "description": ":::caution Interpolation of Message Batches\nIt is assumed that the first message in the batch will resolve the bloblang query and that string will be used for all messages in the batch.\n:::\nThe table to insert messages to." }, "write_disposition": { "type": "string", "description": "Specifies how existing data in a destination table is treated. Default: WRITE_APPEND" } } }, "gcp_bigquery_write_api": { "type": "object", "properties": { "batching": { "type": "object", "description": "\nAllows you to configure a [batching policy](/docs/configuration/batching).", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] }, "dataset": { "type": "string", "description": "The BigQuery Dataset ID." }, "endpoint": { "type": "object", "description": "Used to overwrite the default gRPC and HTTP BigQuery endpoints.", "properties": { "grpc": { "type": "string", "description": "The endpoint used to create the BigQuery Storage API client. Default: " }, "http": { "type": "string", "description": "The endpoint used to create the BigQuery client. Default: " } } }, "max_in_flight": { "type": "integer", "description": "The maximum number of message batches to have in flight at a given time. Increase this to improve throughput. Default: 64" }, "message_format": { "type": "string", "description": "Format of incoming messages Default: json" }, "project": { "type": "string", "description": "The project ID of the dataset to insert data to. If not set, it will be inferred from the credentials or read from the GOOGLE_CLOUD_PROJECT environment variable. Default: " }, "stream_type": { "type": "string", "description": ":::caution Storage API Stream Types\nOnly `DEFAULT` stream types are currently enabled. Future versions will see support extended to `COMMITTED`, `BUFFERED`, and `PENDING`.\n:::\nsets the type of stream this write client is managing. Default: DEFAULT" }, "table": { "type": "string", "description": ":::caution Interpolation of Message Batches\nIt is assumed that the first message in the batch will resolve the bloblang query and that string will be used for all messages in the batch.\n:::\nThe table to insert messages to." } } }, "gcp_cloud_storage": { "type": "object", "properties": { "batching": { "type": "object", "description": "\nAllows you to configure a [batching policy](/docs/configuration/batching).", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] }, "bucket": { "type": "string", "description": "The bucket to upload messages to." }, "chunk_size": { "type": "integer", "description": "An optional chunk size which controls the maximum number of bytes of the object that the Writer will attempt to send to the server in a single request. If ChunkSize is set to zero, chunking will be disabled. Default: 16777216" }, "collision_mode": { "type": "string", "description": "Determines how file path collisions should be dealt with. Default: overwrite" }, "content_encoding": { "type": "string", "description": "An optional content encoding to set for each object. Default: " }, "content_type": { "type": "string", "description": "The content type to set for each object. Default: application/octet-stream" }, "max_in_flight": { "type": "integer", "description": "The maximum number of message batches to have in flight at a given time. Increase this to improve throughput. Default: 64" }, "path": { "type": "string", "description": "The path of each message to upload. Default: ${!count(\"files\")}-${!timestamp_unix_nano()}.txt", "examples": [ "${!count(\"files\")}-${!timestamp_unix_nano()}.txt", "${!metadata(\"kafka_key\")}.json", "${!json(\"doc.namespace\")}/${!json(\"doc.id\")}.json" ] }, "timeout": { "type": "string", "description": "The maximum period to wait on an upload before abandoning it and reattempting. Default: 3s", "examples": [ "1s", "500ms" ] } } }, "gcp_pubsub": { "type": "object", "properties": { "batching": { "type": "object", "description": "Configures a batching policy on this output. While the PubSub client maintains its own internal buffering mechanism, preparing larger batches of messages can further trade-off some latency for throughput.", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] }, "byte_threshold": { "type": "integer", "description": "Publish a batch when its size in bytes reaches this value. Default: 1000000" }, "count_threshold": { "type": "integer", "description": "Publish a pubsub buffer when it has this many messages Default: 100" }, "delay_threshold": { "type": "string", "description": "Publish a non-empty pubsub buffer after this delay has passed. Default: 10ms" }, "endpoint": { "type": "string", "description": "An optional endpoint to override the default of `pubsub.googleapis.com:443`. This can be used to connect to a region specific pubsub endpoint. For a list of valid values check out [this document.](https://cloud.google.com/pubsub/docs/reference/service_apis_overview#list_of_regional_endpoints) Default: ", "examples": [ "us-central1-pubsub.googleapis.com:443", "us-west3-pubsub.googleapis.com:443" ] }, "flow_control": { "type": "object", "description": "For a given topic, configures the PubSub client's internal buffer for messages to be published.", "properties": { "limit_exceeded_behavior": { "type": "string", "description": "Configures the behavior when trying to publish additional messages while the flow controller is full. The available options are block (default), ignore (disable), and signal_error (publish results will return an error). Default: block" }, "max_outstanding_bytes": { "type": "integer", "description": "Maximum size of buffered messages to be published. If less than or equal to zero, this is disabled. Default: -1" }, "max_outstanding_messages": { "type": "integer", "description": "Maximum number of buffered messages to be published. If less than or equal to zero, this is disabled. Default: 1000" } } }, "max_in_flight": { "type": "integer", "description": "The maximum number of messages to have in flight at a given time. Increasing this may improve throughput. Default: 64" }, "metadata": { "type": "object", "description": "Specify criteria for which metadata values are sent as attributes, all are sent by default.", "properties": { "exclude_prefixes": { "type": "array", "description": "Provide a list of explicit metadata key prefixes to be excluded when adding metadata to sent messages. Default: []", "items": { "type": "string" } } } }, "ordering_key": { "type": "string", "description": "The ordering key to use for publishing messages." }, "project": { "type": "string", "description": "The project ID of the topic to publish to." }, "publish_timeout": { "type": "string", "description": "The maximum length of time to wait before abandoning a publish attempt for a message. Default: 1m0s", "examples": [ "10s", "5m", "60m" ] }, "topic": { "type": "string", "description": "The topic to publish to." } } }, "grpc_client": { "type": "object", "properties": { "address": { "type": "string", "description": "The URI of the gRPC target to connect to.", "examples": [ "localhost:50051" ] }, "batching": { "type": "object", "description": "\nAllows you to configure a [batching policy](/docs/configuration/batching).", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] }, "health_check": { "type": "object", "properties": { "enabled": { "type": "boolean", "description": "Whether Bento should healthcheck the unary `Check` rpc endpoint on init connection: [gRPC Health Checking](https://grpc.io/docs/guides/health-checking/) Default: false" }, "service": { "type": "string", "description": "The name of the service to healthcheck, note that the default value of \"\", will attempt to check the health of the whole server Default: " } } }, "max_in_flight": { "type": "integer", "description": "The maximum number of messages to have in flight at a given time. Increase this to improve throughput. Default: 64" }, "method": { "type": "string", "description": "The name of the method to invoke", "examples": [ "SayHello" ] }, "oauth2": { "type": "object", "description": "Allows you to specify open authentication via OAuth version 2 using the client credentials token flow.", "properties": { "client_key": { "type": "string", "description": "A value used to identify the client to the token provider. Default: " }, "client_secret": { "type": "string", "description": "A secret used to establish ownership of the client key. Default: " }, "enabled": { "type": "boolean", "description": "Whether to use OAuth version 2 in requests. Default: false" }, "endpoint_params": { "type": "object", "description": "A list of optional endpoint parameters, values should be arrays of strings. Default: map[]", "examples": [ { "bar": [ "woof" ], "foo": [ "meow", "quack" ] } ] }, "scopes": { "type": "array", "description": "A list of optional requested permissions. Default: []", "items": { "type": "string" } }, "token_url": { "type": "string", "description": "The URL of the token provider. Default: " } } }, "propagate_response": { "type": "boolean", "description": "Whether responses from the server should be [propagated back](/docs/guides/sync_responses) to the input. Default: false" }, "proto_files": { "type": "array", "description": "A list of filepaths of .proto files that should contain the schemas necessary for the gRPC method. Default: []", "items": { "type": "string" }, "examples": [ [ "./grpc_test_server/helloworld.proto" ] ] }, "reflection": { "type": "boolean", "description": "If set to true, Bento will acquire the protobuf schema for the method from the server via [gRPC Reflection](https://grpc.io/docs/guides/reflection/). Default: false" }, "rpc_type": { "type": "string", "description": "The type of the rpc method. Default: unary" }, "service": { "type": "string", "description": "The name of the service.", "examples": [ "helloworld.Greeter" ] }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } } } }, "hdfs": { "type": "object", "properties": { "batching": { "type": "object", "description": "\nAllows you to configure a [batching policy](/docs/configuration/batching).", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] }, "directory": { "type": "string", "description": "A directory to store message files within. If the directory does not exist it will be created." }, "hosts": { "type": "array", "description": "A list of target host addresses to connect to.", "items": { "type": "string" }, "examples": [ "localhost:9000" ] }, "max_in_flight": { "type": "integer", "description": "The maximum number of messages to have in flight at a given time. Increase this to improve throughput. Default: 64" }, "path": { "type": "string", "description": "The path to upload messages as, interpolation functions should be used in order to generate unique file paths. Default: ${!count(\"files\")}-${!timestamp_unix_nano()}.txt" }, "user": { "type": "string", "description": "A user ID to connect as. Default: " } } }, "http_client": { "type": "object", "properties": { "backoff_on": { "type": "integer", "description": "A list of status codes whereby the request should be considered to have failed and retries should be attempted, but the period between them should be increased gradually. Default: [429]", "items": { "type": "integer" } }, "basic_auth": { "type": "object", "description": "Allows you to specify basic authentication.", "properties": { "enabled": { "type": "boolean", "description": "Whether to use basic authentication in requests. Default: false" }, "password": { "type": "string", "description": "A password to authenticate with. Default: " }, "username": { "type": "string", "description": "A username to authenticate as. Default: " } } }, "batch_as_multipart": { "type": "boolean", "description": "Send message batches as a single request using [RFC1341](https://www.w3.org/Protocols/rfc1341/7_2_Multipart.html). If disabled messages in batches will be sent as individual requests. Default: false" }, "batching": { "type": "object", "description": "\nAllows you to configure a [batching policy](/docs/configuration/batching).", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] }, "drop_on": { "type": "integer", "description": "A list of status codes whereby the request should be considered to have failed but retries should not be attempted. This is useful for preventing wasted retries for requests that will never succeed. Note that with these status codes the _request_ is dropped, but _message_ that caused the request will not be dropped. Default: []", "items": { "type": "integer" } }, "dump_request_log_level": { "type": "string", "description": "EXPERIMENTAL: Optionally set a level at which the request and response payload of each request made will be logged. Default: " }, "extract_headers": { "type": "object", "description": "Specify which response headers should be added to resulting synchronous response messages as metadata. Header keys are lowercased before matching, so ensure that your patterns target lowercased versions of the header keys that you expect. This field is not applicable unless `propagate_response` is set to `true`.", "properties": { "include_patterns": { "type": "array", "description": "Provide a list of explicit metadata key regular expression (re2) patterns to match against. Default: []", "items": { "type": "string" }, "examples": [ [ ".*" ], [ "_timestamp_unix$" ] ] }, "include_prefixes": { "type": "array", "description": "Provide a list of explicit metadata key prefixes to match against. Default: []", "items": { "type": "string" }, "examples": [ [ "foo_", "bar_" ], [ "kafka_" ], [ "content-" ] ] } } }, "headers": { "type": "object", "description": "A map of headers to add to the request. Default: map[]", "examples": [ { "Content-Type": "application/octet-stream", "traceparent": "${! tracing_span().traceparent }" } ] }, "jwt": { "type": "object", "description": "BETA: Allows you to specify JWT authentication.", "properties": { "claims": { "type": "object", "description": "A value used to identify the claims that issued the JWT. Default: map[]" }, "enabled": { "type": "boolean", "description": "Whether to use JWT authentication in requests. Default: false" }, "headers": { "type": "object", "description": "Add optional key/value headers to the JWT. Default: map[]" }, "private_key_file": { "type": "string", "description": "A file with the PEM encoded via PKCS1 or PKCS8 as private key. Default: " }, "signing_method": { "type": "string", "description": "A method used to sign the token such as RS256, RS384, RS512 or EdDSA. Default: " } } }, "max_in_flight": { "type": "integer", "description": "The maximum number of parallel message batches to have in flight at any given time. Default: 64" }, "max_retry_backoff": { "type": "string", "description": "The maximum period to wait between failed requests. Default: 300s" }, "metadata": { "type": "object", "description": "Specify optional matching rules to determine which metadata keys should be added to the HTTP request as headers.", "properties": { "include_patterns": { "type": "array", "description": "Provide a list of explicit metadata key regular expression (re2) patterns to match against. Default: []", "items": { "type": "string" }, "examples": [ [ ".*" ], [ "_timestamp_unix$" ] ] }, "include_prefixes": { "type": "array", "description": "Provide a list of explicit metadata key prefixes to match against. Default: []", "items": { "type": "string" }, "examples": [ [ "foo_", "bar_" ], [ "kafka_" ], [ "content-" ] ] } } }, "multipart": { "type": "array", "description": "EXPERIMENTAL: Create explicit multipart HTTP requests by specifying an array of parts to add to the request, each part specified consists of content headers and a data field that can be populated dynamically. If this field is populated it will override the default request creation behaviour. Default: []", "items": { "type": "object" } }, "oauth": { "type": "object", "description": "Allows you to specify open authentication via OAuth version 1.", "properties": { "access_token": { "type": "string", "description": "A value used to gain access to the protected resources on behalf of the user. Default: " }, "access_token_secret": { "type": "string", "description": "A secret provided in order to establish ownership of a given access token. Default: " }, "consumer_key": { "type": "string", "description": "A value used to identify the client to the service provider. Default: " }, "consumer_secret": { "type": "string", "description": "A secret used to establish ownership of the consumer key. Default: " }, "enabled": { "type": "boolean", "description": "Whether to use OAuth version 1 in requests. Default: false" } } }, "oauth2": { "type": "object", "description": "Allows you to specify open authentication via OAuth version 2 using the client credentials token flow.", "properties": { "client_key": { "type": "string", "description": "A value used to identify the client to the token provider. Default: " }, "client_secret": { "type": "string", "description": "A secret used to establish ownership of the client key. Default: " }, "enabled": { "type": "boolean", "description": "Whether to use OAuth version 2 in requests. Default: false" }, "endpoint_params": { "type": "object", "description": "A list of optional endpoint parameters, values should be arrays of strings. Default: map[]", "examples": [ { "bar": [ "woof" ], "foo": [ "meow", "quack" ] } ] }, "scopes": { "type": "array", "description": "A list of optional requested permissions. Default: []", "items": { "type": "string" } }, "token_url": { "type": "string", "description": "The URL of the token provider. Default: " } } }, "propagate_response": { "type": "boolean", "description": "Whether responses from the server should be [propagated back](/docs/guides/sync_responses) to the input. Default: false" }, "proxy_url": { "type": "string", "description": "An optional HTTP proxy URL." }, "rate_limit": { "type": "string", "description": "An optional [rate limit](/docs/components/rate_limits/about) to throttle requests by." }, "retries": { "type": "integer", "description": "The maximum number of retry attempts to make. Default: 3" }, "retry_period": { "type": "string", "description": "The base period to wait between failed requests. Default: 1s" }, "successful_on": { "type": "integer", "description": "A list of status codes whereby the attempt should be considered successful, this is useful for dropping requests that return non-2XX codes indicating that the message has been dealt with, such as a 303 See Other or a 409 Conflict. All 2XX codes are considered successful unless they are present within `backoff_on` or `drop_on`, regardless of this field. Default: []", "items": { "type": "integer" } }, "timeout": { "type": "string", "description": "A static timeout to apply to requests. Default: 5s" }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "transport": { "type": "object", "description": "Custom transport options.", "properties": { "dial_context": { "type": "object", "description": "Settings for the dialer used to create new connections.", "properties": { "keep_alive": { "type": "string", "description": "Keep-alive period for active network connections used by the dialer. Default: 30s" }, "timeout": { "type": "string", "description": "Timeout for establishing new network connections. Default: 30s" } } }, "expect_continue_timeout": { "type": "string", "description": "Time to wait for a server's first response headers after sending request headers when 'Expect: 100-continue' is used. Zero means send body immediately. Default: 1s" }, "force_http2": { "type": "boolean", "description": "If true, the transport will attempt to use HTTP/2. Default: true" }, "idle_connection_timeout": { "type": "string", "description": "Maximum time an idle keep-alive connection remains open before closing itself. Default: 90s" }, "max_idle_connections": { "type": "integer", "description": "Maximum number of idle keep-alive connections. Zero = unlimited. Default: 100" }, "tls_handshake_timeout": { "type": "string", "description": "Maximum time allowed for TLS handshake to complete. Default: 10s" } } }, "url": { "type": "string", "description": "The URL to connect to." }, "verb": { "type": "string", "description": "A verb to connect with Default: POST", "examples": [ "POST", "GET", "DELETE" ] } } }, "http_server": { "type": "object", "properties": { "address": { "type": "string", "description": "An alternative address to host from. If left empty the service wide address is used. Default: " }, "allowed_verbs": { "type": "array", "description": "An array of verbs that are allowed for the `path` and `stream_path` HTTP endpoint. Default: [GET]", "items": { "type": "string" } }, "cert_file": { "type": "string", "description": "Enable TLS by specifying a certificate and key file. Only valid with a custom `address`. Default: " }, "cors": { "type": "object", "description": "Adds Cross-Origin Resource Sharing headers. Only valid with a custom `address`.", "properties": { "allowed_headers": { "type": "array", "description": "Appends additional headers to the list of default allowed headers: Accept, Accept-Language, Content-Language \u0026 Origin. These default headers are therefore always allowed. Default: []", "items": { "type": "string" } }, "allowed_methods": { "type": "array", "description": "Used to explicitly set allowed methods in the Access-Control-Allow-Methods header. Default: [GET HEAD POST PUT PATCH DELETE]", "items": { "type": "string" } }, "allowed_origins": { "type": "array", "description": "An explicit list of origins that are allowed for CORS requests. Default: []", "items": { "type": "string" } }, "enabled": { "type": "boolean", "description": "Whether to allow CORS requests. Default: false" } } }, "heartbeat": { "type": "string", "description": "The time to wait before sending a heartbeat message. Default: 0s" }, "key_file": { "type": "string", "description": "Enable TLS by specifying a certificate and key file. Only valid with a custom `address`. Default: " }, "path": { "type": "string", "description": "The path from which discrete messages can be consumed. Default: /get" }, "ping_period": { "type": "string", "description": "Send pings to client with this period. Must be less than pong wait. Default: 54s" }, "pong_wait": { "type": "string", "description": "The time allowed to read the next pong message from the client. Default: 60s" }, "stream_format": { "type": "string", "description": "The format of the stream endpoint. `raw_bytes` delivers messages directly with newlines between batches, while `event_source` formats according to Server-Sent Events (SSE) specification with `data:` prefixes, compatible with EventSource API. Default: raw_bytes" }, "stream_path": { "type": "string", "description": "The path from which a continuous stream of messages can be consumed. Default: /get/stream" }, "timeout": { "type": "string", "description": "The maximum time to wait before a blocking, inactive connection is dropped (only applies to the `path` endpoint). Default: 5s" }, "write_wait": { "type": "string", "description": "The time allowed to write a message to the websocket. Default: 10s" }, "ws_message_type": { "type": "string", "description": "Type of websocket message Default: binary" }, "ws_path": { "type": "string", "description": "The path from which websocket connections can be established. Default: /get/ws" } } }, "inproc": { "type": "string", "description": " Default: " }, "kafka": { "type": "object", "properties": { "ack_replicas": { "type": "boolean", "description": "Ensure that messages have been copied across all replicas before acknowledging receipt. Default: false" }, "addresses": { "type": "array", "description": "A list of broker addresses to connect to. If an item of the list contains commas it will be expanded into multiple addresses.", "items": { "type": "string" }, "examples": [ [ "localhost:9092" ], [ "localhost:9041,localhost:9042" ], [ "localhost:9041", "localhost:9042" ] ] }, "backoff": { "type": "object", "description": "Control time intervals between retry attempts.", "properties": { "initial_interval": { "type": "string", "description": "The initial period to wait between retry attempts. Default: 3s", "examples": [ "50ms", "1s" ] }, "max_elapsed_time": { "type": "string", "description": "The maximum overall period of time to spend on retry attempts before the request is aborted. Setting this value to a zeroed duration (such as `0s`) will result in unbounded retries. Default: 30s", "examples": [ "1m", "1h" ] }, "max_interval": { "type": "string", "description": "The maximum period to wait between retry attempts Default: 10s", "examples": [ "5s", "1m" ] } } }, "batching": { "type": "object", "description": "\nAllows you to configure a [batching policy](/docs/configuration/batching).", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] }, "client_id": { "type": "string", "description": "An identifier for the client connection. Default: bento" }, "compression": { "type": "string", "description": "The compression algorithm to use. Default: none" }, "custom_topic_creation": { "type": "object", "description": "If enabled, topics will be created with the specified number of partitions and replication factor if they do not already exist.", "properties": { "enabled": { "type": "boolean", "description": "Whether to enable custom topic creation. Default: false" }, "partitions": { "type": "integer", "description": "The number of partitions to create for new topics. Leave at -1 to use the broker configured default. Must be \u003e= 1. Default: -1" }, "replication_factor": { "type": "integer", "description": "The replication factor to use for new topics. Leave at -1 to use the broker configured default. Must be an odd number, and less then or equal to the number of brokers. Default: -1" } } }, "idempotent_write": { "type": "boolean", "description": "Enable the idempotent write producer option. This requires the `IDEMPOTENT_WRITE` permission on `CLUSTER` and can be disabled if this permission is not available. Default: false" }, "inject_tracing_map": { "type": "string", "description": "EXPERIMENTAL: A [Bloblang mapping](/docs/guides/bloblang/about) used to inject an object containing tracing propagation information into outbound messages. The specification of the injected fields will match the format used by the service wide tracer.", "examples": [ "meta = @.assign(this)", "root.meta.span = this" ] }, "key": { "type": "string", "description": "The key to publish messages with. Default: " }, "max_in_flight": { "type": "integer", "description": "The maximum number of messages to have in flight at a given time. Increase this to improve throughput. Default: 64" }, "max_msg_bytes": { "type": "integer", "description": "The maximum size in bytes of messages sent to the target topic. Default: 1000000" }, "max_retries": { "type": "integer", "description": "The maximum number of retries before giving up on the request. If set to zero there is no discrete limit. Default: 0" }, "metadata": { "type": "object", "description": "Specify criteria for which metadata values are sent with messages as headers.", "properties": { "exclude_prefixes": { "type": "array", "description": "Provide a list of explicit metadata key prefixes to be excluded when adding metadata to sent messages. Default: []", "items": { "type": "string" } } } }, "partition": { "type": "string", "description": "The manually-specified partition to publish messages to, relevant only when the field `partitioner` is set to `manual`. Must be able to parse as a 32-bit integer. Default: " }, "partitioner": { "type": "string", "description": "The partitioning algorithm to use. Default: fnv1a_hash" }, "rack_id": { "type": "string", "description": "A rack identifier for this client. Default: " }, "retry_as_batch": { "type": "boolean", "description": "When enabled forces an entire batch of messages to be retried if any individual message fails on a send, otherwise only the individual messages that failed are retried. Disabling this helps to reduce message duplicates during intermittent errors, but also makes it impossible to guarantee strict ordering of messages. Default: false" }, "sasl": { "type": "object", "description": "Enables SASL authentication.", "properties": { "access_token": { "type": "string", "description": "A static OAUTHBEARER access token Default: " }, "aws": { "type": "object", "description": "Contains AWS specific fields for when the `mechanism` is set to `AWS_MSK_IAM`.", "properties": { "credentials": { "type": "object", "description": "Optional manual configuration of AWS credentials to use. More information can be found [in this document](/docs/guides/cloud/aws).", "properties": { "expiry_window": { "type": "string", "description": "Allow the credentials to trigger refreshing prior to the credentials actually expiring. This is beneficial so race conditions with expiring credentials do not cause requests to fail. For example '10s' would refresh credentials ten seconds before expiration. Setting to a duration of `0` disables the expiry window. Default: " }, "from_ec2_role": { "type": "boolean", "description": "Use the credentials of a host EC2 machine configured to assume [an IAM role associated with the instance](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html). Default: false" }, "id": { "type": "string", "description": "The ID of credentials to use. Default: " }, "profile": { "type": "string", "description": "A profile from `~/.aws/credentials` to use. Default: " }, "role": { "type": "string", "description": "A role ARN to assume. Default: " }, "role_external_id": { "type": "string", "description": "An external ID to provide when assuming a role. Default: " }, "secret": { "type": "string", "description": "The secret for the credentials being used. Default: " }, "token": { "type": "string", "description": "The token for the credentials being used, required when using short term credentials. Default: " } } }, "endpoint": { "type": "string", "description": "Allows you to specify a custom endpoint for the AWS API. Default: " }, "region": { "type": "string", "description": "The AWS region to target. Default: " } } }, "mechanism": { "type": "string", "description": "The SASL authentication mechanism, if left empty SASL authentication is not used. Default: none" }, "password": { "type": "string", "description": "A PLAIN password. It is recommended that you use environment variables to populate this field. Default: ", "examples": [ "${PASSWORD}" ] }, "token_cache": { "type": "string", "description": "Instead of using a static `access_token` allows you to query a [`cache`](/docs/components/caches/about) resource to fetch OAUTHBEARER tokens from Default: " }, "token_key": { "type": "string", "description": "Required when using a `token_cache`, the key to query the cache with for tokens. Default: " }, "user": { "type": "string", "description": "A PLAIN username. It is recommended that you use environment variables to populate this field. Default: ", "examples": [ "${USER}" ] } } }, "static_headers": { "type": "object", "description": "An optional map of static headers that should be added to messages in addition to metadata.", "examples": [ { "first-static-header": "value-1", "second-static-header": "value-2" } ] }, "target_version": { "type": "string", "description": "The version of the Kafka protocol to use. This limits the capabilities used by the client and should ideally match the version of your brokers. Defaults to the oldest supported stable version.", "examples": [ "2.1.0", "3.1.0" ] }, "timeout": { "type": "string", "description": "The maximum period of time to wait for message sends before abandoning the request and retrying. Default: 5s" }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "topic": { "type": "string", "description": "The topic to publish messages to." } } }, "kafka_franz": { "type": "object", "properties": { "batching": { "type": "object", "description": "\nAllows you to configure a [batching policy](/docs/configuration/batching).", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] }, "client_id": { "type": "string", "description": "An identifier for the client connection. Default: bento" }, "compression": { "type": "string", "description": "Optionally set an explicit compression type. The default preference is to use snappy when the broker supports it, and fall back to none if not." }, "idempotent_write": { "type": "boolean", "description": "Enable the idempotent write producer option. This requires the `IDEMPOTENT_WRITE` permission on `CLUSTER` and can be disabled if this permission is not available. Default: true" }, "key": { "type": "string", "description": "An optional key to populate for each message." }, "max_buffered_records": { "type": "integer", "description": "Sets the max amount of records the client will buffer, blocking produces until records are finished if this limit is reached. This overrides the `franz-kafka` default of 10,000. Default: 10000" }, "max_in_flight": { "type": "integer", "description": "The maximum number of batches to be sending in parallel at any given time. Default: 10" }, "max_message_bytes": { "type": "string", "description": "The maximum space in bytes than an individual message may take, messages larger than this value will be rejected. This field corresponds to Kafka's `max.message.bytes`. Default: 1MB", "examples": [ "100MB", "50mib" ] }, "metadata": { "type": "object", "description": "Determine which (if any) metadata values should be added to messages as headers.", "properties": { "include_patterns": { "type": "array", "description": "Provide a list of explicit metadata key regular expression (re2) patterns to match against. Default: []", "items": { "type": "string" }, "examples": [ [ ".*" ], [ "_timestamp_unix$" ] ] }, "include_prefixes": { "type": "array", "description": "Provide a list of explicit metadata key prefixes to match against. Default: []", "items": { "type": "string" }, "examples": [ [ "foo_", "bar_" ], [ "kafka_" ], [ "content-" ] ] } } }, "metadata_max_age": { "type": "string", "description": "This sets the maximum age for the client's cached metadata, to allow detection of new topics, partitions, etc. Default: 5m" }, "partition": { "type": "string", "description": "An optional explicit partition to set for each message. This field is only relevant when the `partitioner` is set to `manual`. The provided interpolation string must be a valid integer.", "examples": [ "${! metadata(\"partition\") }" ] }, "partitioner": { "type": "string", "description": "Override the default murmur2 hashing partitioner." }, "rack_id": { "type": "string", "description": "A rack identifier for this client. Default: " }, "sasl": { "type": "array", "description": "Specify one or more methods of SASL authentication. SASL is tried in order; if the broker supports the first mechanism, all connections will use that mechanism. If the first mechanism fails, the client will pick the first supported mechanism. If the broker does not support any client mechanisms, connections will fail.", "items": { "type": "object" }, "examples": [ [ { "mechanism": "SCRAM-SHA-512", "password": "bar", "username": "foo" } ] ] }, "seed_brokers": { "type": "array", "description": "A list of broker addresses to connect to in order to establish connections. If an item of the list contains commas it will be expanded into multiple addresses.", "items": { "type": "string" }, "examples": [ [ "localhost:9092" ], [ "foo:9092", "bar:9092" ], [ "foo:9092,bar:9092" ] ] }, "timeout": { "type": "string", "description": "The maximum period of time to wait for message sends before abandoning the request and retrying Default: 10s" }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "topic": { "type": "string", "description": "A topic to write messages to." }, "uniform_bytes_options": { "type": "object", "description": "Sets partitioner options when `partitioner` is of type `uniform_bytes`. These values will otherwise be ignored. Note, that future versions will likely see this approach reworked.", "properties": { "adaptive": { "type": "boolean", "description": "Sets a slight imbalance so that the partitioner can produce more to brokers that are less loaded. Default: false" }, "bytes": { "type": "string", "description": "The number of bytes the partitioner will return the same partition for. Default: 1MB" }, "keys": { "type": "boolean", "description": "If `true`, uses standard hashing based on record key for records with non-nil keys. Default: false" } } } } }, "mongodb": { "type": "object", "properties": { "backoff": { "type": "object", "description": "Control time intervals between retry attempts.", "properties": { "initial_interval": { "type": "string", "description": "The initial period to wait between retry attempts. Default: 1s" }, "max_elapsed_time": { "type": "string", "description": "The maximum period to wait before retry attempts are abandoned. If zero then no limit is used. Default: 30s" }, "max_interval": { "type": "string", "description": "The maximum period to wait between retry attempts. Default: 5s" } } }, "batching": { "type": "object", "description": "\nAllows you to configure a [batching policy](/docs/configuration/batching).", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] }, "collection": { "type": "string", "description": "The name of the target collection." }, "database": { "type": "string", "description": "The name of the target MongoDB database." }, "document_map": { "type": "string", "description": "A bloblang map representing a document to store within MongoDB, expressed as [extended JSON in canonical form](https://www.mongodb.com/docs/manual/reference/mongodb-extended-json/). The document map is required for the operations insert-one, replace-one and update-one. Default: ", "examples": [ "root.a = this.foo\nroot.b = this.bar" ] }, "filter_map": { "type": "string", "description": "A bloblang map representing a filter for a MongoDB command, expressed as [extended JSON in canonical form](https://www.mongodb.com/docs/manual/reference/mongodb-extended-json/). The filter map is required for all operations except insert-one. It is used to find the document(s) for the operation. For example in a delete-one case, the filter map should have the fields required to locate the document to delete. Default: ", "examples": [ "root.a = this.foo\nroot.b = this.bar" ] }, "hint_map": { "type": "string", "description": "A bloblang map representing the hint for the MongoDB command, expressed as [extended JSON in canonical form](https://www.mongodb.com/docs/manual/reference/mongodb-extended-json/). This map is optional and is used with all operations except insert-one. It is used to improve performance of finding the documents in the mongodb. Default: ", "examples": [ "root.a = this.foo\nroot.b = this.bar" ] }, "max_in_flight": { "type": "integer", "description": "The maximum number of messages to have in flight at a given time. Increase this to improve throughput. Default: 64" }, "max_retries": { "type": "integer", "description": "The maximum number of retries before giving up on the request. If set to zero there is no discrete limit. Default: 3" }, "operation": { "type": "string", "description": "The mongodb operation to perform. Default: update-one" }, "password": { "type": "string", "description": "The password to connect to the database. Default: " }, "upsert": { "type": "boolean", "description": "The upsert setting is optional and only applies for update-one and replace-one operations. If the filter specified in filter_map matches, the document is updated or replaced accordingly, otherwise it is created. Default: false" }, "url": { "type": "string", "description": "The URL of the target MongoDB server.", "examples": [ "mongodb://localhost:27017" ] }, "username": { "type": "string", "description": "The username to connect to the database. Default: " }, "write_concern": { "type": "object", "description": "The write concern settings for the mongo connection.", "properties": { "j": { "type": "boolean", "description": "J requests acknowledgement from MongoDB that write operations are written to the journal. Default: false" }, "w": { "type": "string", "description": "W requests acknowledgement that write operations propagate to the specified number of mongodb instances. Default: " }, "w_timeout": { "type": "string", "description": "The write concern timeout. Default: " } } } } }, "mqtt": { "type": "object", "properties": { "client_id": { "type": "string", "description": "An identifier for the client connection. Default: " }, "connect_timeout": { "type": "string", "description": "The maximum amount of time to wait in order to establish a connection before the attempt is abandoned. Default: 30s", "examples": [ "1s", "500ms" ] }, "dynamic_client_id_suffix": { "type": "string", "description": "Append a dynamically generated suffix to the specified `client_id` on each run of the pipeline. This can be useful when clustering Bento producers." }, "keepalive": { "type": "integer", "description": "Max seconds of inactivity before a keepalive message is sent. Default: 30" }, "max_in_flight": { "type": "integer", "description": "The maximum number of messages to have in flight at a given time. Increase this to improve throughput. Default: 64" }, "password": { "type": "string", "description": "A password to connect with. Default: " }, "qos": { "type": "integer", "description": "The QoS value to set for each message. Has options 0, 1, 2. Default: 1" }, "retained": { "type": "boolean", "description": "Set message as retained on the topic. Default: false" }, "retained_interpolated": { "type": "string", "description": "Override the value of `retained` with an interpolable value, this allows it to be dynamically set based on message contents. The value must resolve to either `true` or `false`." }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "topic": { "type": "string", "description": "The topic to publish messages to." }, "urls": { "type": "array", "description": "A list of URLs to connect to. If an item of the list contains commas it will be expanded into multiple URLs.", "items": { "type": "string" }, "examples": [ [ "tcp://localhost:1883" ] ] }, "user": { "type": "string", "description": "A username to connect with. Default: " }, "will": { "type": "object", "description": "Set last will message in case of Bento failure", "properties": { "enabled": { "type": "boolean", "description": "Whether to enable last will messages. Default: false" }, "payload": { "type": "string", "description": "Set payload for last will message. Default: " }, "qos": { "type": "integer", "description": "Set QoS for last will message. Valid values are: 0, 1, 2. Default: 0" }, "retained": { "type": "boolean", "description": "Set retained for last will message. Default: false" }, "topic": { "type": "string", "description": "Set topic for last will message. Default: " } } }, "write_timeout": { "type": "string", "description": "The maximum amount of time to wait to write data before the attempt is abandoned. Default: 3s", "examples": [ "1s", "500ms" ] } } }, "nanomsg": { "type": "object", "properties": { "bind": { "type": "boolean", "description": "Whether the URLs listed should be bind (otherwise they are connected to). Default: false" }, "max_in_flight": { "type": "integer", "description": "The maximum number of messages to have in flight at a given time. Increase this to improve throughput. Default: 64" }, "poll_timeout": { "type": "string", "description": "The maximum period of time to wait for a message to send before the request is abandoned and reattempted. Default: 5s" }, "socket_type": { "type": "string", "description": "The socket type to send with. Default: PUSH" }, "urls": { "type": "array", "description": "A list of URLs to connect to. If an item of the list contains commas it will be expanded into multiple URLs.", "items": { "type": "string" } } } }, "nats": { "type": "object", "properties": { "auth": { "type": "object", "description": "Optional configuration of NATS authentication parameters.", "properties": { "nkey_file": { "type": "string", "description": "An optional file containing a NKey seed.", "examples": [ "./seed.nk" ] }, "user_credentials_file": { "type": "string", "description": "An optional file containing user credentials which consist of an user JWT and corresponding NKey seed.", "examples": [ "./user.creds" ] }, "user_jwt": { "type": "string", "description": "An optional plain text user JWT (given along with the corresponding user NKey Seed)." }, "user_nkey_seed": { "type": "string", "description": "An optional plain text user NKey Seed (given along with the corresponding user JWT)." } } }, "headers": { "type": "object", "description": "Explicit message headers to add to messages. Default: map[]", "examples": [ { "Content-Type": "application/json", "Timestamp": "${!metadata(\"Timestamp\").string()}" } ] }, "inject_tracing_map": { "type": "string", "description": "EXPERIMENTAL: A [Bloblang mapping](/docs/guides/bloblang/about) used to inject an object containing tracing propagation information into outbound messages. The specification of the injected fields will match the format used by the service wide tracer.", "examples": [ "meta = @.assign(this)", "root.meta.span = this" ] }, "max_in_flight": { "type": "integer", "description": "The maximum number of messages to have in flight at a given time. Increase this to improve throughput. Default: 64" }, "metadata": { "type": "object", "description": "Determine which (if any) metadata values should be added to messages as headers.", "properties": { "include_patterns": { "type": "array", "description": "Provide a list of explicit metadata key regular expression (re2) patterns to match against. Default: []", "items": { "type": "string" }, "examples": [ [ ".*" ], [ "_timestamp_unix$" ] ] }, "include_prefixes": { "type": "array", "description": "Provide a list of explicit metadata key prefixes to match against. Default: []", "items": { "type": "string" }, "examples": [ [ "foo_", "bar_" ], [ "kafka_" ], [ "content-" ] ] } } }, "subject": { "type": "string", "description": "The subject to publish to.", "examples": [ "foo.bar.baz" ] }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "urls": { "type": "array", "description": "A list of URLs to connect to. If an item of the list contains commas it will be expanded into multiple URLs.", "items": { "type": "string" }, "examples": [ [ "nats://127.0.0.1:4222" ], [ "nats://username:password@127.0.0.1:4222" ] ] } } }, "nats_jetstream": { "type": "object", "properties": { "auth": { "type": "object", "description": "Optional configuration of NATS authentication parameters.", "properties": { "nkey_file": { "type": "string", "description": "An optional file containing a NKey seed.", "examples": [ "./seed.nk" ] }, "user_credentials_file": { "type": "string", "description": "An optional file containing user credentials which consist of an user JWT and corresponding NKey seed.", "examples": [ "./user.creds" ] }, "user_jwt": { "type": "string", "description": "An optional plain text user JWT (given along with the corresponding user NKey Seed)." }, "user_nkey_seed": { "type": "string", "description": "An optional plain text user NKey Seed (given along with the corresponding user JWT)." } } }, "headers": { "type": "object", "description": "Explicit message headers to add to messages. Default: map[]", "examples": [ { "Content-Type": "application/json", "Timestamp": "${!metadata(\"Timestamp\").string()}" } ] }, "inject_tracing_map": { "type": "string", "description": "EXPERIMENTAL: A [Bloblang mapping](/docs/guides/bloblang/about) used to inject an object containing tracing propagation information into outbound messages. The specification of the injected fields will match the format used by the service wide tracer.", "examples": [ "meta = @.assign(this)", "root.meta.span = this" ] }, "max_in_flight": { "type": "integer", "description": "The maximum number of messages to have in flight at a given time. Increase this to improve throughput. Default: 1024" }, "metadata": { "type": "object", "description": "Determine which (if any) metadata values should be added to messages as headers.", "properties": { "include_patterns": { "type": "array", "description": "Provide a list of explicit metadata key regular expression (re2) patterns to match against. Default: []", "items": { "type": "string" }, "examples": [ [ ".*" ], [ "_timestamp_unix$" ] ] }, "include_prefixes": { "type": "array", "description": "Provide a list of explicit metadata key prefixes to match against. Default: []", "items": { "type": "string" }, "examples": [ [ "foo_", "bar_" ], [ "kafka_" ], [ "content-" ] ] } } }, "subject": { "type": "string", "description": "A subject to write to.", "examples": [ "foo.bar.baz", "${! metadata(\"kafka_topic\") }", "foo.${! json(\"meta.type\") }" ] }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "urls": { "type": "array", "description": "A list of URLs to connect to. If an item of the list contains commas it will be expanded into multiple URLs.", "items": { "type": "string" }, "examples": [ [ "nats://127.0.0.1:4222" ], [ "nats://username:password@127.0.0.1:4222" ] ] } } }, "nats_kv": { "type": "object", "properties": { "auth": { "type": "object", "description": "Optional configuration of NATS authentication parameters.", "properties": { "nkey_file": { "type": "string", "description": "An optional file containing a NKey seed.", "examples": [ "./seed.nk" ] }, "user_credentials_file": { "type": "string", "description": "An optional file containing user credentials which consist of an user JWT and corresponding NKey seed.", "examples": [ "./user.creds" ] }, "user_jwt": { "type": "string", "description": "An optional plain text user JWT (given along with the corresponding user NKey Seed)." }, "user_nkey_seed": { "type": "string", "description": "An optional plain text user NKey Seed (given along with the corresponding user JWT)." } } }, "bucket": { "type": "string", "description": "The name of the KV bucket.", "examples": [ "my_kv_bucket" ] }, "key": { "type": "string", "description": "The key for each message.", "examples": [ "foo", "foo.bar.baz", "foo.${! json(\"meta.type\") }" ] }, "max_in_flight": { "type": "integer", "description": "The maximum number of messages to have in flight at a given time. Increase this to improve throughput. Default: 1024" }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "urls": { "type": "array", "description": "A list of URLs to connect to. If an item of the list contains commas it will be expanded into multiple URLs.", "items": { "type": "string" }, "examples": [ [ "nats://127.0.0.1:4222" ], [ "nats://username:password@127.0.0.1:4222" ] ] } } }, "nats_object_store": { "type": "object", "properties": { "auth": { "type": "object", "description": "Optional configuration of NATS authentication parameters.", "properties": { "nkey_file": { "type": "string", "description": "An optional file containing a NKey seed.", "examples": [ "./seed.nk" ] }, "user_credentials_file": { "type": "string", "description": "An optional file containing user credentials which consist of an user JWT and corresponding NKey seed.", "examples": [ "./user.creds" ] }, "user_jwt": { "type": "string", "description": "An optional plain text user JWT (given along with the corresponding user NKey Seed)." }, "user_nkey_seed": { "type": "string", "description": "An optional plain text user NKey Seed (given along with the corresponding user JWT)." } } }, "bucket": { "type": "string", "description": "The name of the object store bucket.", "examples": [ "my_bucket" ] }, "max_in_flight": { "type": "integer", "description": "The maximum number of messages to have in flight at a given time. Increase this to improve throughput. Default: 64" }, "object_name": { "type": "string", "description": "The object name for each message." }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "urls": { "type": "array", "description": "A list of URLs to connect to. If an item of the list contains commas it will be expanded into multiple URLs.", "items": { "type": "string" }, "examples": [ [ "nats://127.0.0.1:4222" ], [ "nats://username:password@127.0.0.1:4222" ] ] } } }, "nats_stream": { "type": "object", "properties": { "auth": { "type": "object", "description": "Optional configuration of NATS authentication parameters.", "properties": { "nkey_file": { "type": "string", "description": "An optional file containing a NKey seed.", "examples": [ "./seed.nk" ] }, "user_credentials_file": { "type": "string", "description": "An optional file containing user credentials which consist of an user JWT and corresponding NKey seed.", "examples": [ "./user.creds" ] }, "user_jwt": { "type": "string", "description": "An optional plain text user JWT (given along with the corresponding user NKey Seed)." }, "user_nkey_seed": { "type": "string", "description": "An optional plain text user NKey Seed (given along with the corresponding user JWT)." } } }, "client_id": { "type": "string", "description": "The client ID to connect with. Default: " }, "cluster_id": { "type": "string", "description": "The cluster ID to publish to." }, "inject_tracing_map": { "type": "string", "description": "EXPERIMENTAL: A [Bloblang mapping](/docs/guides/bloblang/about) used to inject an object containing tracing propagation information into outbound messages. The specification of the injected fields will match the format used by the service wide tracer.", "examples": [ "meta = @.assign(this)", "root.meta.span = this" ] }, "max_in_flight": { "type": "integer", "description": "The maximum number of messages to have in flight at a given time. Increase this to improve throughput. Default: 64" }, "subject": { "type": "string", "description": "The subject to publish to." }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "urls": { "type": "array", "description": "A list of URLs to connect to. If an item of the list contains commas it will be expanded into multiple URLs.", "items": { "type": "string" }, "examples": [ [ "nats://127.0.0.1:4222" ], [ "nats://username:password@127.0.0.1:4222" ] ] } } }, "nsq": { "type": "object", "properties": { "auth_secret": { "type": "string", "description": "An optional secret for NSQ authentication (requires nsqd 0.2.29+)." }, "max_in_flight": { "type": "integer", "description": "The maximum number of messages to have in flight at a given time. Increase this to improve throughput. Default: 64" }, "nsqd_tcp_address": { "type": "string", "description": "The address of the target NSQD server." }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "topic": { "type": "string", "description": "The topic to publish to." }, "user_agent": { "type": "string", "description": "A user agent to assume when connecting." } } }, "opensearch": { "type": "object", "properties": { "action": { "type": "string", "description": "The action to take on the document. This field must resolve to one of the following action types: `create`, `index`, `update` or `delete`. To write to a data stream, the action must be set to `create`." }, "aws": { "type": "object", "description": "Enables and customises connectivity to Amazon Elastic Service.", "properties": { "credentials": { "type": "object", "description": "Optional manual configuration of AWS credentials to use. More information can be found [in this document](/docs/guides/cloud/aws).", "properties": { "expiry_window": { "type": "string", "description": "Allow the credentials to trigger refreshing prior to the credentials actually expiring. This is beneficial so race conditions with expiring credentials do not cause requests to fail. For example '10s' would refresh credentials ten seconds before expiration. Setting to a duration of `0` disables the expiry window. Default: " }, "from_ec2_role": { "type": "boolean", "description": "Use the credentials of a host EC2 machine configured to assume [an IAM role associated with the instance](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html). Default: false" }, "id": { "type": "string", "description": "The ID of credentials to use. Default: " }, "profile": { "type": "string", "description": "A profile from `~/.aws/credentials` to use. Default: " }, "role": { "type": "string", "description": "A role ARN to assume. Default: " }, "role_external_id": { "type": "string", "description": "An external ID to provide when assuming a role. Default: " }, "secret": { "type": "string", "description": "The secret for the credentials being used. Default: " }, "token": { "type": "string", "description": "The token for the credentials being used, required when using short term credentials. Default: " } } }, "enabled": { "type": "boolean", "description": "Whether to connect to Amazon Elastic Service. Default: false" }, "endpoint": { "type": "string", "description": "Allows you to specify a custom endpoint for the AWS API. Default: " }, "region": { "type": "string", "description": "The AWS region to target. Default: " } } }, "basic_auth": { "type": "object", "description": "Allows you to specify basic authentication.", "properties": { "enabled": { "type": "boolean", "description": "Whether to use basic authentication in requests. Default: false" }, "password": { "type": "string", "description": "A password to authenticate with. Default: " }, "username": { "type": "string", "description": "A username to authenticate as. Default: " } } }, "batching": { "type": "object", "description": "\nAllows you to configure a [batching policy](/docs/configuration/batching).", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] }, "id": { "type": "string", "description": "The ID for indexed messages. Interpolation should be used in order to create a unique ID for each message.", "examples": [ "${!counter()}-${!timestamp_unix()}" ] }, "index": { "type": "string", "description": "The index to place messages." }, "max_in_flight": { "type": "integer", "description": "The maximum number of messages to have in flight at a given time. Increase this to improve throughput. Default: 64" }, "pipeline": { "type": "string", "description": "An optional pipeline id to preprocess incoming documents. Default: " }, "routing": { "type": "string", "description": "The routing key to use for the document. Default: " }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "urls": { "type": "array", "description": "A list of URLs to connect to. If an item of the list contains commas it will be expanded into multiple URLs.", "items": { "type": "string" }, "examples": [ [ "http://localhost:9200" ] ] } } }, "pulsar": { "type": "object", "properties": { "auth": { "type": "object", "description": "Optional configuration of Pulsar authentication methods.", "properties": { "oauth2": { "type": "object", "description": "Parameters for Pulsar OAuth2 authentication.", "properties": { "audience": { "type": "string", "description": "OAuth2 audience. Default: " }, "enabled": { "type": "boolean", "description": "Whether OAuth2 is enabled. Default: false" }, "issuer_url": { "type": "string", "description": "OAuth2 issuer URL. Default: " }, "private_key_file": { "type": "string", "description": "The path to a file containing a private key. Default: " } } }, "token": { "type": "object", "description": "Parameters for Pulsar Token authentication.", "properties": { "enabled": { "type": "boolean", "description": "Whether Token Auth is enabled. Default: false" }, "token": { "type": "string", "description": "Actual base64 encoded token. Default: " } } } } }, "key": { "type": "string", "description": "The key to publish messages with. Default: " }, "max_in_flight": { "type": "integer", "description": "The maximum number of messages to have in flight at a given time. Increase this to improve throughput. Default: 64" }, "ordering_key": { "type": "string", "description": "The ordering key to publish messages with. Default: " }, "tls": { "type": "object", "description": "Specify the path to a custom CA certificate to trust broker TLS service.", "properties": { "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] } } }, "topic": { "type": "string", "description": "The topic to publish to." }, "url": { "type": "string", "description": "A URL to connect to.", "examples": [ "pulsar://localhost:6650", "pulsar://pulsar.us-west.example.com:6650", "pulsar+ssl://pulsar.us-west.example.com:6651" ] } } }, "pusher": { "type": "object", "properties": { "appId": { "type": "string", "description": "Pusher app id" }, "batching": { "type": "object", "description": "maximum batch size is 10 (limit of the pusher library)", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] }, "channel": { "type": "string", "description": "Pusher channel to publish to. Interpolation functions can also be used", "examples": [ "my_channel", "${!json(\"id\")}" ] }, "cluster": { "type": "string", "description": "Pusher cluster" }, "event": { "type": "string", "description": "Event to publish to" }, "key": { "type": "string", "description": "Pusher key" }, "max_in_flight": { "type": "integer", "description": "The maximum number of parallel message batches to have in flight at any given time. Default: 1" }, "secret": { "type": "string", "description": "Pusher secret" }, "secure": { "type": "boolean", "description": "Enable SSL encryption Default: true" } } }, "questdb": { "type": "object", "properties": { "address": { "type": "string", "description": "Address of the QuestDB server's HTTP port (excluding protocol)", "examples": [ "localhost:9000" ] }, "batching": { "type": "object", "description": "\nAllows you to configure a [batching policy](/docs/configuration/batching).", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] }, "designated_timestamp_field": { "type": "string", "description": "Name of the designated timestamp field" }, "designated_timestamp_unit": { "type": "string", "description": "Designated timestamp field units Default: auto" }, "doubles": { "type": "array", "description": "Columns that should be double type, (int is default)", "items": { "type": "string" } }, "error_on_empty_messages": { "type": "boolean", "description": "Mark a message as errored if it is empty after field validation Default: false" }, "max_in_flight": { "type": "integer", "description": "The maximum number of messages to have in flight at a given time. Increase this to improve throughput. Default: 64" }, "password": { "type": "string", "description": "Password for HTTP basic auth" }, "request_min_throughput": { "type": "integer", "description": "Minimum expected throughput in bytes per second for HTTP requests. If the throughput is lower than this value, the connection will time out. This is used to calculate an additional timeout on top of request_timeout. This is useful for large requests. You can set this value to 0 to disable this logic." }, "request_timeout": { "type": "string", "description": "The time to wait for a response from the server. This is in addition to the calculation derived from the request_min_throughput parameter." }, "retry_timeout": { "type": "string", "description": "The time to continue retrying after a failed HTTP request. The interval between retries is an exponential backoff starting at 10ms and doubling after each failed attempt up to a maximum of 1 second." }, "symbols": { "type": "array", "description": "Columns that should be the SYMBOL type (string values default to STRING)", "items": { "type": "string" } }, "table": { "type": "string", "description": "Destination table", "examples": [ "trades" ] }, "timestamp_string_fields": { "type": "array", "description": "String fields with textual timestamps", "items": { "type": "string" } }, "timestamp_string_format": { "type": "string", "description": "Timestamp format, used when parsing timestamp string fields. Specified in golang's time.Parse layout Default: Jan _2 15:04:05.000000Z0700" }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "token": { "type": "string", "description": "Bearer token for HTTP auth (takes precedence over basic auth username \u0026 password)" }, "username": { "type": "string", "description": "Username for HTTP basic auth" } } }, "redis_hash": { "type": "object", "properties": { "fields": { "type": "object", "description": "A map of key/value pairs to set as hash fields. Default: map[]" }, "key": { "type": "string", "description": "The key for each message, function interpolations should be used to create a unique key per message.", "examples": [ "${! @.kafka_key )}", "${! this.doc.id }", "${! count(\"msgs\") }" ] }, "kind": { "type": "string", "description": "Specifies a simple, cluster-aware, or failover-aware redis client. Default: simple" }, "master": { "type": "string", "description": "Name of the redis master when `kind` is `failover` Default: ", "examples": [ "mymaster" ] }, "max_in_flight": { "type": "integer", "description": "The maximum number of messages to have in flight at a given time. Increase this to improve throughput. Default: 64" }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.\n\n**Troubleshooting**\n\nSome cloud hosted instances of Redis (such as Azure Cache) might need some hand holding in order to establish stable connections. Unfortunately, it is often the case that TLS issues will manifest as generic error messages such as \"i/o timeout\". If you're using TLS and are seeing connectivity problems consider setting `enable_renegotiation` to `true`, and ensuring that the server supports at least TLS version 1.2.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "url": { "type": "string", "description": "The URL of the target Redis server. Database is optional and is supplied as the URL path.", "examples": [ "redis://:6397", "redis://localhost:6379", "redis://foousername:foopassword@redisplace:6379", "redis://:foopassword@redisplace:6379", "redis://localhost:6379/1", "redis://localhost:6379/1,redis://localhost:6380/1" ] }, "walk_json_object": { "type": "boolean", "description": "Whether to walk each message as a JSON object and add each key/value pair to the list of hash fields to set. Default: false" }, "walk_metadata": { "type": "boolean", "description": "Whether all metadata fields of messages should be walked and added to the list of hash fields to set. Default: false" } } }, "redis_list": { "type": "object", "properties": { "batching": { "type": "object", "description": "\nAllows you to configure a [batching policy](/docs/configuration/batching).", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] }, "command": { "type": "string", "description": "The command used to push elements to the Redis list Default: rpush" }, "key": { "type": "string", "description": "The key for each message, function interpolations can be optionally used to create a unique key per message.", "examples": [ "some_list", "${! @.kafka_key )}", "${! this.doc.id }", "${! count(\"msgs\") }" ] }, "kind": { "type": "string", "description": "Specifies a simple, cluster-aware, or failover-aware redis client. Default: simple" }, "master": { "type": "string", "description": "Name of the redis master when `kind` is `failover` Default: ", "examples": [ "mymaster" ] }, "max_in_flight": { "type": "integer", "description": "The maximum number of messages to have in flight at a given time. Increase this to improve throughput. Default: 64" }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.\n\n**Troubleshooting**\n\nSome cloud hosted instances of Redis (such as Azure Cache) might need some hand holding in order to establish stable connections. Unfortunately, it is often the case that TLS issues will manifest as generic error messages such as \"i/o timeout\". If you're using TLS and are seeing connectivity problems consider setting `enable_renegotiation` to `true`, and ensuring that the server supports at least TLS version 1.2.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "url": { "type": "string", "description": "The URL of the target Redis server. Database is optional and is supplied as the URL path.", "examples": [ "redis://:6397", "redis://localhost:6379", "redis://foousername:foopassword@redisplace:6379", "redis://:foopassword@redisplace:6379", "redis://localhost:6379/1", "redis://localhost:6379/1,redis://localhost:6380/1" ] } } }, "redis_pubsub": { "type": "object", "properties": { "batching": { "type": "object", "description": "\nAllows you to configure a [batching policy](/docs/configuration/batching).", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] }, "channel": { "type": "string", "description": "The channel to publish messages to." }, "kind": { "type": "string", "description": "Specifies a simple, cluster-aware, or failover-aware redis client. Default: simple" }, "master": { "type": "string", "description": "Name of the redis master when `kind` is `failover` Default: ", "examples": [ "mymaster" ] }, "max_in_flight": { "type": "integer", "description": "The maximum number of messages to have in flight at a given time. Increase this to improve throughput. Default: 64" }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.\n\n**Troubleshooting**\n\nSome cloud hosted instances of Redis (such as Azure Cache) might need some hand holding in order to establish stable connections. Unfortunately, it is often the case that TLS issues will manifest as generic error messages such as \"i/o timeout\". If you're using TLS and are seeing connectivity problems consider setting `enable_renegotiation` to `true`, and ensuring that the server supports at least TLS version 1.2.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "url": { "type": "string", "description": "The URL of the target Redis server. Database is optional and is supplied as the URL path.", "examples": [ "redis://:6397", "redis://localhost:6379", "redis://foousername:foopassword@redisplace:6379", "redis://:foopassword@redisplace:6379", "redis://localhost:6379/1", "redis://localhost:6379/1,redis://localhost:6380/1" ] } } }, "redis_streams": { "type": "object", "properties": { "batching": { "type": "object", "description": "\nAllows you to configure a [batching policy](/docs/configuration/batching).", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] }, "body_key": { "type": "string", "description": "A key to set the raw body of the message to. Default: body" }, "kind": { "type": "string", "description": "Specifies a simple, cluster-aware, or failover-aware redis client. Default: simple" }, "master": { "type": "string", "description": "Name of the redis master when `kind` is `failover` Default: ", "examples": [ "mymaster" ] }, "max_in_flight": { "type": "integer", "description": "The maximum number of messages to have in flight at a given time. Increase this to improve throughput. Default: 64" }, "max_length": { "type": "integer", "description": "When greater than zero enforces a rough cap on the length of the target stream. Default: 0" }, "metadata": { "type": "object", "description": "Specify criteria for which metadata values are included in the message body.", "properties": { "exclude_prefixes": { "type": "array", "description": "Provide a list of explicit metadata key prefixes to be excluded when adding metadata to sent messages. Default: []", "items": { "type": "string" } } } }, "stream": { "type": "string", "description": "The stream to add messages to." }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.\n\n**Troubleshooting**\n\nSome cloud hosted instances of Redis (such as Azure Cache) might need some hand holding in order to establish stable connections. Unfortunately, it is often the case that TLS issues will manifest as generic error messages such as \"i/o timeout\". If you're using TLS and are seeing connectivity problems consider setting `enable_renegotiation` to `true`, and ensuring that the server supports at least TLS version 1.2.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "url": { "type": "string", "description": "The URL of the target Redis server. Database is optional and is supplied as the URL path.", "examples": [ "redis://:6397", "redis://localhost:6379", "redis://foousername:foopassword@redisplace:6379", "redis://:foopassword@redisplace:6379", "redis://localhost:6379/1", "redis://localhost:6379/1,redis://localhost:6380/1" ] } } }, "reject": { "type": "string", "description": " Default: " }, "reject_errored": { "type": "object" }, "resource": { "type": "string", "description": " Default: " }, "retry": { "type": "object", "properties": { "backoff": { "type": "object", "description": "Control time intervals between retry attempts.", "properties": { "initial_interval": { "type": "string", "description": "The initial period to wait between retry attempts. Default: 500ms" }, "max_elapsed_time": { "type": "string", "description": "The maximum period to wait before retry attempts are abandoned. If zero then no limit is used. Default: 0s" }, "max_interval": { "type": "string", "description": "The maximum period to wait between retry attempts. Default: 3s" } } }, "max_retries": { "type": "integer", "description": "The maximum number of retries before giving up on the request. If set to zero there is no discrete limit. Default: 0" }, "output": { "type": "object", "description": "A child output." } } }, "s2": { "type": "object", "properties": { "access_token": { "type": "string", "description": "Access token for S2 account" }, "basin": { "type": "string", "description": "Basin name" }, "batching": { "type": "object", "description": "\nAllows you to configure a [batching policy](/docs/configuration/batching).", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] }, "fencing_token": { "type": "string", "description": "Enforce a fencing token (base64 encoded)", "examples": [ "aGVsbG8gczI=" ] }, "max_in_flight": { "type": "integer", "description": "The maximum number of messages to have in flight at a given time. Increase this to improve throughput. Default: 64" }, "stream": { "type": "string", "description": "Stream name" } } }, "sftp": { "type": "object", "properties": { "address": { "type": "string", "description": "The address of the server to connect to." }, "codec": { "type": "string", "description": "The way in which the bytes of messages should be written out into the output data stream. It's possible to write lines using a custom delimiter with the `delim:x` codec, where x is the character sequence custom delimiter. Default: all-bytes", "examples": [ "lines", "delim:\t", "delim:foobar" ] }, "credentials": { "type": "object", "description": "The credentials to use to log into the target server.", "properties": { "password": { "type": "string", "description": "The password for the username to connect to the SFTP server. Default: " }, "private_key_file": { "type": "string", "description": "The private key for the username to connect to the SFTP server. Default: " }, "private_key_pass": { "type": "string", "description": "Optional passphrase for private key. Default: " }, "username": { "type": "string", "description": "The username to connect to the SFTP server. Default: " } } }, "max_in_flight": { "type": "integer", "description": "The maximum number of messages to have in flight at a given time. Increase this to improve throughput. Default: 64" }, "path": { "type": "string", "description": "The file to save the messages to on the server." } } }, "slack_webhook": { "type": "object", "properties": { "timeout": { "type": "string", "description": "The maximum time to wait before abandoning a request (and trying again). Default: 5s" }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "webhook": { "type": "string", "description": "Slack webhook URL to post messages", "examples": [ "https://hooks.slack.com/services/T00000000/B00000000/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" ] } } }, "snowflake_put": { "type": "object", "properties": { "account": { "type": "string", "description": "Account name, which is the same as the Account Identifier\nas described [here](https://docs.snowflake.com/en/user-guide/admin-account-identifier.html#where-are-account-identifiers-used).\nHowever, when using an [Account Locator](https://docs.snowflake.com/en/user-guide/admin-account-identifier.html#using-an-account-locator-as-an-identifier),\nthe Account Identifier is formatted as `\u003caccount_locator\u003e.\u003cregion_id\u003e.\u003ccloud\u003e` and this field needs to be\npopulated using the `\u003caccount_locator\u003e` part.\n" }, "batching": { "type": "object", "description": "\nAllows you to configure a [batching policy](/docs/configuration/batching).", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] }, "client_session_keep_alive": { "type": "boolean", "description": "Enable Snowflake keepalive mechanism to prevent the client session from expiring after 4 hours (error 390114). Default: false" }, "cloud": { "type": "string", "description": "Optional cloud platform field which needs to be populated\nwhen using an [Account Locator](https://docs.snowflake.com/en/user-guide/admin-account-identifier.html#using-an-account-locator-as-an-identifier)\nand it must be set to the `\u003ccloud\u003e` part of the Account Identifier\n(`\u003caccount_locator\u003e.\u003cregion_id\u003e.\u003ccloud\u003e`).\n", "examples": [ "aws", "gcp", "azure" ] }, "compression": { "type": "string", "description": "Compression type. Default: AUTO" }, "database": { "type": "string", "description": "Database." }, "file_extension": { "type": "string", "description": "Stage file extension. Will be derived from the configured `compression` if not set or empty. Default: ", "examples": [ "csv", "parquet" ] }, "file_name": { "type": "string", "description": "Stage file name. Will be equal to the Request ID if not set or empty. Default: " }, "max_in_flight": { "type": "integer", "description": "The maximum number of parallel message batches to have in flight at any given time. Default: 1" }, "password": { "type": "string", "description": "An optional password." }, "path": { "type": "string", "description": "Stage path. Default: " }, "private_key_file": { "type": "string", "description": "The path to a file containing the private SSH key." }, "private_key_pass": { "type": "string", "description": "An optional private SSH key passphrase." }, "region": { "type": "string", "description": "Optional region field which needs to be populated when using\nan [Account Locator](https://docs.snowflake.com/en/user-guide/admin-account-identifier.html#using-an-account-locator-as-an-identifier)\nand it must be set to the `\u003cregion_id\u003e` part of the Account Identifier\n(`\u003caccount_locator\u003e.\u003cregion_id\u003e.\u003ccloud\u003e`).\n", "examples": [ "us-west-2" ] }, "request_id": { "type": "string", "description": "Request ID. Will be assigned a random UUID (v4) string if not set or empty. Default: " }, "role": { "type": "string", "description": "Role." }, "schema": { "type": "string", "description": "Schema." }, "snowpipe": { "type": "string", "description": "An optional Snowpipe name. Use the `\u003csnowpipe\u003e` part from `\u003cdatabase\u003e.\u003cschema\u003e.\u003csnowpipe\u003e`." }, "stage": { "type": "string", "description": "Stage name. Use either one of the\n[supported](https://docs.snowflake.com/en/user-guide/data-load-local-file-system-create-stage.html) stage types." }, "upload_parallel_threads": { "type": "integer", "description": "Specifies the number of threads to use for uploading files. Default: 4" }, "user": { "type": "string", "description": "Username." }, "warehouse": { "type": "string", "description": "Warehouse." } } }, "socket": { "type": "object", "properties": { "address": { "type": "string", "description": "The address to connect to.", "examples": [ "/tmp/bento.sock", "127.0.0.1:6000" ] }, "codec": { "type": "string", "description": "The way in which the bytes of messages should be written out into the output data stream. It's possible to write lines using a custom delimiter with the `delim:x` codec, where x is the character sequence custom delimiter. Default: lines", "examples": [ "lines", "delim:\t", "delim:foobar" ] }, "network": { "type": "string", "description": "A network type to connect as." } } }, "splunk_hec": { "type": "object", "properties": { "batching_byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If 0 disables size based batching. Splunk Cloud recommends limiting content length of HEC payload to 1 MB. Default: 1000000" }, "batching_count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If 0 disables count based batching. Default: 100" }, "batching_period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: 30s" }, "event_host": { "type": "string", "description": "Set the host value to assign to the event data. Overrides existing host field if present. Default: " }, "event_index": { "type": "string", "description": "Set the index value to assign to the event data. Overrides existing index field if present. Default: " }, "event_source": { "type": "string", "description": "Set the source value to assign to the event data. Overrides existing source field if present. Default: " }, "event_sourcetype": { "type": "string", "description": "Set the sourcetype value to assign to the event data. Overrides existing sourcetype field if present. Default: " }, "gzip": { "type": "boolean", "description": "Enable gzip compression Default: false" }, "max_in_flight": { "type": "integer", "description": "The maximum number of parallel message batches to have in flight at any given time. Default: 64" }, "rate_limit": { "type": "string", "description": "An optional rate limit resource to restrict API requests with. Default: " }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" }, "token": { "type": "string", "description": "A bot token used for authentication." }, "url": { "type": "string", "description": "Full HTTP Endpoint Collector (HEC) URL, ie. https://foobar.splunkcloud.com/services/collector/event" } } }, "sql": { "type": "object", "properties": { "args_mapping": { "type": "string", "description": "An optional [Bloblang mapping](/docs/guides/bloblang/about) which should evaluate to an array of values matching in size to the number of placeholder arguments in the field `query`.", "examples": [ "root = [ this.cat.meow, this.doc.woofs[0] ]", "root = [ metadata(\"user.id\").string() ]" ] }, "batching": { "type": "object", "description": "\nAllows you to configure a [batching policy](/docs/configuration/batching).", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] }, "data_source_name": { "type": "string", "description": "Data source name." }, "driver": { "type": "string", "description": "A database [driver](#drivers) to use." }, "max_in_flight": { "type": "integer", "description": "The maximum number of inserts to run in parallel. Default: 64" }, "query": { "type": "string", "description": "The query to execute. The style of placeholder to use depends on the driver, some drivers require question marks (`?`) whereas others expect incrementing dollar signs (`$1`, `$2`, and so on) or colons (`:1`, `:2` and so on). The style to use is outlined in this table:\n\n| Driver | Placeholder Style |\n|---|---|\n| `clickhouse` | Dollar sign |\n| `mysql` | Question mark |\n| `postgres` | Dollar sign |\n| `mssql` | Question mark |\n| `sqlite` | Question mark |\n| `oracle` | Colon |\n| `snowflake` | Question mark |\n| `spanner` | Question mark |\n| `trino` | Question mark |\n| `gocosmos` | Colon |\n", "examples": [ "INSERT INTO footable (foo, bar, baz) VALUES (?, ?, ?);" ] } } }, "sql_insert": { "type": "object", "properties": { "args_mapping": { "type": "string", "description": "A [Bloblang mapping](/docs/guides/bloblang/about) which should evaluate to an array of values matching in size to the number of columns specified.", "examples": [ "root = [ this.cat.meow, this.doc.woofs[0] ]", "root = [ metadata(\"user.id\").string() ]" ] }, "azure": { "type": "object", "description": "Optional Fields that can be set to use Azure based authentication for Azure Postgres SQL", "properties": { "entra_enabled": { "type": "boolean", "description": "An optional field used to generate an entra token to connect to 'Azure Database for PostgreSQL flexible server', This will create a new connection string with the host, user and database from the DSN field - you may need to URL encode the dsn! The [Default Azure Credential Chain](https://learn.microsoft.com/en-gb/azure/developer/go/sdk/authentication/authentication-overview#defaultazurecredential) is used from the Azure SDK. Default: false" }, "token_request_options": { "type": "object", "properties": { "claims": { "type": "string", "description": "Set additional claims for the token. Default: " }, "enable_cae": { "type": "boolean", "description": "Indicates whether to enable Continuous Access Evaluation (CAE) for the requested token Default: false" }, "scopes": { "type": "array", "description": "Scopes contains the list of permission scopes required for the token. Default: [https://ossrdbms-aad.database.windows.net/.default]", "items": { "type": "string" } }, "tenant_id": { "type": "string", "description": "tenant_id identifies the tenant from which to request the token. azure credentials authenticate in their configured default tenants when this field isn't set. Default: " } } } } }, "batching": { "type": "object", "description": "\nAllows you to configure a [batching policy](/docs/configuration/batching).", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] }, "columns": { "type": "array", "description": "A list of columns to insert.", "items": { "type": "string" }, "examples": [ [ "foo", "bar", "baz" ] ] }, "conn_max_idle": { "type": "integer", "description": "An optional maximum number of connections in the idle connection pool. If conn_max_open is greater than 0 but less than the new conn_max_idle, then the new conn_max_idle will be reduced to match the conn_max_open limit. If `value \u003c= 0`, no idle connections are retained. The default max idle connections is currently 2. This may change in a future release. Default: 2" }, "conn_max_idle_time": { "type": "string", "description": "An optional maximum amount of time a connection may be idle. Expired connections may be closed lazily before reuse. If `value \u003c= 0`, connections are not closed due to a connections idle time." }, "conn_max_life_time": { "type": "string", "description": "An optional maximum amount of time a connection may be reused. Expired connections may be closed lazily before reuse. If `value \u003c= 0`, connections are not closed due to a connections age." }, "conn_max_open": { "type": "integer", "description": "An optional maximum number of open connections to the database. If conn_max_idle is greater than 0 and the new conn_max_open is less than conn_max_idle, then conn_max_idle will be reduced to match the new conn_max_open limit. If `value \u003c= 0`, then there is no limit on the number of open connections. The default is 0 (unlimited)." }, "credentials": { "type": "object", "description": "Optional manual configuration of AWS credentials to use. More information can be found [in this document](/docs/guides/cloud/aws).", "properties": { "expiry_window": { "type": "string", "description": "Allow the credentials to trigger refreshing prior to the credentials actually expiring. This is beneficial so race conditions with expiring credentials do not cause requests to fail. For example '10s' would refresh credentials ten seconds before expiration. Setting to a duration of `0` disables the expiry window. Default: " }, "from_ec2_role": { "type": "boolean", "description": "Use the credentials of a host EC2 machine configured to assume [an IAM role associated with the instance](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html). Default: false" }, "id": { "type": "string", "description": "The ID of credentials to use. Default: " }, "profile": { "type": "string", "description": "A profile from `~/.aws/credentials` to use. Default: " }, "role": { "type": "string", "description": "A role ARN to assume. Default: " }, "role_external_id": { "type": "string", "description": "An external ID to provide when assuming a role. Default: " }, "secret": { "type": "string", "description": "The secret for the credentials being used. Default: " }, "token": { "type": "string", "description": "The token for the credentials being used, required when using short term credentials. Default: " } } }, "driver": { "type": "string", "description": "A database [driver](#drivers) to use." }, "dsn": { "type": "string", "description": "A Data Source Name to identify the target database.\n\n#### Drivers\n\nThe following is a list of supported drivers, their placeholder style, and their respective DSN formats:\n\n| Driver | Data Source Name Format |\n|---|---|\n| `clickhouse` | [`clickhouse://[username[:password]@][netloc][:port]/dbname[?param1=value1\u0026...\u0026paramN=valueN]`](https://github.com/ClickHouse/clickhouse-go#dsn) |\n| `mysql` | `[username[:password]@][protocol[(address)]]/dbname[?param1=value1\u0026...\u0026paramN=valueN]` |\n| `postgres` | `postgres://[user[:password]@][netloc][:port][/dbname][?param1=value1\u0026...]` |\n| `mssql` | `sqlserver://[user[:password]@][netloc][:port][?database=dbname\u0026param1=value1\u0026...]` |\n| `sqlite` | `file:/path/to/filename.db[?param\u0026=value1\u0026...]` |\n| `oracle` | `oracle://[username[:password]@][netloc][:port]/service_name?server=server2\u0026server=server3` |\n| `snowflake` | `username[:password]@account_identifier/dbname/schemaname[?param1=value\u0026...\u0026paramN=valueN]` |\n| `spanner` | `projects/[project]/instances/[instance]/databases/dbname` |\n| `trino` | [`http[s]://user[:pass]@host[:port][?parameters]`](https://github.com/trinodb/trino-go-client#dsn-data-source-name) |\n| `gocosmos` | [`AccountEndpoint=\u003ccosmosdb-endpoint\u003e;AccountKey=\u003ccosmosdb-account-key\u003e[;TimeoutMs=\u003ctimeout-in-ms\u003e][;Version=\u003ccosmosdb-api-version\u003e][;DefaultDb/Db=\u003cdb-name\u003e][;AutoId=\u003ctrue/false\u003e][;InsecureSkipVerify=\u003ctrue/false\u003e]`](https://pkg.go.dev/github.com/microsoft/gocosmos#readme-example-usage) |\n\nPlease note that the `postgres` driver enforces SSL by default, you can override this with the parameter `sslmode=disable` if required.\n\nThe `snowflake` driver supports multiple DSN formats. Please consult [the docs](https://pkg.go.dev/github.com/snowflakedb/gosnowflake#hdr-Connection_String) for more details. For [key pair authentication](https://docs.snowflake.com/en/user-guide/key-pair-auth.html#configuring-key-pair-authentication), the DSN has the following format: `\u003csnowflake_user\u003e@\u003csnowflake_account\u003e/\u003cdb_name\u003e/\u003cschema_name\u003e?warehouse=\u003cwarehouse\u003e\u0026role=\u003crole\u003e\u0026authenticator=snowflake_jwt\u0026privateKey=\u003cbase64_url_encoded_private_key\u003e`, where the value for the `privateKey` parameter can be constructed from an unencrypted RSA private key file `rsa_key.p8` using `openssl enc -d -base64 -in rsa_key.p8 | basenc --base64url -w0` (you can use `gbasenc` insted of `basenc` on OSX if you install `coreutils` via Homebrew). If you have a password-encrypted private key, you can decrypt it using `openssl pkcs8 -in rsa_key_encrypted.p8 -out rsa_key.p8`. Also, make sure fields such as the username are URL-encoded.\n\nThe [`gocosmos`](https://pkg.go.dev/github.com/microsoft/gocosmos) driver is still experimental, but it has support for [hierarchical partition keys](https://learn.microsoft.com/en-us/azure/cosmos-db/hierarchical-partition-keys) as well as [cross-partition queries](https://learn.microsoft.com/en-us/azure/cosmos-db/nosql/how-to-query-container#cross-partition-query). Please refer to the [SQL notes](https://github.com/microsoft/gocosmos/blob/main/SQL.md) for details.", "examples": [ "clickhouse://username:password@host1:9000,host2:9000/database?dial_timeout=200ms\u0026max_execution_time=60", "foouser:foopassword@tcp(localhost:3306)/foodb", "postgres://foouser:foopass@localhost:5432/foodb?sslmode=disable", "oracle://foouser:foopass@localhost:1521/service_name" ] }, "endpoint": { "type": "string", "description": "Allows you to specify a custom endpoint for the AWS API. Default: " }, "iam_enabled": { "type": "boolean", "description": "An optional field used to generate an IAM authentication token to connect to an Amazon Relational Database (RDS) DB instance. This will overwrite the Password in the DSN with the generated token only if the drivers are `mysql` or `postgres`. Default: false" }, "init_files": { "type": "array", "description": "\nAn optional list of file paths containing SQL statements to execute immediately upon the first connection to the target database. This is a useful way to initialise tables before processing data. Glob patterns are supported, including super globs (double star).\n\nCare should be taken to ensure that the statements are idempotent, and therefore would not cause issues when run multiple times after service restarts. If both `init_statement` and `init_files` are specified the `init_statement` is executed _after_ the `init_files`.\n\nIf a statement fails for any reason a warning log will be emitted but the operation of this component will not be stopped.\n", "items": { "type": "string" }, "examples": [ [ "./init/*.sql" ], [ "./foo.sql", "./bar.sql" ] ] }, "init_statement": { "type": "string", "description": "\nAn optional SQL statement to execute immediately upon the first connection to the target database. This is a useful way to initialise tables before processing data. Care should be taken to ensure that the statement is idempotent, and therefore would not cause issues when run multiple times after service restarts.\n\nIf both `init_statement` and `init_files` are specified the `init_statement` is executed _after_ the `init_files`.\n\nIf the statement fails for any reason a warning log will be emitted but the operation of this component will not be stopped.\n", "examples": [ "\nCREATE TABLE IF NOT EXISTS some_table (\n foo varchar(50) not null,\n bar integer,\n baz varchar(50),\n primary key (foo)\n) WITHOUT ROWID;\n" ] }, "init_verify_conn": { "type": "boolean", "description": "Whether to verify the database connection on startup by performing a simple ping, by default this is disabled. Default: false" }, "max_in_flight": { "type": "integer", "description": "The maximum number of inserts to run in parallel. Default: 64" }, "prefix": { "type": "string", "description": "An optional prefix to prepend to the insert query (before INSERT)." }, "region": { "type": "string", "description": "The AWS region to target. Default: " }, "secret_name": { "type": "string", "description": "An optional field that can be used to get the Username + Password from AWS Secrets Manager. This will overwrite the Username + Password in the DSN with the values from the Secret only if the driver is set to `postgres`." }, "suffix": { "type": "string", "description": "An optional suffix to append to the insert query.", "examples": [ "ON CONFLICT (name) DO NOTHING" ] }, "table": { "type": "string", "description": "The table to insert to.", "examples": [ "foo" ] } } }, "sql_raw": { "type": "object", "properties": { "args_mapping": { "type": "string", "description": "An optional [Bloblang mapping](/docs/guides/bloblang/about) which should evaluate to an array of values matching in size to the number of placeholder arguments in the field `query`.", "examples": [ "root = [ this.cat.meow, this.doc.woofs[0] ]", "root = [ metadata(\"user.id\").string() ]" ] }, "azure": { "type": "object", "description": "Optional Fields that can be set to use Azure based authentication for Azure Postgres SQL", "properties": { "entra_enabled": { "type": "boolean", "description": "An optional field used to generate an entra token to connect to 'Azure Database for PostgreSQL flexible server', This will create a new connection string with the host, user and database from the DSN field - you may need to URL encode the dsn! The [Default Azure Credential Chain](https://learn.microsoft.com/en-gb/azure/developer/go/sdk/authentication/authentication-overview#defaultazurecredential) is used from the Azure SDK. Default: false" }, "token_request_options": { "type": "object", "properties": { "claims": { "type": "string", "description": "Set additional claims for the token. Default: " }, "enable_cae": { "type": "boolean", "description": "Indicates whether to enable Continuous Access Evaluation (CAE) for the requested token Default: false" }, "scopes": { "type": "array", "description": "Scopes contains the list of permission scopes required for the token. Default: [https://ossrdbms-aad.database.windows.net/.default]", "items": { "type": "string" } }, "tenant_id": { "type": "string", "description": "tenant_id identifies the tenant from which to request the token. azure credentials authenticate in their configured default tenants when this field isn't set. Default: " } } } } }, "batching": { "type": "object", "description": "\nAllows you to configure a [batching policy](/docs/configuration/batching).", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] }, "conn_max_idle": { "type": "integer", "description": "An optional maximum number of connections in the idle connection pool. If conn_max_open is greater than 0 but less than the new conn_max_idle, then the new conn_max_idle will be reduced to match the conn_max_open limit. If `value \u003c= 0`, no idle connections are retained. The default max idle connections is currently 2. This may change in a future release. Default: 2" }, "conn_max_idle_time": { "type": "string", "description": "An optional maximum amount of time a connection may be idle. Expired connections may be closed lazily before reuse. If `value \u003c= 0`, connections are not closed due to a connections idle time." }, "conn_max_life_time": { "type": "string", "description": "An optional maximum amount of time a connection may be reused. Expired connections may be closed lazily before reuse. If `value \u003c= 0`, connections are not closed due to a connections age." }, "conn_max_open": { "type": "integer", "description": "An optional maximum number of open connections to the database. If conn_max_idle is greater than 0 and the new conn_max_open is less than conn_max_idle, then conn_max_idle will be reduced to match the new conn_max_open limit. If `value \u003c= 0`, then there is no limit on the number of open connections. The default is 0 (unlimited)." }, "credentials": { "type": "object", "description": "Optional manual configuration of AWS credentials to use. More information can be found [in this document](/docs/guides/cloud/aws).", "properties": { "expiry_window": { "type": "string", "description": "Allow the credentials to trigger refreshing prior to the credentials actually expiring. This is beneficial so race conditions with expiring credentials do not cause requests to fail. For example '10s' would refresh credentials ten seconds before expiration. Setting to a duration of `0` disables the expiry window. Default: " }, "from_ec2_role": { "type": "boolean", "description": "Use the credentials of a host EC2 machine configured to assume [an IAM role associated with the instance](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html). Default: false" }, "id": { "type": "string", "description": "The ID of credentials to use. Default: " }, "profile": { "type": "string", "description": "A profile from `~/.aws/credentials` to use. Default: " }, "role": { "type": "string", "description": "A role ARN to assume. Default: " }, "role_external_id": { "type": "string", "description": "An external ID to provide when assuming a role. Default: " }, "secret": { "type": "string", "description": "The secret for the credentials being used. Default: " }, "token": { "type": "string", "description": "The token for the credentials being used, required when using short term credentials. Default: " } } }, "driver": { "type": "string", "description": "A database [driver](#drivers) to use." }, "dsn": { "type": "string", "description": "A Data Source Name to identify the target database.\n\n#### Drivers\n\nThe following is a list of supported drivers, their placeholder style, and their respective DSN formats:\n\n| Driver | Data Source Name Format |\n|---|---|\n| `clickhouse` | [`clickhouse://[username[:password]@][netloc][:port]/dbname[?param1=value1\u0026...\u0026paramN=valueN]`](https://github.com/ClickHouse/clickhouse-go#dsn) |\n| `mysql` | `[username[:password]@][protocol[(address)]]/dbname[?param1=value1\u0026...\u0026paramN=valueN]` |\n| `postgres` | `postgres://[user[:password]@][netloc][:port][/dbname][?param1=value1\u0026...]` |\n| `mssql` | `sqlserver://[user[:password]@][netloc][:port][?database=dbname\u0026param1=value1\u0026...]` |\n| `sqlite` | `file:/path/to/filename.db[?param\u0026=value1\u0026...]` |\n| `oracle` | `oracle://[username[:password]@][netloc][:port]/service_name?server=server2\u0026server=server3` |\n| `snowflake` | `username[:password]@account_identifier/dbname/schemaname[?param1=value\u0026...\u0026paramN=valueN]` |\n| `spanner` | `projects/[project]/instances/[instance]/databases/dbname` |\n| `trino` | [`http[s]://user[:pass]@host[:port][?parameters]`](https://github.com/trinodb/trino-go-client#dsn-data-source-name) |\n| `gocosmos` | [`AccountEndpoint=\u003ccosmosdb-endpoint\u003e;AccountKey=\u003ccosmosdb-account-key\u003e[;TimeoutMs=\u003ctimeout-in-ms\u003e][;Version=\u003ccosmosdb-api-version\u003e][;DefaultDb/Db=\u003cdb-name\u003e][;AutoId=\u003ctrue/false\u003e][;InsecureSkipVerify=\u003ctrue/false\u003e]`](https://pkg.go.dev/github.com/microsoft/gocosmos#readme-example-usage) |\n\nPlease note that the `postgres` driver enforces SSL by default, you can override this with the parameter `sslmode=disable` if required.\n\nThe `snowflake` driver supports multiple DSN formats. Please consult [the docs](https://pkg.go.dev/github.com/snowflakedb/gosnowflake#hdr-Connection_String) for more details. For [key pair authentication](https://docs.snowflake.com/en/user-guide/key-pair-auth.html#configuring-key-pair-authentication), the DSN has the following format: `\u003csnowflake_user\u003e@\u003csnowflake_account\u003e/\u003cdb_name\u003e/\u003cschema_name\u003e?warehouse=\u003cwarehouse\u003e\u0026role=\u003crole\u003e\u0026authenticator=snowflake_jwt\u0026privateKey=\u003cbase64_url_encoded_private_key\u003e`, where the value for the `privateKey` parameter can be constructed from an unencrypted RSA private key file `rsa_key.p8` using `openssl enc -d -base64 -in rsa_key.p8 | basenc --base64url -w0` (you can use `gbasenc` insted of `basenc` on OSX if you install `coreutils` via Homebrew). If you have a password-encrypted private key, you can decrypt it using `openssl pkcs8 -in rsa_key_encrypted.p8 -out rsa_key.p8`. Also, make sure fields such as the username are URL-encoded.\n\nThe [`gocosmos`](https://pkg.go.dev/github.com/microsoft/gocosmos) driver is still experimental, but it has support for [hierarchical partition keys](https://learn.microsoft.com/en-us/azure/cosmos-db/hierarchical-partition-keys) as well as [cross-partition queries](https://learn.microsoft.com/en-us/azure/cosmos-db/nosql/how-to-query-container#cross-partition-query). Please refer to the [SQL notes](https://github.com/microsoft/gocosmos/blob/main/SQL.md) for details.", "examples": [ "clickhouse://username:password@host1:9000,host2:9000/database?dial_timeout=200ms\u0026max_execution_time=60", "foouser:foopassword@tcp(localhost:3306)/foodb", "postgres://foouser:foopass@localhost:5432/foodb?sslmode=disable", "oracle://foouser:foopass@localhost:1521/service_name" ] }, "endpoint": { "type": "string", "description": "Allows you to specify a custom endpoint for the AWS API. Default: " }, "iam_enabled": { "type": "boolean", "description": "An optional field used to generate an IAM authentication token to connect to an Amazon Relational Database (RDS) DB instance. This will overwrite the Password in the DSN with the generated token only if the drivers are `mysql` or `postgres`. Default: false" }, "init_files": { "type": "array", "description": "\nAn optional list of file paths containing SQL statements to execute immediately upon the first connection to the target database. This is a useful way to initialise tables before processing data. Glob patterns are supported, including super globs (double star).\n\nCare should be taken to ensure that the statements are idempotent, and therefore would not cause issues when run multiple times after service restarts. If both `init_statement` and `init_files` are specified the `init_statement` is executed _after_ the `init_files`.\n\nIf a statement fails for any reason a warning log will be emitted but the operation of this component will not be stopped.\n", "items": { "type": "string" }, "examples": [ [ "./init/*.sql" ], [ "./foo.sql", "./bar.sql" ] ] }, "init_statement": { "type": "string", "description": "\nAn optional SQL statement to execute immediately upon the first connection to the target database. This is a useful way to initialise tables before processing data. Care should be taken to ensure that the statement is idempotent, and therefore would not cause issues when run multiple times after service restarts.\n\nIf both `init_statement` and `init_files` are specified the `init_statement` is executed _after_ the `init_files`.\n\nIf the statement fails for any reason a warning log will be emitted but the operation of this component will not be stopped.\n", "examples": [ "\nCREATE TABLE IF NOT EXISTS some_table (\n foo varchar(50) not null,\n bar integer,\n baz varchar(50),\n primary key (foo)\n) WITHOUT ROWID;\n" ] }, "init_verify_conn": { "type": "boolean", "description": "Whether to verify the database connection on startup by performing a simple ping, by default this is disabled. Default: false" }, "max_in_flight": { "type": "integer", "description": "The maximum number of inserts to run in parallel. Default: 64" }, "query": { "type": "string", "description": "The query to execute. The style of placeholder to use depends on the driver, some drivers require question marks (`?`) whereas others expect incrementing dollar signs (`$1`, `$2`, and so on) or colons (`:1`, `:2` and so on). The style to use is outlined in this table:\n\n| Driver | Placeholder Style |\n|---|---|\n| `clickhouse` | Dollar sign |\n| `mysql` | Question mark |\n| `postgres` | Dollar sign |\n| `mssql` | Question mark |\n| `sqlite` | Question mark |\n| `oracle` | Colon |\n| `snowflake` | Question mark |\n| `spanner` | Question mark |\n| `trino` | Question mark |\n| `gocosmos` | Colon |\n", "examples": [ "INSERT INTO footable (foo, bar, baz) VALUES (?, ?, ?);" ] }, "region": { "type": "string", "description": "The AWS region to target. Default: " }, "secret_name": { "type": "string", "description": "An optional field that can be used to get the Username + Password from AWS Secrets Manager. This will overwrite the Username + Password in the DSN with the values from the Secret only if the driver is set to `postgres`." }, "unsafe_dynamic_query": { "type": "boolean", "description": "Whether to enable [interpolation functions](/docs/configuration/interpolation/#bloblang-queries) in the query. Great care should be made to ensure your queries are defended against injection attacks. Default: false" } } }, "stdout": { "type": "object", "properties": { "codec": { "type": "string", "description": "The way in which the bytes of messages should be written out into the output data stream. It's possible to write lines using a custom delimiter with the `delim:x` codec, where x is the character sequence custom delimiter. Default: lines", "examples": [ "lines", "delim:\t", "delim:foobar" ] } } }, "subprocess": { "type": "object", "properties": { "args": { "type": "array", "description": "A list of arguments to provide the command. Default: []", "items": { "type": "string" } }, "codec": { "type": "string", "description": "The way in which messages should be written to the subprocess. Default: lines" }, "name": { "type": "string", "description": "The command to execute as a subprocess." } } }, "switch": { "type": "object", "properties": { "cases": { "type": "array", "description": "A list of switch cases, outlining outputs that can be routed to.", "items": { "type": "object" }, "examples": [ [ { "check": "this.urls.contains(\"https://warpstreamlabs.github.io/bento\")", "continue": true, "output": { "cache": { "key": "${!json(\"id\")}", "target": "foo" } } }, { "output": { "s3": { "bucket": "bar", "path": "${!json(\"id\")}" } } } ] ] }, "retry_until_success": { "type": "boolean", "description": "\nIf a selected output fails to send a message this field determines whether it is reattempted indefinitely. If set to false the error is instead propagated back to the input level.\n\nIf a message can be routed to \u003e1 outputs it is usually best to set this to true in order to avoid duplicate messages being routed to an output. Default: false" }, "strict_mode": { "type": "boolean", "description": "This field determines whether an error should be reported if no condition is met. If set to true, an error is propagated back to the input level. The default behavior is false, which will drop the message. Default: false" } } }, "sync_response": { "type": "object", "description": " Default: map[]" }, "websocket": { "type": "object", "properties": { "basic_auth": { "type": "object", "description": "Allows you to specify basic authentication.", "properties": { "enabled": { "type": "boolean", "description": "Whether to use basic authentication in requests. Default: false" }, "password": { "type": "string", "description": "A password to authenticate with. Default: " }, "username": { "type": "string", "description": "A username to authenticate as. Default: " } } }, "jwt": { "type": "object", "description": "BETA: Allows you to specify JWT authentication.", "properties": { "claims": { "type": "object", "description": "A value used to identify the claims that issued the JWT. Default: map[]" }, "enabled": { "type": "boolean", "description": "Whether to use JWT authentication in requests. Default: false" }, "headers": { "type": "object", "description": "Add optional key/value headers to the JWT. Default: map[]" }, "private_key_file": { "type": "string", "description": "A file with the PEM encoded via PKCS1 or PKCS8 as private key. Default: " }, "signing_method": { "type": "string", "description": "A method used to sign the token such as RS256, RS384, RS512 or EdDSA. Default: " } } }, "oauth": { "type": "object", "description": "Allows you to specify open authentication via OAuth version 1.", "properties": { "access_token": { "type": "string", "description": "A value used to gain access to the protected resources on behalf of the user. Default: " }, "access_token_secret": { "type": "string", "description": "A secret provided in order to establish ownership of a given access token. Default: " }, "consumer_key": { "type": "string", "description": "A value used to identify the client to the service provider. Default: " }, "consumer_secret": { "type": "string", "description": "A secret used to establish ownership of the consumer key. Default: " }, "enabled": { "type": "boolean", "description": "Whether to use OAuth version 1 in requests. Default: false" } } }, "proxy_url": { "type": "string", "description": "An optional HTTP proxy URL." }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "url": { "type": "string", "description": "The URL to connect to." } } }, "zmq4n": { "type": "object", "properties": { "bind": { "type": "boolean", "description": "Whether to bind to the specified URLs (otherwise they are connected to). Default: true" }, "dial_max_retries": { "type": "integer", "description": "The maximum number of dial retries (-1 for infinite retries). Default: 10" }, "dial_retry_delay": { "type": "string", "description": "The time to wait between failed dial attempts. Default: 250ms" }, "dial_timeout": { "type": "string", "description": "The maximum time to wait for a dial to complete. Default: 5m" }, "high_water_mark": { "type": "integer", "description": "The message high water mark to use. (experimental i go-zeromq) Default: 0" }, "poll_timeout": { "type": "string", "description": "The poll timeout to use. Default: 5s" }, "socket_auto_reconnect": { "type": "boolean", "description": "Whether to automatically attempt internal reconnection on connection loss.\n:::warning Important\nSince this is an internal retry, the zmq4n component will silently attempt reconnection until failure. This means that while retrying, no metric will indicate the component is in a retrying state until attempts have been exhausted.\n::: Default: true" }, "socket_type": { "type": "string", "description": "The socket type to connect as." }, "urls": { "type": "array", "description": "A list of URLs to connect to. If an item of the list contains commas it will be expanded into multiple URLs.", "items": { "type": "string" }, "examples": [ [ "tcp://localhost:5556" ] ] } } } } }, "output_resources": { "type": "array", "description": "A list of output resources, each must have a unique label. Default: []", "properties": { "amqp_0_9": { "type": "object", "properties": { "app_id": { "type": "string", "description": "Set the application ID of each message with a dynamic interpolated expression. Default: " }, "content_encoding": { "type": "string", "description": "The content encoding attribute to set for each message. Default: " }, "content_type": { "type": "string", "description": "The content type attribute to set for each message. Default: application/octet-stream" }, "correlation_id": { "type": "string", "description": "Set the correlation ID of each message with a dynamic interpolated expression. Default: " }, "exchange": { "type": "string", "description": "An AMQP exchange to publish to." }, "exchange_declare": { "type": "object", "description": "Optionally declare the target exchange (passive).", "properties": { "durable": { "type": "boolean", "description": "Whether the exchange should be durable. Default: true" }, "enabled": { "type": "boolean", "description": "Whether to declare the exchange. Default: false" }, "type": { "type": "string", "description": "The type of the exchange. Default: direct" } } }, "expiration": { "type": "string", "description": "Set the per-message TTL Default: " }, "immediate": { "type": "boolean", "description": "Whether to set the immediate flag on published messages. When set if there are no ready consumers of a queue then the message is dropped instead of waiting. Default: false" }, "key": { "type": "string", "description": "The binding key to set for each message. Default: " }, "mandatory": { "type": "boolean", "description": "Whether to set the mandatory flag on published messages. When set if a published message is routed to zero queues it is returned. Default: false" }, "max_in_flight": { "type": "integer", "description": "The maximum number of messages to have in flight at a given time. Increase this to improve throughput. Default: 64" }, "message_id": { "type": "string", "description": "Set the message ID of each message with a dynamic interpolated expression. Default: " }, "metadata": { "type": "object", "description": "Specify criteria for which metadata values are attached to messages as headers.", "properties": { "exclude_prefixes": { "type": "array", "description": "Provide a list of explicit metadata key prefixes to be excluded when adding metadata to sent messages. Default: []", "items": { "type": "string" } } } }, "persistent": { "type": "boolean", "description": "Whether message delivery should be persistent (transient by default). Default: false" }, "priority": { "type": "string", "description": "Set the priority of each message with a dynamic interpolated expression. Default: ", "examples": [ "0", "${! metadata(\"amqp_priority\") }", "${! json(\"doc.priority\") }" ] }, "reply_to": { "type": "string", "description": "Carries response queue name - set with a dynamic interpolated expression. Default: " }, "timeout": { "type": "string", "description": "The maximum period to wait before abandoning it and reattempting. If not set, wait indefinitely. Default: " }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "type": { "type": "string", "description": "The type property to set for each message. Default: " }, "urls": { "type": "array", "description": "A list of URLs to connect to. The first URL to successfully establish a connection will be used until the connection is closed. If an item of the list contains commas it will be expanded into multiple URLs.", "items": { "type": "string" }, "examples": [ [ "amqp://guest:guest@127.0.0.1:5672/" ], [ "amqp://127.0.0.1:5672/,amqp://127.0.0.2:5672/" ], [ "amqp://127.0.0.1:5672/", "amqp://127.0.0.2:5672/" ] ] }, "user_id": { "type": "string", "description": "Set the user ID to the name of the publisher. If this property is set by a publisher, its value must be equal to the name of the user used to open the connection. Default: " } } }, "amqp_1": { "type": "object", "properties": { "application_properties_map": { "type": "string", "description": "An optional Bloblang mapping that can be defined in order to set the `application-properties` on output messages." }, "max_in_flight": { "type": "integer", "description": "The maximum number of messages to have in flight at a given time. Increase this to improve throughput. Default: 64" }, "metadata": { "type": "object", "description": "Specify criteria for which metadata values are attached to messages as headers.", "properties": { "exclude_prefixes": { "type": "array", "description": "Provide a list of explicit metadata key prefixes to be excluded when adding metadata to sent messages. Default: []", "items": { "type": "string" } } } }, "sasl": { "type": "object", "description": "Enables SASL authentication.", "properties": { "mechanism": { "type": "string", "description": "The SASL authentication mechanism to use. Default: none" }, "password": { "type": "string", "description": "A SASL plain text password. It is recommended that you use environment variables to populate this field. Default: ", "examples": [ "${PASSWORD}" ] }, "user": { "type": "string", "description": "A SASL plain text username. It is recommended that you use environment variables to populate this field. Default: ", "examples": [ "${USER}" ] } } }, "target_address": { "type": "string", "description": "The target address to write to.", "examples": [ "/foo", "queue:/bar", "topic:/baz" ] }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "url": { "type": "string", "description": "A URL to connect to.", "examples": [ "amqp://localhost:5672/", "amqps://guest:guest@localhost:5672/" ] }, "urls": { "type": "array", "description": "A list of URLs to connect to. The first URL to successfully establish a connection will be used until the connection is closed. If an item of the list contains commas it will be expanded into multiple URLs.", "items": { "type": "string" }, "examples": [ [ "amqp://guest:guest@127.0.0.1:5672/" ], [ "amqp://127.0.0.1:5672/,amqp://127.0.0.2:5672/" ], [ "amqp://127.0.0.1:5672/", "amqp://127.0.0.2:5672/" ] ] } } }, "aws_dynamodb": { "type": "object", "properties": { "backoff": { "type": "object", "description": "Control time intervals between retry attempts.", "properties": { "initial_interval": { "type": "string", "description": "The initial period to wait between retry attempts. Default: 1s" }, "max_elapsed_time": { "type": "string", "description": "The maximum period to wait before retry attempts are abandoned. If zero then no limit is used. Default: 30s" }, "max_interval": { "type": "string", "description": "The maximum period to wait between retry attempts. Default: 5s" } } }, "batching": { "type": "object", "description": "\nAllows you to configure a [batching policy](/docs/configuration/batching).", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] }, "credentials": { "type": "object", "description": "Optional manual configuration of AWS credentials to use. More information can be found [in this document](/docs/guides/cloud/aws).", "properties": { "expiry_window": { "type": "string", "description": "Allow the credentials to trigger refreshing prior to the credentials actually expiring. This is beneficial so race conditions with expiring credentials do not cause requests to fail. For example '10s' would refresh credentials ten seconds before expiration. Setting to a duration of `0` disables the expiry window. Default: " }, "from_ec2_role": { "type": "boolean", "description": "Use the credentials of a host EC2 machine configured to assume [an IAM role associated with the instance](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html). Default: false" }, "id": { "type": "string", "description": "The ID of credentials to use. Default: " }, "profile": { "type": "string", "description": "A profile from `~/.aws/credentials` to use. Default: " }, "role": { "type": "string", "description": "A role ARN to assume. Default: " }, "role_external_id": { "type": "string", "description": "An external ID to provide when assuming a role. Default: " }, "secret": { "type": "string", "description": "The secret for the credentials being used. Default: " }, "token": { "type": "string", "description": "The token for the credentials being used, required when using short term credentials. Default: " } } }, "delete": { "type": "object", "description": "Optional config fields that enable creating Delete requests from messages. If the bloblang mapping provided in `delete.condition` resolves to true, a delete request for the corresponding partition key will be made.", "properties": { "condition": { "type": "string", "description": "A bloblang mapping that should return a bool, that will determine if the message will be used to create a Delete rather than Put Default: ", "examples": [ "root = this.isDelete == \"true\"" ] }, "partition_key": { "type": "string", "description": "The partition key for DeleteItem requests. Required when `delete.condition` is true. The value of the key will be resolved from either `string_columns or json_map_columns` Default: " }, "sort_key": { "type": "string", "description": "The sort key for DeleteItem requests. The value of the key will be resolved from either `string_columns or json_map_columns` Default: " } } }, "endpoint": { "type": "string", "description": "Allows you to specify a custom endpoint for the AWS API. Default: " }, "json_map_columns": { "type": "object", "description": "A map of column keys to [field paths](/docs/configuration/field_paths) pointing to value data within messages. Default: map[]", "examples": [ { "user": "path.to.user", "whole_document": "." }, { "": "." } ] }, "max_in_flight": { "type": "integer", "description": "The maximum number of messages to have in flight at a given time. Increase this to improve throughput. Default: 64" }, "max_retries": { "type": "integer", "description": "The maximum number of retries before giving up on the request. If set to zero there is no discrete limit. Default: 3" }, "region": { "type": "string", "description": "The AWS region to target. Default: " }, "string_columns": { "type": "object", "description": "A map of column keys to string values to store. Default: map[]", "examples": [ { "full_content": "${!content()}", "id": "${!json(\"id\")}", "title": "${!json(\"body.title\")}", "topic": "${!meta(\"kafka_topic\")}" } ] }, "table": { "type": "string", "description": "The table to store messages in." }, "ttl": { "type": "string", "description": "An optional TTL to set for items, calculated from the moment the message is sent. Default: " }, "ttl_key": { "type": "string", "description": "The column key to place the TTL value within. Default: " } } }, "aws_kinesis": { "type": "object", "properties": { "backoff": { "type": "object", "description": "Control time intervals between retry attempts.", "properties": { "initial_interval": { "type": "string", "description": "The initial period to wait between retry attempts. Default: 1s" }, "max_elapsed_time": { "type": "string", "description": "The maximum period to wait before retry attempts are abandoned. If zero then no limit is used. Default: 30s" }, "max_interval": { "type": "string", "description": "The maximum period to wait between retry attempts. Default: 5s" } } }, "batching": { "type": "object", "description": "\nAllows you to configure a [batching policy](/docs/configuration/batching).", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] }, "credentials": { "type": "object", "description": "Optional manual configuration of AWS credentials to use. More information can be found [in this document](/docs/guides/cloud/aws).", "properties": { "expiry_window": { "type": "string", "description": "Allow the credentials to trigger refreshing prior to the credentials actually expiring. This is beneficial so race conditions with expiring credentials do not cause requests to fail. For example '10s' would refresh credentials ten seconds before expiration. Setting to a duration of `0` disables the expiry window. Default: " }, "from_ec2_role": { "type": "boolean", "description": "Use the credentials of a host EC2 machine configured to assume [an IAM role associated with the instance](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html). Default: false" }, "id": { "type": "string", "description": "The ID of credentials to use. Default: " }, "profile": { "type": "string", "description": "A profile from `~/.aws/credentials` to use. Default: " }, "role": { "type": "string", "description": "A role ARN to assume. Default: " }, "role_external_id": { "type": "string", "description": "An external ID to provide when assuming a role. Default: " }, "secret": { "type": "string", "description": "The secret for the credentials being used. Default: " }, "token": { "type": "string", "description": "The token for the credentials being used, required when using short term credentials. Default: " } } }, "endpoint": { "type": "string", "description": "Allows you to specify a custom endpoint for the AWS API. Default: " }, "hash_key": { "type": "string", "description": "A optional hash key for partitioning messages." }, "max_in_flight": { "type": "integer", "description": "The maximum number of parallel message batches to have in flight at any given time. Default: 64" }, "max_retries": { "type": "integer", "description": "The maximum number of retries before giving up on the request. If set to zero there is no discrete limit. Default: 0" }, "partition_key": { "type": "string", "description": "A required key for partitioning messages." }, "region": { "type": "string", "description": "The AWS region to target. Default: " }, "stream": { "type": "string", "description": "The stream to publish messages to. Streams can either be specified by their name or full ARN.", "examples": [ "foo", "arn:aws:kinesis:*:111122223333:stream/my-stream" ] } } }, "aws_kinesis_firehose": { "type": "object", "properties": { "backoff": { "type": "object", "description": "Control time intervals between retry attempts.", "properties": { "initial_interval": { "type": "string", "description": "The initial period to wait between retry attempts. Default: 1s" }, "max_elapsed_time": { "type": "string", "description": "The maximum period to wait before retry attempts are abandoned. If zero then no limit is used. Default: 30s" }, "max_interval": { "type": "string", "description": "The maximum period to wait between retry attempts. Default: 5s" } } }, "batching": { "type": "object", "description": "\nAllows you to configure a [batching policy](/docs/configuration/batching).", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] }, "credentials": { "type": "object", "description": "Optional manual configuration of AWS credentials to use. More information can be found [in this document](/docs/guides/cloud/aws).", "properties": { "expiry_window": { "type": "string", "description": "Allow the credentials to trigger refreshing prior to the credentials actually expiring. This is beneficial so race conditions with expiring credentials do not cause requests to fail. For example '10s' would refresh credentials ten seconds before expiration. Setting to a duration of `0` disables the expiry window. Default: " }, "from_ec2_role": { "type": "boolean", "description": "Use the credentials of a host EC2 machine configured to assume [an IAM role associated with the instance](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html). Default: false" }, "id": { "type": "string", "description": "The ID of credentials to use. Default: " }, "profile": { "type": "string", "description": "A profile from `~/.aws/credentials` to use. Default: " }, "role": { "type": "string", "description": "A role ARN to assume. Default: " }, "role_external_id": { "type": "string", "description": "An external ID to provide when assuming a role. Default: " }, "secret": { "type": "string", "description": "The secret for the credentials being used. Default: " }, "token": { "type": "string", "description": "The token for the credentials being used, required when using short term credentials. Default: " } } }, "endpoint": { "type": "string", "description": "Allows you to specify a custom endpoint for the AWS API. Default: " }, "max_in_flight": { "type": "integer", "description": "The maximum number of messages to have in flight at a given time. Increase this to improve throughput. Default: 64" }, "max_retries": { "type": "integer", "description": "The maximum number of retries before giving up on the request. If set to zero there is no discrete limit. Default: 0" }, "region": { "type": "string", "description": "The AWS region to target. Default: " }, "stream": { "type": "string", "description": "The stream to publish messages to." } } }, "aws_s3": { "type": "object", "properties": { "batching": { "type": "object", "description": "\nAllows you to configure a [batching policy](/docs/configuration/batching).", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] }, "bucket": { "type": "string", "description": "The bucket to upload messages to." }, "cache_control": { "type": "string", "description": "The cache control to set for each object. Default: " }, "content_disposition": { "type": "string", "description": "The content disposition to set for each object. Default: " }, "content_encoding": { "type": "string", "description": "An optional content encoding to set for each object. Default: " }, "content_language": { "type": "string", "description": "The content language to set for each object. Default: " }, "content_type": { "type": "string", "description": "The content type to set for each object. Default: application/octet-stream" }, "credentials": { "type": "object", "description": "Optional manual configuration of AWS credentials to use. More information can be found [in this document](/docs/guides/cloud/aws).", "properties": { "expiry_window": { "type": "string", "description": "Allow the credentials to trigger refreshing prior to the credentials actually expiring. This is beneficial so race conditions with expiring credentials do not cause requests to fail. For example '10s' would refresh credentials ten seconds before expiration. Setting to a duration of `0` disables the expiry window. Default: " }, "from_ec2_role": { "type": "boolean", "description": "Use the credentials of a host EC2 machine configured to assume [an IAM role associated with the instance](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html). Default: false" }, "id": { "type": "string", "description": "The ID of credentials to use. Default: " }, "profile": { "type": "string", "description": "A profile from `~/.aws/credentials` to use. Default: " }, "role": { "type": "string", "description": "A role ARN to assume. Default: " }, "role_external_id": { "type": "string", "description": "An external ID to provide when assuming a role. Default: " }, "secret": { "type": "string", "description": "The secret for the credentials being used. Default: " }, "token": { "type": "string", "description": "The token for the credentials being used, required when using short term credentials. Default: " } } }, "endpoint": { "type": "string", "description": "Allows you to specify a custom endpoint for the AWS API. Default: " }, "force_path_style_urls": { "type": "boolean", "description": "Forces the client API to use path style URLs, which helps when connecting to custom endpoints. Default: false" }, "kms_key_id": { "type": "string", "description": "An optional server side encryption key. Default: " }, "max_in_flight": { "type": "integer", "description": "The maximum number of messages to have in flight at a given time. Increase this to improve throughput. Default: 64" }, "metadata": { "type": "object", "description": "Specify criteria for which metadata values are attached to objects as headers.", "properties": { "exclude_prefixes": { "type": "array", "description": "Provide a list of explicit metadata key prefixes to be excluded when adding metadata to sent messages. Default: []", "items": { "type": "string" } } } }, "path": { "type": "string", "description": "The path of each message to upload. Default: ${!count(\"files\")}-${!timestamp_unix_nano()}.txt", "examples": [ "${!count(\"files\")}-${!timestamp_unix_nano()}.txt", "${!metadata(\"kafka_key\")}.json", "${!json(\"doc.namespace\")}/${!json(\"doc.id\")}.json" ] }, "region": { "type": "string", "description": "The AWS region to target. Default: " }, "server_side_encryption": { "type": "string", "description": "An optional server side encryption algorithm. Default: " }, "storage_class": { "type": "string", "description": "The storage class to set for each object. Default: STANDARD" }, "tags": { "type": "object", "description": "Key/value pairs to store with the object as tags. Default: map[]", "examples": [ { "Key1": "Value1", "Timestamp": "${!metadata(\"Timestamp\")}" } ] }, "timeout": { "type": "string", "description": "The maximum period to wait on an upload before abandoning it and reattempting. Default: 5s" }, "website_redirect_location": { "type": "string", "description": "The website redirect location to set for each object. Default: " } } }, "aws_sns": { "type": "object", "properties": { "credentials": { "type": "object", "description": "Optional manual configuration of AWS credentials to use. More information can be found [in this document](/docs/guides/cloud/aws).", "properties": { "expiry_window": { "type": "string", "description": "Allow the credentials to trigger refreshing prior to the credentials actually expiring. This is beneficial so race conditions with expiring credentials do not cause requests to fail. For example '10s' would refresh credentials ten seconds before expiration. Setting to a duration of `0` disables the expiry window. Default: " }, "from_ec2_role": { "type": "boolean", "description": "Use the credentials of a host EC2 machine configured to assume [an IAM role associated with the instance](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html). Default: false" }, "id": { "type": "string", "description": "The ID of credentials to use. Default: " }, "profile": { "type": "string", "description": "A profile from `~/.aws/credentials` to use. Default: " }, "role": { "type": "string", "description": "A role ARN to assume. Default: " }, "role_external_id": { "type": "string", "description": "An external ID to provide when assuming a role. Default: " }, "secret": { "type": "string", "description": "The secret for the credentials being used. Default: " }, "token": { "type": "string", "description": "The token for the credentials being used, required when using short term credentials. Default: " } } }, "endpoint": { "type": "string", "description": "Allows you to specify a custom endpoint for the AWS API. Default: " }, "max_in_flight": { "type": "integer", "description": "The maximum number of messages to have in flight at a given time. Increase this to improve throughput. Default: 64" }, "message_deduplication_id": { "type": "string", "description": "An optional deduplication ID to set for messages." }, "message_group_id": { "type": "string", "description": "An optional group ID to set for messages." }, "metadata": { "type": "object", "description": "Specify criteria for which metadata values are sent as headers.", "properties": { "exclude_prefixes": { "type": "array", "description": "Provide a list of explicit metadata key prefixes to be excluded when adding metadata to sent messages. Default: []", "items": { "type": "string" } } } }, "region": { "type": "string", "description": "The AWS region to target. Default: " }, "timeout": { "type": "string", "description": "The maximum period to wait on an upload before abandoning it and reattempting. Default: 5s" }, "topic_arn": { "type": "string", "description": "The topic to publish to." } } }, "aws_sqs": { "type": "object", "properties": { "backoff": { "type": "object", "description": "Control time intervals between retry attempts.", "properties": { "initial_interval": { "type": "string", "description": "The initial period to wait between retry attempts. Default: 1s" }, "max_elapsed_time": { "type": "string", "description": "The maximum period to wait before retry attempts are abandoned. If zero then no limit is used. Default: 30s" }, "max_interval": { "type": "string", "description": "The maximum period to wait between retry attempts. Default: 5s" } } }, "batching": { "type": "object", "description": "\nAllows you to configure a [batching policy](/docs/configuration/batching).", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] }, "credentials": { "type": "object", "description": "Optional manual configuration of AWS credentials to use. More information can be found [in this document](/docs/guides/cloud/aws).", "properties": { "expiry_window": { "type": "string", "description": "Allow the credentials to trigger refreshing prior to the credentials actually expiring. This is beneficial so race conditions with expiring credentials do not cause requests to fail. For example '10s' would refresh credentials ten seconds before expiration. Setting to a duration of `0` disables the expiry window. Default: " }, "from_ec2_role": { "type": "boolean", "description": "Use the credentials of a host EC2 machine configured to assume [an IAM role associated with the instance](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html). Default: false" }, "id": { "type": "string", "description": "The ID of credentials to use. Default: " }, "profile": { "type": "string", "description": "A profile from `~/.aws/credentials` to use. Default: " }, "role": { "type": "string", "description": "A role ARN to assume. Default: " }, "role_external_id": { "type": "string", "description": "An external ID to provide when assuming a role. Default: " }, "secret": { "type": "string", "description": "The secret for the credentials being used. Default: " }, "token": { "type": "string", "description": "The token for the credentials being used, required when using short term credentials. Default: " } } }, "delay_seconds": { "type": "string", "description": "An optional delay time in seconds for message. Value between 0 and 900" }, "endpoint": { "type": "string", "description": "Allows you to specify a custom endpoint for the AWS API. Default: " }, "max_in_flight": { "type": "integer", "description": "The maximum number of parallel message batches to have in flight at any given time. Default: 64" }, "max_retries": { "type": "integer", "description": "The maximum number of retries before giving up on the request. If set to zero there is no discrete limit. Default: 0" }, "message_deduplication_id": { "type": "string", "description": "An optional deduplication ID to set for messages." }, "message_group_id": { "type": "string", "description": "An optional group ID to set for messages." }, "metadata": { "type": "object", "description": "Specify criteria for which metadata values are sent as headers.", "properties": { "exclude_prefixes": { "type": "array", "description": "Provide a list of explicit metadata key prefixes to be excluded when adding metadata to sent messages. Default: []", "items": { "type": "string" } } } }, "region": { "type": "string", "description": "The AWS region to target. Default: " }, "url": { "type": "string", "description": "The URL of the target SQS queue." } } }, "azure_blob_storage": { "type": "object", "properties": { "blob_type": { "type": "string", "description": "Block and Append blobs are comprised of blocks, and each blob can support up to 50,000 blocks. The default value is `+\"`BLOCK`\"+`.` Default: BLOCK" }, "container": { "type": "string", "description": "The container for uploading the messages to.", "examples": [ "messages-${!timestamp(\"2006\")}" ] }, "max_in_flight": { "type": "integer", "description": "The maximum number of messages to have in flight at a given time. Increase this to improve throughput. Default: 64" }, "path": { "type": "string", "description": "The path of each message to upload. Default: ${!count(\"files\")}-${!timestamp_unix_nano()}.txt", "examples": [ "${!count(\"files\")}-${!timestamp_unix_nano()}.json", "${!metadata(\"kafka_key\")}.json", "${!json(\"doc.namespace\")}/${!json(\"doc.id\")}.json" ] }, "public_access_level": { "type": "string", "description": "The container's public access level. The default value is `PRIVATE`. Default: PRIVATE" }, "storage_access_key": { "type": "string", "description": "The storage account access key. This field is ignored if `storage_connection_string` is set. Default: " }, "storage_account": { "type": "string", "description": "The storage account to access. This field is ignored if `storage_connection_string` is set. Default: " }, "storage_connection_string": { "type": "string", "description": "A storage account connection string. This field is required if `storage_account` and `storage_access_key` / `storage_sas_token` are not set. Default: " }, "storage_sas_token": { "type": "string", "description": "The storage account SAS token. This field is ignored if `storage_connection_string` or `storage_access_key` are set. Default: " } } }, "azure_cosmosdb": { "type": "object", "properties": { "account_key": { "type": "string", "description": "Account key.", "examples": [ "C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==" ] }, "auto_id": { "type": "boolean", "description": "Automatically set the item `id` field to a random UUID v4. If the `id` field is already set, then it will not be overwritten. Setting this to `false` can improve performance, since the messages will not have to be parsed. Default: true" }, "batching": { "type": "object", "description": "\nAllows you to configure a [batching policy](/docs/configuration/batching).", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] }, "connection_string": { "type": "string", "description": "Connection string.", "examples": [ "AccountEndpoint=https://localhost:8081/;AccountKey=C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==;" ] }, "container": { "type": "string", "description": "Container.", "examples": [ "testcontainer" ] }, "database": { "type": "string", "description": "Database.", "examples": [ "testdb" ] }, "endpoint": { "type": "string", "description": "CosmosDB endpoint.", "examples": [ "https://localhost:8081" ] }, "item_id": { "type": "string", "description": "ID of item to replace or delete. Only used by the Replace and Delete operations", "examples": [ "${! json(\"id\") }" ] }, "max_in_flight": { "type": "integer", "description": "The maximum number of messages to have in flight at a given time. Increase this to improve throughput. Default: 64" }, "operation": { "type": "string", "description": "Operation. Default: Create" }, "partition_keys_map": { "type": "string", "description": "A [Bloblang mapping](/docs/guides/bloblang/about) which should evaluate to a single partition key value or an array of partition key values of type string, integer or boolean. Currently, hierarchical partition keys are not supported so only one value may be provided.", "examples": [ "root = \"blobfish\"", "root = 41", "root = true", "root = null", "root = json(\"blobfish\").depth" ] }, "patch_condition": { "type": "string", "description": "Patch operation condition.", "examples": [ "from c where not is_defined(c.blobfish)" ] }, "patch_operations": { "type": "array", "description": "Patch operations to be performed when `operation: Patch` .", "items": { "type": "object" } } } }, "azure_queue_storage": { "type": "object", "properties": { "batching": { "type": "object", "description": "\nAllows you to configure a [batching policy](/docs/configuration/batching).", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] }, "max_in_flight": { "type": "integer", "description": "The maximum number of parallel message batches to have in flight at any given time. Default: 64" }, "queue_name": { "type": "string", "description": "The name of the target Queue Storage queue." }, "storage_access_key": { "type": "string", "description": "The storage account access key. This field is ignored if `storage_connection_string` is set. Default: " }, "storage_account": { "type": "string", "description": "The storage account to access. This field is ignored if `storage_connection_string` is set. Default: " }, "storage_connection_string": { "type": "string", "description": "A storage account connection string. This field is required if `storage_account` and `storage_access_key` / `storage_sas_token` are not set. Default: " }, "storage_sas_token": { "type": "string", "description": "The storage account SAS token. This field is ignored if `storage_connection_string` or `storage_access_key` are set. Default: " }, "ttl": { "type": "string", "description": "The TTL of each individual message as a duration string. Defaults to 0, meaning no retention period is set Default: ", "examples": [ "60s", "5m", "36h" ] } } }, "azure_table_storage": { "type": "object", "properties": { "batching": { "type": "object", "description": "\nAllows you to configure a [batching policy](/docs/configuration/batching).", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] }, "insert_type": { "type": "string", "description": "Type of insert operation. Valid options are `INSERT`, `INSERT_MERGE` and `INSERT_REPLACE` Default: ", "examples": [ "${! json(\"operation\") }", "${! metadata(\"operation\") }", "INSERT" ] }, "max_in_flight": { "type": "integer", "description": "The maximum number of parallel message batches to have in flight at any given time. Default: 64" }, "partition_key": { "type": "string", "description": "The partition key. Default: ", "examples": [ "${! json(\"date\") }" ] }, "properties": { "type": "object", "description": "A map of properties to store into the table. Default: map[]" }, "row_key": { "type": "string", "description": "The row key. Default: ", "examples": [ "${! json(\"device\")}-${!uuid_v4() }" ] }, "storage_access_key": { "type": "string", "description": "The storage account access key. This field is ignored if `storage_connection_string` is set. Default: " }, "storage_account": { "type": "string", "description": "The storage account to access. This field is ignored if `storage_connection_string` is set. Default: " }, "storage_connection_string": { "type": "string", "description": "A storage account connection string. This field is required if `storage_account` and `storage_access_key` / `storage_sas_token` are not set. Default: " }, "storage_sas_token": { "type": "string", "description": "The storage account SAS token. This field is ignored if `storage_connection_string` or `storage_access_key` are set. Default: " }, "table_name": { "type": "string", "description": "The table to store messages into.", "examples": [ "${! metadata(\"kafka_topic\") }", "${! json(\"table\") }" ] }, "timeout": { "type": "string", "description": "The maximum period to wait on an upload before abandoning it and reattempting. Default: 5s" }, "transaction_type": { "type": "string", "description": "Type of transaction operation. Default: INSERT", "examples": [ "${! json(\"operation\") }", "${! metadata(\"operation\") }", "INSERT" ] } } }, "beanstalkd": { "type": "object", "properties": { "address": { "type": "string", "description": "An address to connect to.", "examples": [ "127.0.0.1:11300" ] }, "max_in_flight": { "type": "integer", "description": "The maximum number of messages to have in flight at a given time. Increase to improve throughput. Default: 64" } } }, "broker": { "type": "object", "properties": { "batching": { "type": "object", "description": "\nAllows you to configure a [batching policy](/docs/configuration/batching).", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] }, "copies": { "type": "integer", "description": "The number of copies of each configured output to spawn. Default: 1" }, "outputs": { "type": "array", "description": "A list of child outputs to broker. Each item can be either a complete inline output configuration or a reference to an output resource.", "items": { "type": "object" } }, "pattern": { "type": "string", "description": "The brokering pattern to use. Default: fan_out" } } }, "cache": { "type": "object", "properties": { "key": { "type": "string", "description": "The key to store messages by, function interpolation should be used in order to derive a unique key for each message. Default: ${!count(\"items\")}-${!timestamp_unix_nano()}", "examples": [ "${!count(\"items\")}-${!timestamp_unix_nano()}", "${!json(\"doc.id\")}", "${!metadata(\"kafka_key\")}" ] }, "max_in_flight": { "type": "integer", "description": "The maximum number of messages to have in flight at a given time. Increase this to improve throughput. Default: 64" }, "target": { "type": "string", "description": "The target cache to store messages in." }, "ttl": { "type": "string", "description": "The TTL of each individual item as a duration string. After this period an item will be eligible for removal during the next compaction. Not all caches support per-key TTLs, and those that do not will fall back to their generally configured TTL setting.", "examples": [ "60s", "5m", "36h" ] } } }, "cassandra": { "type": "object", "properties": { "addresses": { "type": "array", "description": "A list of Cassandra nodes to connect to. Multiple comma separated addresses can be specified on a single line.", "items": { "type": "string" }, "examples": [ [ "localhost:9042" ], [ "foo:9042", "bar:9042" ], [ "foo:9042,bar:9042" ] ] }, "args_mapping": { "type": "string", "description": "A [Bloblang mapping](/docs/guides/bloblang/about) that can be used to provide arguments to Cassandra queries. The result of the query must be an array containing a matching number of elements to the query arguments." }, "backoff": { "type": "object", "description": "Control time intervals between retry attempts.", "properties": { "initial_interval": { "type": "string", "description": "The initial period to wait between retry attempts. Default: 1s" }, "max_interval": { "type": "string", "description": "The maximum period to wait between retry attempts. Default: 5s" } } }, "batching": { "type": "object", "description": "\nAllows you to configure a [batching policy](/docs/configuration/batching).", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] }, "consistency": { "type": "string", "description": "The consistency level to use. Default: QUORUM" }, "disable_initial_host_lookup": { "type": "boolean", "description": "If enabled the driver will not attempt to get host info from the system.peers table. This can speed up queries but will mean that data_centre, rack and token information will not be available. Default: false" }, "logged_batch": { "type": "boolean", "description": "If enabled the driver will perform a logged batch. Disabling this prompts unlogged batches to be used instead, which are less efficient but necessary for alternative storages that do not support logged batches. Default: true" }, "max_in_flight": { "type": "integer", "description": "The maximum number of messages to have in flight at a given time. Increase this to improve throughput. Default: 64" }, "max_retries": { "type": "integer", "description": "The maximum number of retries before giving up on a request. Default: 3" }, "password_authenticator": { "type": "object", "description": "Optional configuration of Cassandra authentication parameters.", "properties": { "enabled": { "type": "boolean", "description": "Whether to use password authentication Default: false" }, "password": { "type": "string", "description": "The password to authenticate with. Default: " }, "username": { "type": "string", "description": "The username to authenticate as. Default: " } } }, "query": { "type": "string", "description": "A query to execute for each message." }, "timeout": { "type": "string", "description": "The client connection timeout. Default: 600ms" }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } } } }, "couchbase": { "type": "object", "properties": { "batching": { "type": "object", "description": "\nAllows you to configure a [batching policy](/docs/configuration/batching).", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] }, "bucket": { "type": "string", "description": "Couchbase bucket." }, "cas_enabled": { "type": "boolean", "description": "Enable CAS validation. Default: true" }, "collection": { "type": "string", "description": "Bucket collection. Default: _default" }, "content": { "type": "string", "description": "Document content." }, "id": { "type": "string", "description": "Document id.", "examples": [ "${! json(\"id\") }" ] }, "max_in_flight": { "type": "integer", "description": "The maximum number of messages to have in flight at a given time. Increase this to improve throughput. Default: 64" }, "operation": { "type": "string", "description": "Couchbase operation to perform. Default: upsert" }, "password": { "type": "string", "description": "Password to connect to the cluster." }, "timeout": { "type": "string", "description": "Operation timeout. Default: 15s" }, "transcoder": { "type": "string", "description": "Couchbase transcoder to use. Default: legacy" }, "ttl": { "type": "string", "description": "An optional TTL to set for items." }, "url": { "type": "string", "description": "Couchbase connection string.", "examples": [ "couchbase://localhost:11210" ] }, "username": { "type": "string", "description": "Username to connect to the cluster." } } }, "cypher": { "type": "object", "properties": { "basic_auth": { "type": "object", "description": "Basic Authentication fields", "properties": { "password": { "type": "string", "description": "The password for basic auth. Default: " }, "realm": { "type": "string", "description": "The realm for basic auth. Default: " }, "user": { "type": "string", "description": "The username for basic auth. Default: " } } }, "batching": { "type": "object", "description": "\nAllows you to configure a [batching policy](/docs/configuration/batching).", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] }, "database": { "type": "string", "description": "The name of the database to connect to.", "examples": [ "neo4j" ] }, "max_in_flight": { "type": "integer", "description": "The maximum number of messages to have in flight at a given time. Increase this to improve throughput. Default: 64" }, "no_auth": { "type": "boolean", "description": "Set to true to connect without authentication. Default: false" }, "query": { "type": "string", "description": "The cypher query to execute.", "examples": [ "CREATE (p:Person {name: $name}) RETURN p" ] }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "uri": { "type": "string", "description": "The URL of the database engine.", "examples": [ "bolt://localhost:7687" ] }, "values": { "type": "object", "description": "A map of strings -\u003e bloblang interpolations that form the values of the references in the query i.e. $name. Default: map[]", "examples": [ { "name": "${! json(\"name\") }" } ] } } }, "discord": { "type": "object", "properties": { "bot_token": { "type": "string", "description": "A bot token used for authentication." }, "channel_id": { "type": "string", "description": "A discord channel ID to write messages to." }, "rate_limit": { "type": "string", "description": " Default: An optional rate limit resource to restrict API requests with." } } }, "drop": { "type": "object", "description": " Default: map[]" }, "drop_on": { "type": "object", "properties": { "back_pressure": { "type": "string", "description": "An optional duration string that determines the maximum length of time to wait for a given message to be accepted by the child output before the message should be dropped instead. The most common reason for an output to block is when waiting for a lost connection to be re-established. Once a message has been dropped due to back pressure all subsequent messages are dropped immediately until the output is ready to process them again. Note that if `error` is set to `false` and this field is specified then messages dropped due to back pressure will return an error response (are nacked or reattempted).", "examples": [ "30s", "1m" ] }, "error": { "type": "boolean", "description": "Whether messages should be dropped when the child output returns an error of any type. For example, this could be when an `http_client` output gets a 4XX response code. In order to instead drop only on specific error patterns use the `error_matches` field instead. Default: false" }, "error_patterns": { "type": "array", "description": "A list of regular expressions (re2) where if the child output returns an error that matches any part of any of these patterns the message will be dropped.", "items": { "type": "string" }, "examples": [ [ "and that was really bad$" ], [ "roughly [0-9]+ issues occurred" ] ] }, "output": { "type": "object", "description": "A child output to wrap with this drop mechanism." } } }, "dynamic": { "type": "object", "properties": { "outputs": { "type": "object", "description": "A map of outputs to statically create. Default: map[]" }, "prefix": { "type": "string", "description": "A path prefix for HTTP endpoints that are registered. Default: " } } }, "elasticsearch": { "type": "object", "properties": { "action": { "type": "string", "description": "The action to take on the document. This field must resolve to one of the following action types: `create`, `index`, `update`, `upsert` or `delete`. Default: index" }, "aws": { "type": "object", "description": "Enables and customises connectivity to Amazon Elastic Service.", "properties": { "credentials": { "type": "object", "description": "Optional manual configuration of AWS credentials to use. More information can be found [in this document](/docs/guides/cloud/aws).", "properties": { "expiry_window": { "type": "string", "description": "Allow the credentials to trigger refreshing prior to the credentials actually expiring. This is beneficial so race conditions with expiring credentials do not cause requests to fail. For example '10s' would refresh credentials ten seconds before expiration. Setting to a duration of `0` disables the expiry window. Default: " }, "from_ec2_role": { "type": "boolean", "description": "Use the credentials of a host EC2 machine configured to assume [an IAM role associated with the instance](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html). Default: false" }, "id": { "type": "string", "description": "The ID of credentials to use. Default: " }, "profile": { "type": "string", "description": "A profile from `~/.aws/credentials` to use. Default: " }, "role": { "type": "string", "description": "A role ARN to assume. Default: " }, "role_external_id": { "type": "string", "description": "An external ID to provide when assuming a role. Default: " }, "secret": { "type": "string", "description": "The secret for the credentials being used. Default: " }, "token": { "type": "string", "description": "The token for the credentials being used, required when using short term credentials. Default: " } } }, "enabled": { "type": "boolean", "description": "Whether to connect to Amazon Elastic Service. Default: false" }, "endpoint": { "type": "string", "description": "Allows you to specify a custom endpoint for the AWS API. Default: " }, "region": { "type": "string", "description": "The AWS region to target. Default: " } } }, "backoff": { "type": "object", "description": "Control time intervals between retry attempts.", "properties": { "initial_interval": { "type": "string", "description": "The initial period to wait between retry attempts. Default: 1s" }, "max_elapsed_time": { "type": "string", "description": "The maximum period to wait before retry attempts are abandoned. If zero then no limit is used. Default: 30s" }, "max_interval": { "type": "string", "description": "The maximum period to wait between retry attempts. Default: 5s" } } }, "basic_auth": { "type": "object", "description": "Allows you to specify basic authentication.", "properties": { "enabled": { "type": "boolean", "description": "Whether to use basic authentication in requests. Default: false" }, "password": { "type": "string", "description": "A password to authenticate with. Default: " }, "username": { "type": "string", "description": "A username to authenticate as. Default: " } } }, "batching": { "type": "object", "description": "\nAllows you to configure a [batching policy](/docs/configuration/batching).", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] }, "gzip_compression": { "type": "boolean", "description": "Enable gzip compression on the request side. Default: false" }, "healthcheck": { "type": "boolean", "description": "Whether to enable healthchecks. Default: true" }, "id": { "type": "string", "description": "The ID for indexed messages. Interpolation should be used in order to create a unique ID for each message. Default: ${!count(\"elastic_ids\")}-${!timestamp_unix()}" }, "index": { "type": "string", "description": "The index to place messages." }, "max_in_flight": { "type": "integer", "description": "The maximum number of messages to have in flight at a given time. Increase this to improve throughput. Default: 64" }, "max_retries": { "type": "integer", "description": "The maximum number of retries before giving up on the request. If set to zero there is no discrete limit. Default: 0" }, "pipeline": { "type": "string", "description": "An optional pipeline id to preprocess incoming documents. Default: " }, "routing": { "type": "string", "description": "The routing key to use for the document. Default: " }, "sniff": { "type": "boolean", "description": "Prompts Bento to sniff for brokers to connect to when establishing a connection. Default: true" }, "timeout": { "type": "string", "description": "The maximum time to wait before abandoning a request (and trying again). Default: 5s" }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "type": { "type": "string", "description": "The document mapping type. This field is required for versions of elasticsearch earlier than 6.0.0, but are invalid for versions 7.0.0 or later. Default: " }, "urls": { "type": "array", "description": "A list of URLs to connect to. If an item of the list contains commas it will be expanded into multiple URLs.", "items": { "type": "string" }, "examples": [ [ "http://localhost:9200" ] ] } } }, "elasticsearch_v2": { "type": "object", "properties": { "action": { "type": "string", "description": "The action to take on the document. This field must resolve to one of the following action types: `create`, `index`, `update`, `upsert` or `delete`. Default: index" }, "backoff": { "type": "object", "description": "Control time intervals between retry attempts.", "properties": { "initial_interval": { "type": "string", "description": "The initial period to wait between retry attempts. Default: 1s" }, "max_elapsed_time": { "type": "string", "description": "The maximum period to wait before retry attempts are abandoned. If zero then no limit is used. Default: 30s" }, "max_interval": { "type": "string", "description": "The maximum period to wait between retry attempts. Default: 5s" } } }, "basic_auth": { "type": "object", "description": "Allows you to specify basic authentication.", "properties": { "enabled": { "type": "boolean", "description": "Whether to use basic authentication in requests. Default: false" }, "password": { "type": "string", "description": "A password to authenticate with. Default: " }, "username": { "type": "string", "description": "A username to authenticate as. Default: " } } }, "batching": { "type": "object", "description": "\nAllows you to configure a [batching policy](/docs/configuration/batching).", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] }, "compress_request_body": { "type": "boolean", "description": "Enable gzip compression on the request side. Default: false" }, "discover_nodes_interval": { "type": "string", "description": "Discover nodes periodically. Default: 0s" }, "discover_nodes_on_start": { "type": "boolean", "description": "Discover nodes when initializing the client. Default: false" }, "id": { "type": "string", "description": "The ID for indexed messages. Interpolation should be used in order to create a unique ID for each message. Default: ${!count(\"elastic_ids\")}-${!timestamp_unix()}" }, "index": { "type": "string", "description": "The index to place messages." }, "max_in_flight": { "type": "integer", "description": "The maximum number of messages to have in flight at a given time. Increase this to improve throughput. Default: 64" }, "max_retries": { "type": "integer", "description": "The maximum number of retries before giving up on the request. If set to zero there is no discrete limit. Default: 0" }, "pipeline": { "type": "string", "description": "An optional pipeline id to preprocess incoming documents. Default: " }, "retry_on_status": { "type": "integer", "description": "HTTP Status codes that should be retried. Default: [502 503 504]", "items": { "type": "integer" } }, "routing": { "type": "string", "description": "The routing key to use for the document. Default: " }, "timeout": { "type": "string", "description": "The maximum time to wait before abandoning a request (and trying again). Default: 5s" }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "urls": { "type": "array", "description": "A list of URLs to connect to. If an item of the list contains commas it will be expanded into multiple URLs.", "items": { "type": "string" }, "examples": [ [ "http://localhost:9200" ] ] } } }, "fallback": { "type": "array", "description": " Default: []", "items": { "type": "object" } }, "file": { "type": "object", "properties": { "codec": { "type": "string", "description": "The way in which the bytes of messages should be written out into the output data stream. It's possible to write lines using a custom delimiter with the `delim:x` codec, where x is the character sequence custom delimiter. Default: lines", "examples": [ "lines", "delim:\t", "delim:foobar" ] }, "path": { "type": "string", "description": "The file to write to, if the file does not yet exist it will be created.", "examples": [ "/tmp/data.txt", "/tmp/${! timestamp_unix() }.txt", "/tmp/${! json(\"document.id\") }.json" ] } } }, "gcp_bigquery": { "type": "object", "properties": { "auto_detect": { "type": "boolean", "description": "Indicates if we should automatically infer the options and schema for CSV and JSON sources. If the table doesn't exist and this field is set to `false` the output may not be able to insert data and will throw insertion error. Be careful using this field since it delegates to the GCP BigQuery service the schema detection and values like `\"no\"` may be treated as booleans for the CSV format. Default: false" }, "batching": { "type": "object", "description": "\nAllows you to configure a [batching policy](/docs/configuration/batching).", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] }, "create_disposition": { "type": "string", "description": "Specifies the circumstances under which destination table will be created. If CREATE_IF_NEEDED is used the GCP BigQuery will create the table if it does not already exist and tables are created atomically on successful completion of a job. The CREATE_NEVER option ensures the table must already exist and will not be automatically created. Default: CREATE_IF_NEEDED" }, "csv": { "type": "object", "description": "Specify how CSV data should be interpretted.", "properties": { "allow_jagged_rows": { "type": "boolean", "description": "Causes missing trailing optional columns to be tolerated when reading CSV data. Missing values are treated as nulls. Default: false" }, "allow_quoted_newlines": { "type": "boolean", "description": "Sets whether quoted data sections containing newlines are allowed when reading CSV data. Default: false" }, "encoding": { "type": "string", "description": "Encoding is the character encoding of data to be read. Default: UTF-8" }, "field_delimiter": { "type": "string", "description": "The separator for fields in a CSV file, used when reading or exporting data. Default: ," }, "header": { "type": "array", "description": "A list of values to use as header for each batch of messages. If not specified the first line of each message will be used as header. Default: []", "items": { "type": "string" } }, "skip_leading_rows": { "type": "integer", "description": "The number of rows at the top of a CSV file that BigQuery will skip when reading data. The default value is 1 since Bento will add the specified header in the first line of each batch sent to BigQuery. Default: 1" } } }, "dataset": { "type": "string", "description": "The BigQuery Dataset ID." }, "format": { "type": "string", "description": "The format of each incoming message. Default: NEWLINE_DELIMITED_JSON" }, "ignore_unknown_values": { "type": "boolean", "description": "Causes values not matching the schema to be tolerated. Unknown values are ignored. For CSV this ignores extra values at the end of a line. For JSON this ignores named values that do not match any column name. If this field is set to false (the default value), records containing unknown values are treated as bad records. The max_bad_records field can be used to customize how bad records are handled. Default: false" }, "job_labels": { "type": "object", "description": "A list of labels to add to the load job. Default: map[]" }, "max_bad_records": { "type": "integer", "description": "The maximum number of bad records that will be ignored when reading data. Default: 0" }, "max_in_flight": { "type": "integer", "description": "The maximum number of message batches to have in flight at a given time. Increase this to improve throughput. Default: 64" }, "project": { "type": "string", "description": "The project ID of the dataset to insert data to. If not set, it will be inferred from the credentials or read from the GOOGLE_CLOUD_PROJECT environment variable. Default: " }, "table": { "type": "string", "description": ":::caution Interpolation of Message Batches\nIt is assumed that the first message in the batch will resolve the bloblang query and that string will be used for all messages in the batch.\n:::\nThe table to insert messages to." }, "write_disposition": { "type": "string", "description": "Specifies how existing data in a destination table is treated. Default: WRITE_APPEND" } } }, "gcp_bigquery_write_api": { "type": "object", "properties": { "batching": { "type": "object", "description": "\nAllows you to configure a [batching policy](/docs/configuration/batching).", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] }, "dataset": { "type": "string", "description": "The BigQuery Dataset ID." }, "endpoint": { "type": "object", "description": "Used to overwrite the default gRPC and HTTP BigQuery endpoints.", "properties": { "grpc": { "type": "string", "description": "The endpoint used to create the BigQuery Storage API client. Default: " }, "http": { "type": "string", "description": "The endpoint used to create the BigQuery client. Default: " } } }, "max_in_flight": { "type": "integer", "description": "The maximum number of message batches to have in flight at a given time. Increase this to improve throughput. Default: 64" }, "message_format": { "type": "string", "description": "Format of incoming messages Default: json" }, "project": { "type": "string", "description": "The project ID of the dataset to insert data to. If not set, it will be inferred from the credentials or read from the GOOGLE_CLOUD_PROJECT environment variable. Default: " }, "stream_type": { "type": "string", "description": ":::caution Storage API Stream Types\nOnly `DEFAULT` stream types are currently enabled. Future versions will see support extended to `COMMITTED`, `BUFFERED`, and `PENDING`.\n:::\nsets the type of stream this write client is managing. Default: DEFAULT" }, "table": { "type": "string", "description": ":::caution Interpolation of Message Batches\nIt is assumed that the first message in the batch will resolve the bloblang query and that string will be used for all messages in the batch.\n:::\nThe table to insert messages to." } } }, "gcp_cloud_storage": { "type": "object", "properties": { "batching": { "type": "object", "description": "\nAllows you to configure a [batching policy](/docs/configuration/batching).", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] }, "bucket": { "type": "string", "description": "The bucket to upload messages to." }, "chunk_size": { "type": "integer", "description": "An optional chunk size which controls the maximum number of bytes of the object that the Writer will attempt to send to the server in a single request. If ChunkSize is set to zero, chunking will be disabled. Default: 16777216" }, "collision_mode": { "type": "string", "description": "Determines how file path collisions should be dealt with. Default: overwrite" }, "content_encoding": { "type": "string", "description": "An optional content encoding to set for each object. Default: " }, "content_type": { "type": "string", "description": "The content type to set for each object. Default: application/octet-stream" }, "max_in_flight": { "type": "integer", "description": "The maximum number of message batches to have in flight at a given time. Increase this to improve throughput. Default: 64" }, "path": { "type": "string", "description": "The path of each message to upload. Default: ${!count(\"files\")}-${!timestamp_unix_nano()}.txt", "examples": [ "${!count(\"files\")}-${!timestamp_unix_nano()}.txt", "${!metadata(\"kafka_key\")}.json", "${!json(\"doc.namespace\")}/${!json(\"doc.id\")}.json" ] }, "timeout": { "type": "string", "description": "The maximum period to wait on an upload before abandoning it and reattempting. Default: 3s", "examples": [ "1s", "500ms" ] } } }, "gcp_pubsub": { "type": "object", "properties": { "batching": { "type": "object", "description": "Configures a batching policy on this output. While the PubSub client maintains its own internal buffering mechanism, preparing larger batches of messages can further trade-off some latency for throughput.", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] }, "byte_threshold": { "type": "integer", "description": "Publish a batch when its size in bytes reaches this value. Default: 1000000" }, "count_threshold": { "type": "integer", "description": "Publish a pubsub buffer when it has this many messages Default: 100" }, "delay_threshold": { "type": "string", "description": "Publish a non-empty pubsub buffer after this delay has passed. Default: 10ms" }, "endpoint": { "type": "string", "description": "An optional endpoint to override the default of `pubsub.googleapis.com:443`. This can be used to connect to a region specific pubsub endpoint. For a list of valid values check out [this document.](https://cloud.google.com/pubsub/docs/reference/service_apis_overview#list_of_regional_endpoints) Default: ", "examples": [ "us-central1-pubsub.googleapis.com:443", "us-west3-pubsub.googleapis.com:443" ] }, "flow_control": { "type": "object", "description": "For a given topic, configures the PubSub client's internal buffer for messages to be published.", "properties": { "limit_exceeded_behavior": { "type": "string", "description": "Configures the behavior when trying to publish additional messages while the flow controller is full. The available options are block (default), ignore (disable), and signal_error (publish results will return an error). Default: block" }, "max_outstanding_bytes": { "type": "integer", "description": "Maximum size of buffered messages to be published. If less than or equal to zero, this is disabled. Default: -1" }, "max_outstanding_messages": { "type": "integer", "description": "Maximum number of buffered messages to be published. If less than or equal to zero, this is disabled. Default: 1000" } } }, "max_in_flight": { "type": "integer", "description": "The maximum number of messages to have in flight at a given time. Increasing this may improve throughput. Default: 64" }, "metadata": { "type": "object", "description": "Specify criteria for which metadata values are sent as attributes, all are sent by default.", "properties": { "exclude_prefixes": { "type": "array", "description": "Provide a list of explicit metadata key prefixes to be excluded when adding metadata to sent messages. Default: []", "items": { "type": "string" } } } }, "ordering_key": { "type": "string", "description": "The ordering key to use for publishing messages." }, "project": { "type": "string", "description": "The project ID of the topic to publish to." }, "publish_timeout": { "type": "string", "description": "The maximum length of time to wait before abandoning a publish attempt for a message. Default: 1m0s", "examples": [ "10s", "5m", "60m" ] }, "topic": { "type": "string", "description": "The topic to publish to." } } }, "grpc_client": { "type": "object", "properties": { "address": { "type": "string", "description": "The URI of the gRPC target to connect to.", "examples": [ "localhost:50051" ] }, "batching": { "type": "object", "description": "\nAllows you to configure a [batching policy](/docs/configuration/batching).", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] }, "health_check": { "type": "object", "properties": { "enabled": { "type": "boolean", "description": "Whether Bento should healthcheck the unary `Check` rpc endpoint on init connection: [gRPC Health Checking](https://grpc.io/docs/guides/health-checking/) Default: false" }, "service": { "type": "string", "description": "The name of the service to healthcheck, note that the default value of \"\", will attempt to check the health of the whole server Default: " } } }, "max_in_flight": { "type": "integer", "description": "The maximum number of messages to have in flight at a given time. Increase this to improve throughput. Default: 64" }, "method": { "type": "string", "description": "The name of the method to invoke", "examples": [ "SayHello" ] }, "oauth2": { "type": "object", "description": "Allows you to specify open authentication via OAuth version 2 using the client credentials token flow.", "properties": { "client_key": { "type": "string", "description": "A value used to identify the client to the token provider. Default: " }, "client_secret": { "type": "string", "description": "A secret used to establish ownership of the client key. Default: " }, "enabled": { "type": "boolean", "description": "Whether to use OAuth version 2 in requests. Default: false" }, "endpoint_params": { "type": "object", "description": "A list of optional endpoint parameters, values should be arrays of strings. Default: map[]", "examples": [ { "bar": [ "woof" ], "foo": [ "meow", "quack" ] } ] }, "scopes": { "type": "array", "description": "A list of optional requested permissions. Default: []", "items": { "type": "string" } }, "token_url": { "type": "string", "description": "The URL of the token provider. Default: " } } }, "propagate_response": { "type": "boolean", "description": "Whether responses from the server should be [propagated back](/docs/guides/sync_responses) to the input. Default: false" }, "proto_files": { "type": "array", "description": "A list of filepaths of .proto files that should contain the schemas necessary for the gRPC method. Default: []", "items": { "type": "string" }, "examples": [ [ "./grpc_test_server/helloworld.proto" ] ] }, "reflection": { "type": "boolean", "description": "If set to true, Bento will acquire the protobuf schema for the method from the server via [gRPC Reflection](https://grpc.io/docs/guides/reflection/). Default: false" }, "rpc_type": { "type": "string", "description": "The type of the rpc method. Default: unary" }, "service": { "type": "string", "description": "The name of the service.", "examples": [ "helloworld.Greeter" ] }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } } } }, "hdfs": { "type": "object", "properties": { "batching": { "type": "object", "description": "\nAllows you to configure a [batching policy](/docs/configuration/batching).", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] }, "directory": { "type": "string", "description": "A directory to store message files within. If the directory does not exist it will be created." }, "hosts": { "type": "array", "description": "A list of target host addresses to connect to.", "items": { "type": "string" }, "examples": [ "localhost:9000" ] }, "max_in_flight": { "type": "integer", "description": "The maximum number of messages to have in flight at a given time. Increase this to improve throughput. Default: 64" }, "path": { "type": "string", "description": "The path to upload messages as, interpolation functions should be used in order to generate unique file paths. Default: ${!count(\"files\")}-${!timestamp_unix_nano()}.txt" }, "user": { "type": "string", "description": "A user ID to connect as. Default: " } } }, "http_client": { "type": "object", "properties": { "backoff_on": { "type": "integer", "description": "A list of status codes whereby the request should be considered to have failed and retries should be attempted, but the period between them should be increased gradually. Default: [429]", "items": { "type": "integer" } }, "basic_auth": { "type": "object", "description": "Allows you to specify basic authentication.", "properties": { "enabled": { "type": "boolean", "description": "Whether to use basic authentication in requests. Default: false" }, "password": { "type": "string", "description": "A password to authenticate with. Default: " }, "username": { "type": "string", "description": "A username to authenticate as. Default: " } } }, "batch_as_multipart": { "type": "boolean", "description": "Send message batches as a single request using [RFC1341](https://www.w3.org/Protocols/rfc1341/7_2_Multipart.html). If disabled messages in batches will be sent as individual requests. Default: false" }, "batching": { "type": "object", "description": "\nAllows you to configure a [batching policy](/docs/configuration/batching).", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] }, "drop_on": { "type": "integer", "description": "A list of status codes whereby the request should be considered to have failed but retries should not be attempted. This is useful for preventing wasted retries for requests that will never succeed. Note that with these status codes the _request_ is dropped, but _message_ that caused the request will not be dropped. Default: []", "items": { "type": "integer" } }, "dump_request_log_level": { "type": "string", "description": "EXPERIMENTAL: Optionally set a level at which the request and response payload of each request made will be logged. Default: " }, "extract_headers": { "type": "object", "description": "Specify which response headers should be added to resulting synchronous response messages as metadata. Header keys are lowercased before matching, so ensure that your patterns target lowercased versions of the header keys that you expect. This field is not applicable unless `propagate_response` is set to `true`.", "properties": { "include_patterns": { "type": "array", "description": "Provide a list of explicit metadata key regular expression (re2) patterns to match against. Default: []", "items": { "type": "string" }, "examples": [ [ ".*" ], [ "_timestamp_unix$" ] ] }, "include_prefixes": { "type": "array", "description": "Provide a list of explicit metadata key prefixes to match against. Default: []", "items": { "type": "string" }, "examples": [ [ "foo_", "bar_" ], [ "kafka_" ], [ "content-" ] ] } } }, "headers": { "type": "object", "description": "A map of headers to add to the request. Default: map[]", "examples": [ { "Content-Type": "application/octet-stream", "traceparent": "${! tracing_span().traceparent }" } ] }, "jwt": { "type": "object", "description": "BETA: Allows you to specify JWT authentication.", "properties": { "claims": { "type": "object", "description": "A value used to identify the claims that issued the JWT. Default: map[]" }, "enabled": { "type": "boolean", "description": "Whether to use JWT authentication in requests. Default: false" }, "headers": { "type": "object", "description": "Add optional key/value headers to the JWT. Default: map[]" }, "private_key_file": { "type": "string", "description": "A file with the PEM encoded via PKCS1 or PKCS8 as private key. Default: " }, "signing_method": { "type": "string", "description": "A method used to sign the token such as RS256, RS384, RS512 or EdDSA. Default: " } } }, "max_in_flight": { "type": "integer", "description": "The maximum number of parallel message batches to have in flight at any given time. Default: 64" }, "max_retry_backoff": { "type": "string", "description": "The maximum period to wait between failed requests. Default: 300s" }, "metadata": { "type": "object", "description": "Specify optional matching rules to determine which metadata keys should be added to the HTTP request as headers.", "properties": { "include_patterns": { "type": "array", "description": "Provide a list of explicit metadata key regular expression (re2) patterns to match against. Default: []", "items": { "type": "string" }, "examples": [ [ ".*" ], [ "_timestamp_unix$" ] ] }, "include_prefixes": { "type": "array", "description": "Provide a list of explicit metadata key prefixes to match against. Default: []", "items": { "type": "string" }, "examples": [ [ "foo_", "bar_" ], [ "kafka_" ], [ "content-" ] ] } } }, "multipart": { "type": "array", "description": "EXPERIMENTAL: Create explicit multipart HTTP requests by specifying an array of parts to add to the request, each part specified consists of content headers and a data field that can be populated dynamically. If this field is populated it will override the default request creation behaviour. Default: []", "items": { "type": "object" } }, "oauth": { "type": "object", "description": "Allows you to specify open authentication via OAuth version 1.", "properties": { "access_token": { "type": "string", "description": "A value used to gain access to the protected resources on behalf of the user. Default: " }, "access_token_secret": { "type": "string", "description": "A secret provided in order to establish ownership of a given access token. Default: " }, "consumer_key": { "type": "string", "description": "A value used to identify the client to the service provider. Default: " }, "consumer_secret": { "type": "string", "description": "A secret used to establish ownership of the consumer key. Default: " }, "enabled": { "type": "boolean", "description": "Whether to use OAuth version 1 in requests. Default: false" } } }, "oauth2": { "type": "object", "description": "Allows you to specify open authentication via OAuth version 2 using the client credentials token flow.", "properties": { "client_key": { "type": "string", "description": "A value used to identify the client to the token provider. Default: " }, "client_secret": { "type": "string", "description": "A secret used to establish ownership of the client key. Default: " }, "enabled": { "type": "boolean", "description": "Whether to use OAuth version 2 in requests. Default: false" }, "endpoint_params": { "type": "object", "description": "A list of optional endpoint parameters, values should be arrays of strings. Default: map[]", "examples": [ { "bar": [ "woof" ], "foo": [ "meow", "quack" ] } ] }, "scopes": { "type": "array", "description": "A list of optional requested permissions. Default: []", "items": { "type": "string" } }, "token_url": { "type": "string", "description": "The URL of the token provider. Default: " } } }, "propagate_response": { "type": "boolean", "description": "Whether responses from the server should be [propagated back](/docs/guides/sync_responses) to the input. Default: false" }, "proxy_url": { "type": "string", "description": "An optional HTTP proxy URL." }, "rate_limit": { "type": "string", "description": "An optional [rate limit](/docs/components/rate_limits/about) to throttle requests by." }, "retries": { "type": "integer", "description": "The maximum number of retry attempts to make. Default: 3" }, "retry_period": { "type": "string", "description": "The base period to wait between failed requests. Default: 1s" }, "successful_on": { "type": "integer", "description": "A list of status codes whereby the attempt should be considered successful, this is useful for dropping requests that return non-2XX codes indicating that the message has been dealt with, such as a 303 See Other or a 409 Conflict. All 2XX codes are considered successful unless they are present within `backoff_on` or `drop_on`, regardless of this field. Default: []", "items": { "type": "integer" } }, "timeout": { "type": "string", "description": "A static timeout to apply to requests. Default: 5s" }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "transport": { "type": "object", "description": "Custom transport options.", "properties": { "dial_context": { "type": "object", "description": "Settings for the dialer used to create new connections.", "properties": { "keep_alive": { "type": "string", "description": "Keep-alive period for active network connections used by the dialer. Default: 30s" }, "timeout": { "type": "string", "description": "Timeout for establishing new network connections. Default: 30s" } } }, "expect_continue_timeout": { "type": "string", "description": "Time to wait for a server's first response headers after sending request headers when 'Expect: 100-continue' is used. Zero means send body immediately. Default: 1s" }, "force_http2": { "type": "boolean", "description": "If true, the transport will attempt to use HTTP/2. Default: true" }, "idle_connection_timeout": { "type": "string", "description": "Maximum time an idle keep-alive connection remains open before closing itself. Default: 90s" }, "max_idle_connections": { "type": "integer", "description": "Maximum number of idle keep-alive connections. Zero = unlimited. Default: 100" }, "tls_handshake_timeout": { "type": "string", "description": "Maximum time allowed for TLS handshake to complete. Default: 10s" } } }, "url": { "type": "string", "description": "The URL to connect to." }, "verb": { "type": "string", "description": "A verb to connect with Default: POST", "examples": [ "POST", "GET", "DELETE" ] } } }, "http_server": { "type": "object", "properties": { "address": { "type": "string", "description": "An alternative address to host from. If left empty the service wide address is used. Default: " }, "allowed_verbs": { "type": "array", "description": "An array of verbs that are allowed for the `path` and `stream_path` HTTP endpoint. Default: [GET]", "items": { "type": "string" } }, "cert_file": { "type": "string", "description": "Enable TLS by specifying a certificate and key file. Only valid with a custom `address`. Default: " }, "cors": { "type": "object", "description": "Adds Cross-Origin Resource Sharing headers. Only valid with a custom `address`.", "properties": { "allowed_headers": { "type": "array", "description": "Appends additional headers to the list of default allowed headers: Accept, Accept-Language, Content-Language \u0026 Origin. These default headers are therefore always allowed. Default: []", "items": { "type": "string" } }, "allowed_methods": { "type": "array", "description": "Used to explicitly set allowed methods in the Access-Control-Allow-Methods header. Default: [GET HEAD POST PUT PATCH DELETE]", "items": { "type": "string" } }, "allowed_origins": { "type": "array", "description": "An explicit list of origins that are allowed for CORS requests. Default: []", "items": { "type": "string" } }, "enabled": { "type": "boolean", "description": "Whether to allow CORS requests. Default: false" } } }, "heartbeat": { "type": "string", "description": "The time to wait before sending a heartbeat message. Default: 0s" }, "key_file": { "type": "string", "description": "Enable TLS by specifying a certificate and key file. Only valid with a custom `address`. Default: " }, "path": { "type": "string", "description": "The path from which discrete messages can be consumed. Default: /get" }, "ping_period": { "type": "string", "description": "Send pings to client with this period. Must be less than pong wait. Default: 54s" }, "pong_wait": { "type": "string", "description": "The time allowed to read the next pong message from the client. Default: 60s" }, "stream_format": { "type": "string", "description": "The format of the stream endpoint. `raw_bytes` delivers messages directly with newlines between batches, while `event_source` formats according to Server-Sent Events (SSE) specification with `data:` prefixes, compatible with EventSource API. Default: raw_bytes" }, "stream_path": { "type": "string", "description": "The path from which a continuous stream of messages can be consumed. Default: /get/stream" }, "timeout": { "type": "string", "description": "The maximum time to wait before a blocking, inactive connection is dropped (only applies to the `path` endpoint). Default: 5s" }, "write_wait": { "type": "string", "description": "The time allowed to write a message to the websocket. Default: 10s" }, "ws_message_type": { "type": "string", "description": "Type of websocket message Default: binary" }, "ws_path": { "type": "string", "description": "The path from which websocket connections can be established. Default: /get/ws" } } }, "inproc": { "type": "string", "description": " Default: " }, "kafka": { "type": "object", "properties": { "ack_replicas": { "type": "boolean", "description": "Ensure that messages have been copied across all replicas before acknowledging receipt. Default: false" }, "addresses": { "type": "array", "description": "A list of broker addresses to connect to. If an item of the list contains commas it will be expanded into multiple addresses.", "items": { "type": "string" }, "examples": [ [ "localhost:9092" ], [ "localhost:9041,localhost:9042" ], [ "localhost:9041", "localhost:9042" ] ] }, "backoff": { "type": "object", "description": "Control time intervals between retry attempts.", "properties": { "initial_interval": { "type": "string", "description": "The initial period to wait between retry attempts. Default: 3s", "examples": [ "50ms", "1s" ] }, "max_elapsed_time": { "type": "string", "description": "The maximum overall period of time to spend on retry attempts before the request is aborted. Setting this value to a zeroed duration (such as `0s`) will result in unbounded retries. Default: 30s", "examples": [ "1m", "1h" ] }, "max_interval": { "type": "string", "description": "The maximum period to wait between retry attempts Default: 10s", "examples": [ "5s", "1m" ] } } }, "batching": { "type": "object", "description": "\nAllows you to configure a [batching policy](/docs/configuration/batching).", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] }, "client_id": { "type": "string", "description": "An identifier for the client connection. Default: bento" }, "compression": { "type": "string", "description": "The compression algorithm to use. Default: none" }, "custom_topic_creation": { "type": "object", "description": "If enabled, topics will be created with the specified number of partitions and replication factor if they do not already exist.", "properties": { "enabled": { "type": "boolean", "description": "Whether to enable custom topic creation. Default: false" }, "partitions": { "type": "integer", "description": "The number of partitions to create for new topics. Leave at -1 to use the broker configured default. Must be \u003e= 1. Default: -1" }, "replication_factor": { "type": "integer", "description": "The replication factor to use for new topics. Leave at -1 to use the broker configured default. Must be an odd number, and less then or equal to the number of brokers. Default: -1" } } }, "idempotent_write": { "type": "boolean", "description": "Enable the idempotent write producer option. This requires the `IDEMPOTENT_WRITE` permission on `CLUSTER` and can be disabled if this permission is not available. Default: false" }, "inject_tracing_map": { "type": "string", "description": "EXPERIMENTAL: A [Bloblang mapping](/docs/guides/bloblang/about) used to inject an object containing tracing propagation information into outbound messages. The specification of the injected fields will match the format used by the service wide tracer.", "examples": [ "meta = @.assign(this)", "root.meta.span = this" ] }, "key": { "type": "string", "description": "The key to publish messages with. Default: " }, "max_in_flight": { "type": "integer", "description": "The maximum number of messages to have in flight at a given time. Increase this to improve throughput. Default: 64" }, "max_msg_bytes": { "type": "integer", "description": "The maximum size in bytes of messages sent to the target topic. Default: 1000000" }, "max_retries": { "type": "integer", "description": "The maximum number of retries before giving up on the request. If set to zero there is no discrete limit. Default: 0" }, "metadata": { "type": "object", "description": "Specify criteria for which metadata values are sent with messages as headers.", "properties": { "exclude_prefixes": { "type": "array", "description": "Provide a list of explicit metadata key prefixes to be excluded when adding metadata to sent messages. Default: []", "items": { "type": "string" } } } }, "partition": { "type": "string", "description": "The manually-specified partition to publish messages to, relevant only when the field `partitioner` is set to `manual`. Must be able to parse as a 32-bit integer. Default: " }, "partitioner": { "type": "string", "description": "The partitioning algorithm to use. Default: fnv1a_hash" }, "rack_id": { "type": "string", "description": "A rack identifier for this client. Default: " }, "retry_as_batch": { "type": "boolean", "description": "When enabled forces an entire batch of messages to be retried if any individual message fails on a send, otherwise only the individual messages that failed are retried. Disabling this helps to reduce message duplicates during intermittent errors, but also makes it impossible to guarantee strict ordering of messages. Default: false" }, "sasl": { "type": "object", "description": "Enables SASL authentication.", "properties": { "access_token": { "type": "string", "description": "A static OAUTHBEARER access token Default: " }, "aws": { "type": "object", "description": "Contains AWS specific fields for when the `mechanism` is set to `AWS_MSK_IAM`.", "properties": { "credentials": { "type": "object", "description": "Optional manual configuration of AWS credentials to use. More information can be found [in this document](/docs/guides/cloud/aws).", "properties": { "expiry_window": { "type": "string", "description": "Allow the credentials to trigger refreshing prior to the credentials actually expiring. This is beneficial so race conditions with expiring credentials do not cause requests to fail. For example '10s' would refresh credentials ten seconds before expiration. Setting to a duration of `0` disables the expiry window. Default: " }, "from_ec2_role": { "type": "boolean", "description": "Use the credentials of a host EC2 machine configured to assume [an IAM role associated with the instance](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html). Default: false" }, "id": { "type": "string", "description": "The ID of credentials to use. Default: " }, "profile": { "type": "string", "description": "A profile from `~/.aws/credentials` to use. Default: " }, "role": { "type": "string", "description": "A role ARN to assume. Default: " }, "role_external_id": { "type": "string", "description": "An external ID to provide when assuming a role. Default: " }, "secret": { "type": "string", "description": "The secret for the credentials being used. Default: " }, "token": { "type": "string", "description": "The token for the credentials being used, required when using short term credentials. Default: " } } }, "endpoint": { "type": "string", "description": "Allows you to specify a custom endpoint for the AWS API. Default: " }, "region": { "type": "string", "description": "The AWS region to target. Default: " } } }, "mechanism": { "type": "string", "description": "The SASL authentication mechanism, if left empty SASL authentication is not used. Default: none" }, "password": { "type": "string", "description": "A PLAIN password. It is recommended that you use environment variables to populate this field. Default: ", "examples": [ "${PASSWORD}" ] }, "token_cache": { "type": "string", "description": "Instead of using a static `access_token` allows you to query a [`cache`](/docs/components/caches/about) resource to fetch OAUTHBEARER tokens from Default: " }, "token_key": { "type": "string", "description": "Required when using a `token_cache`, the key to query the cache with for tokens. Default: " }, "user": { "type": "string", "description": "A PLAIN username. It is recommended that you use environment variables to populate this field. Default: ", "examples": [ "${USER}" ] } } }, "static_headers": { "type": "object", "description": "An optional map of static headers that should be added to messages in addition to metadata.", "examples": [ { "first-static-header": "value-1", "second-static-header": "value-2" } ] }, "target_version": { "type": "string", "description": "The version of the Kafka protocol to use. This limits the capabilities used by the client and should ideally match the version of your brokers. Defaults to the oldest supported stable version.", "examples": [ "2.1.0", "3.1.0" ] }, "timeout": { "type": "string", "description": "The maximum period of time to wait for message sends before abandoning the request and retrying. Default: 5s" }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "topic": { "type": "string", "description": "The topic to publish messages to." } } }, "kafka_franz": { "type": "object", "properties": { "batching": { "type": "object", "description": "\nAllows you to configure a [batching policy](/docs/configuration/batching).", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] }, "client_id": { "type": "string", "description": "An identifier for the client connection. Default: bento" }, "compression": { "type": "string", "description": "Optionally set an explicit compression type. The default preference is to use snappy when the broker supports it, and fall back to none if not." }, "idempotent_write": { "type": "boolean", "description": "Enable the idempotent write producer option. This requires the `IDEMPOTENT_WRITE` permission on `CLUSTER` and can be disabled if this permission is not available. Default: true" }, "key": { "type": "string", "description": "An optional key to populate for each message." }, "max_buffered_records": { "type": "integer", "description": "Sets the max amount of records the client will buffer, blocking produces until records are finished if this limit is reached. This overrides the `franz-kafka` default of 10,000. Default: 10000" }, "max_in_flight": { "type": "integer", "description": "The maximum number of batches to be sending in parallel at any given time. Default: 10" }, "max_message_bytes": { "type": "string", "description": "The maximum space in bytes than an individual message may take, messages larger than this value will be rejected. This field corresponds to Kafka's `max.message.bytes`. Default: 1MB", "examples": [ "100MB", "50mib" ] }, "metadata": { "type": "object", "description": "Determine which (if any) metadata values should be added to messages as headers.", "properties": { "include_patterns": { "type": "array", "description": "Provide a list of explicit metadata key regular expression (re2) patterns to match against. Default: []", "items": { "type": "string" }, "examples": [ [ ".*" ], [ "_timestamp_unix$" ] ] }, "include_prefixes": { "type": "array", "description": "Provide a list of explicit metadata key prefixes to match against. Default: []", "items": { "type": "string" }, "examples": [ [ "foo_", "bar_" ], [ "kafka_" ], [ "content-" ] ] } } }, "metadata_max_age": { "type": "string", "description": "This sets the maximum age for the client's cached metadata, to allow detection of new topics, partitions, etc. Default: 5m" }, "partition": { "type": "string", "description": "An optional explicit partition to set for each message. This field is only relevant when the `partitioner` is set to `manual`. The provided interpolation string must be a valid integer.", "examples": [ "${! metadata(\"partition\") }" ] }, "partitioner": { "type": "string", "description": "Override the default murmur2 hashing partitioner." }, "rack_id": { "type": "string", "description": "A rack identifier for this client. Default: " }, "sasl": { "type": "array", "description": "Specify one or more methods of SASL authentication. SASL is tried in order; if the broker supports the first mechanism, all connections will use that mechanism. If the first mechanism fails, the client will pick the first supported mechanism. If the broker does not support any client mechanisms, connections will fail.", "items": { "type": "object" }, "examples": [ [ { "mechanism": "SCRAM-SHA-512", "password": "bar", "username": "foo" } ] ] }, "seed_brokers": { "type": "array", "description": "A list of broker addresses to connect to in order to establish connections. If an item of the list contains commas it will be expanded into multiple addresses.", "items": { "type": "string" }, "examples": [ [ "localhost:9092" ], [ "foo:9092", "bar:9092" ], [ "foo:9092,bar:9092" ] ] }, "timeout": { "type": "string", "description": "The maximum period of time to wait for message sends before abandoning the request and retrying Default: 10s" }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "topic": { "type": "string", "description": "A topic to write messages to." }, "uniform_bytes_options": { "type": "object", "description": "Sets partitioner options when `partitioner` is of type `uniform_bytes`. These values will otherwise be ignored. Note, that future versions will likely see this approach reworked.", "properties": { "adaptive": { "type": "boolean", "description": "Sets a slight imbalance so that the partitioner can produce more to brokers that are less loaded. Default: false" }, "bytes": { "type": "string", "description": "The number of bytes the partitioner will return the same partition for. Default: 1MB" }, "keys": { "type": "boolean", "description": "If `true`, uses standard hashing based on record key for records with non-nil keys. Default: false" } } } } }, "mongodb": { "type": "object", "properties": { "backoff": { "type": "object", "description": "Control time intervals between retry attempts.", "properties": { "initial_interval": { "type": "string", "description": "The initial period to wait between retry attempts. Default: 1s" }, "max_elapsed_time": { "type": "string", "description": "The maximum period to wait before retry attempts are abandoned. If zero then no limit is used. Default: 30s" }, "max_interval": { "type": "string", "description": "The maximum period to wait between retry attempts. Default: 5s" } } }, "batching": { "type": "object", "description": "\nAllows you to configure a [batching policy](/docs/configuration/batching).", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] }, "collection": { "type": "string", "description": "The name of the target collection." }, "database": { "type": "string", "description": "The name of the target MongoDB database." }, "document_map": { "type": "string", "description": "A bloblang map representing a document to store within MongoDB, expressed as [extended JSON in canonical form](https://www.mongodb.com/docs/manual/reference/mongodb-extended-json/). The document map is required for the operations insert-one, replace-one and update-one. Default: ", "examples": [ "root.a = this.foo\nroot.b = this.bar" ] }, "filter_map": { "type": "string", "description": "A bloblang map representing a filter for a MongoDB command, expressed as [extended JSON in canonical form](https://www.mongodb.com/docs/manual/reference/mongodb-extended-json/). The filter map is required for all operations except insert-one. It is used to find the document(s) for the operation. For example in a delete-one case, the filter map should have the fields required to locate the document to delete. Default: ", "examples": [ "root.a = this.foo\nroot.b = this.bar" ] }, "hint_map": { "type": "string", "description": "A bloblang map representing the hint for the MongoDB command, expressed as [extended JSON in canonical form](https://www.mongodb.com/docs/manual/reference/mongodb-extended-json/). This map is optional and is used with all operations except insert-one. It is used to improve performance of finding the documents in the mongodb. Default: ", "examples": [ "root.a = this.foo\nroot.b = this.bar" ] }, "max_in_flight": { "type": "integer", "description": "The maximum number of messages to have in flight at a given time. Increase this to improve throughput. Default: 64" }, "max_retries": { "type": "integer", "description": "The maximum number of retries before giving up on the request. If set to zero there is no discrete limit. Default: 3" }, "operation": { "type": "string", "description": "The mongodb operation to perform. Default: update-one" }, "password": { "type": "string", "description": "The password to connect to the database. Default: " }, "upsert": { "type": "boolean", "description": "The upsert setting is optional and only applies for update-one and replace-one operations. If the filter specified in filter_map matches, the document is updated or replaced accordingly, otherwise it is created. Default: false" }, "url": { "type": "string", "description": "The URL of the target MongoDB server.", "examples": [ "mongodb://localhost:27017" ] }, "username": { "type": "string", "description": "The username to connect to the database. Default: " }, "write_concern": { "type": "object", "description": "The write concern settings for the mongo connection.", "properties": { "j": { "type": "boolean", "description": "J requests acknowledgement from MongoDB that write operations are written to the journal. Default: false" }, "w": { "type": "string", "description": "W requests acknowledgement that write operations propagate to the specified number of mongodb instances. Default: " }, "w_timeout": { "type": "string", "description": "The write concern timeout. Default: " } } } } }, "mqtt": { "type": "object", "properties": { "client_id": { "type": "string", "description": "An identifier for the client connection. Default: " }, "connect_timeout": { "type": "string", "description": "The maximum amount of time to wait in order to establish a connection before the attempt is abandoned. Default: 30s", "examples": [ "1s", "500ms" ] }, "dynamic_client_id_suffix": { "type": "string", "description": "Append a dynamically generated suffix to the specified `client_id` on each run of the pipeline. This can be useful when clustering Bento producers." }, "keepalive": { "type": "integer", "description": "Max seconds of inactivity before a keepalive message is sent. Default: 30" }, "max_in_flight": { "type": "integer", "description": "The maximum number of messages to have in flight at a given time. Increase this to improve throughput. Default: 64" }, "password": { "type": "string", "description": "A password to connect with. Default: " }, "qos": { "type": "integer", "description": "The QoS value to set for each message. Has options 0, 1, 2. Default: 1" }, "retained": { "type": "boolean", "description": "Set message as retained on the topic. Default: false" }, "retained_interpolated": { "type": "string", "description": "Override the value of `retained` with an interpolable value, this allows it to be dynamically set based on message contents. The value must resolve to either `true` or `false`." }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "topic": { "type": "string", "description": "The topic to publish messages to." }, "urls": { "type": "array", "description": "A list of URLs to connect to. If an item of the list contains commas it will be expanded into multiple URLs.", "items": { "type": "string" }, "examples": [ [ "tcp://localhost:1883" ] ] }, "user": { "type": "string", "description": "A username to connect with. Default: " }, "will": { "type": "object", "description": "Set last will message in case of Bento failure", "properties": { "enabled": { "type": "boolean", "description": "Whether to enable last will messages. Default: false" }, "payload": { "type": "string", "description": "Set payload for last will message. Default: " }, "qos": { "type": "integer", "description": "Set QoS for last will message. Valid values are: 0, 1, 2. Default: 0" }, "retained": { "type": "boolean", "description": "Set retained for last will message. Default: false" }, "topic": { "type": "string", "description": "Set topic for last will message. Default: " } } }, "write_timeout": { "type": "string", "description": "The maximum amount of time to wait to write data before the attempt is abandoned. Default: 3s", "examples": [ "1s", "500ms" ] } } }, "nanomsg": { "type": "object", "properties": { "bind": { "type": "boolean", "description": "Whether the URLs listed should be bind (otherwise they are connected to). Default: false" }, "max_in_flight": { "type": "integer", "description": "The maximum number of messages to have in flight at a given time. Increase this to improve throughput. Default: 64" }, "poll_timeout": { "type": "string", "description": "The maximum period of time to wait for a message to send before the request is abandoned and reattempted. Default: 5s" }, "socket_type": { "type": "string", "description": "The socket type to send with. Default: PUSH" }, "urls": { "type": "array", "description": "A list of URLs to connect to. If an item of the list contains commas it will be expanded into multiple URLs.", "items": { "type": "string" } } } }, "nats": { "type": "object", "properties": { "auth": { "type": "object", "description": "Optional configuration of NATS authentication parameters.", "properties": { "nkey_file": { "type": "string", "description": "An optional file containing a NKey seed.", "examples": [ "./seed.nk" ] }, "user_credentials_file": { "type": "string", "description": "An optional file containing user credentials which consist of an user JWT and corresponding NKey seed.", "examples": [ "./user.creds" ] }, "user_jwt": { "type": "string", "description": "An optional plain text user JWT (given along with the corresponding user NKey Seed)." }, "user_nkey_seed": { "type": "string", "description": "An optional plain text user NKey Seed (given along with the corresponding user JWT)." } } }, "headers": { "type": "object", "description": "Explicit message headers to add to messages. Default: map[]", "examples": [ { "Content-Type": "application/json", "Timestamp": "${!metadata(\"Timestamp\").string()}" } ] }, "inject_tracing_map": { "type": "string", "description": "EXPERIMENTAL: A [Bloblang mapping](/docs/guides/bloblang/about) used to inject an object containing tracing propagation information into outbound messages. The specification of the injected fields will match the format used by the service wide tracer.", "examples": [ "meta = @.assign(this)", "root.meta.span = this" ] }, "max_in_flight": { "type": "integer", "description": "The maximum number of messages to have in flight at a given time. Increase this to improve throughput. Default: 64" }, "metadata": { "type": "object", "description": "Determine which (if any) metadata values should be added to messages as headers.", "properties": { "include_patterns": { "type": "array", "description": "Provide a list of explicit metadata key regular expression (re2) patterns to match against. Default: []", "items": { "type": "string" }, "examples": [ [ ".*" ], [ "_timestamp_unix$" ] ] }, "include_prefixes": { "type": "array", "description": "Provide a list of explicit metadata key prefixes to match against. Default: []", "items": { "type": "string" }, "examples": [ [ "foo_", "bar_" ], [ "kafka_" ], [ "content-" ] ] } } }, "subject": { "type": "string", "description": "The subject to publish to.", "examples": [ "foo.bar.baz" ] }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "urls": { "type": "array", "description": "A list of URLs to connect to. If an item of the list contains commas it will be expanded into multiple URLs.", "items": { "type": "string" }, "examples": [ [ "nats://127.0.0.1:4222" ], [ "nats://username:password@127.0.0.1:4222" ] ] } } }, "nats_jetstream": { "type": "object", "properties": { "auth": { "type": "object", "description": "Optional configuration of NATS authentication parameters.", "properties": { "nkey_file": { "type": "string", "description": "An optional file containing a NKey seed.", "examples": [ "./seed.nk" ] }, "user_credentials_file": { "type": "string", "description": "An optional file containing user credentials which consist of an user JWT and corresponding NKey seed.", "examples": [ "./user.creds" ] }, "user_jwt": { "type": "string", "description": "An optional plain text user JWT (given along with the corresponding user NKey Seed)." }, "user_nkey_seed": { "type": "string", "description": "An optional plain text user NKey Seed (given along with the corresponding user JWT)." } } }, "headers": { "type": "object", "description": "Explicit message headers to add to messages. Default: map[]", "examples": [ { "Content-Type": "application/json", "Timestamp": "${!metadata(\"Timestamp\").string()}" } ] }, "inject_tracing_map": { "type": "string", "description": "EXPERIMENTAL: A [Bloblang mapping](/docs/guides/bloblang/about) used to inject an object containing tracing propagation information into outbound messages. The specification of the injected fields will match the format used by the service wide tracer.", "examples": [ "meta = @.assign(this)", "root.meta.span = this" ] }, "max_in_flight": { "type": "integer", "description": "The maximum number of messages to have in flight at a given time. Increase this to improve throughput. Default: 1024" }, "metadata": { "type": "object", "description": "Determine which (if any) metadata values should be added to messages as headers.", "properties": { "include_patterns": { "type": "array", "description": "Provide a list of explicit metadata key regular expression (re2) patterns to match against. Default: []", "items": { "type": "string" }, "examples": [ [ ".*" ], [ "_timestamp_unix$" ] ] }, "include_prefixes": { "type": "array", "description": "Provide a list of explicit metadata key prefixes to match against. Default: []", "items": { "type": "string" }, "examples": [ [ "foo_", "bar_" ], [ "kafka_" ], [ "content-" ] ] } } }, "subject": { "type": "string", "description": "A subject to write to.", "examples": [ "foo.bar.baz", "${! metadata(\"kafka_topic\") }", "foo.${! json(\"meta.type\") }" ] }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "urls": { "type": "array", "description": "A list of URLs to connect to. If an item of the list contains commas it will be expanded into multiple URLs.", "items": { "type": "string" }, "examples": [ [ "nats://127.0.0.1:4222" ], [ "nats://username:password@127.0.0.1:4222" ] ] } } }, "nats_kv": { "type": "object", "properties": { "auth": { "type": "object", "description": "Optional configuration of NATS authentication parameters.", "properties": { "nkey_file": { "type": "string", "description": "An optional file containing a NKey seed.", "examples": [ "./seed.nk" ] }, "user_credentials_file": { "type": "string", "description": "An optional file containing user credentials which consist of an user JWT and corresponding NKey seed.", "examples": [ "./user.creds" ] }, "user_jwt": { "type": "string", "description": "An optional plain text user JWT (given along with the corresponding user NKey Seed)." }, "user_nkey_seed": { "type": "string", "description": "An optional plain text user NKey Seed (given along with the corresponding user JWT)." } } }, "bucket": { "type": "string", "description": "The name of the KV bucket.", "examples": [ "my_kv_bucket" ] }, "key": { "type": "string", "description": "The key for each message.", "examples": [ "foo", "foo.bar.baz", "foo.${! json(\"meta.type\") }" ] }, "max_in_flight": { "type": "integer", "description": "The maximum number of messages to have in flight at a given time. Increase this to improve throughput. Default: 1024" }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "urls": { "type": "array", "description": "A list of URLs to connect to. If an item of the list contains commas it will be expanded into multiple URLs.", "items": { "type": "string" }, "examples": [ [ "nats://127.0.0.1:4222" ], [ "nats://username:password@127.0.0.1:4222" ] ] } } }, "nats_object_store": { "type": "object", "properties": { "auth": { "type": "object", "description": "Optional configuration of NATS authentication parameters.", "properties": { "nkey_file": { "type": "string", "description": "An optional file containing a NKey seed.", "examples": [ "./seed.nk" ] }, "user_credentials_file": { "type": "string", "description": "An optional file containing user credentials which consist of an user JWT and corresponding NKey seed.", "examples": [ "./user.creds" ] }, "user_jwt": { "type": "string", "description": "An optional plain text user JWT (given along with the corresponding user NKey Seed)." }, "user_nkey_seed": { "type": "string", "description": "An optional plain text user NKey Seed (given along with the corresponding user JWT)." } } }, "bucket": { "type": "string", "description": "The name of the object store bucket.", "examples": [ "my_bucket" ] }, "max_in_flight": { "type": "integer", "description": "The maximum number of messages to have in flight at a given time. Increase this to improve throughput. Default: 64" }, "object_name": { "type": "string", "description": "The object name for each message." }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "urls": { "type": "array", "description": "A list of URLs to connect to. If an item of the list contains commas it will be expanded into multiple URLs.", "items": { "type": "string" }, "examples": [ [ "nats://127.0.0.1:4222" ], [ "nats://username:password@127.0.0.1:4222" ] ] } } }, "nats_stream": { "type": "object", "properties": { "auth": { "type": "object", "description": "Optional configuration of NATS authentication parameters.", "properties": { "nkey_file": { "type": "string", "description": "An optional file containing a NKey seed.", "examples": [ "./seed.nk" ] }, "user_credentials_file": { "type": "string", "description": "An optional file containing user credentials which consist of an user JWT and corresponding NKey seed.", "examples": [ "./user.creds" ] }, "user_jwt": { "type": "string", "description": "An optional plain text user JWT (given along with the corresponding user NKey Seed)." }, "user_nkey_seed": { "type": "string", "description": "An optional plain text user NKey Seed (given along with the corresponding user JWT)." } } }, "client_id": { "type": "string", "description": "The client ID to connect with. Default: " }, "cluster_id": { "type": "string", "description": "The cluster ID to publish to." }, "inject_tracing_map": { "type": "string", "description": "EXPERIMENTAL: A [Bloblang mapping](/docs/guides/bloblang/about) used to inject an object containing tracing propagation information into outbound messages. The specification of the injected fields will match the format used by the service wide tracer.", "examples": [ "meta = @.assign(this)", "root.meta.span = this" ] }, "max_in_flight": { "type": "integer", "description": "The maximum number of messages to have in flight at a given time. Increase this to improve throughput. Default: 64" }, "subject": { "type": "string", "description": "The subject to publish to." }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "urls": { "type": "array", "description": "A list of URLs to connect to. If an item of the list contains commas it will be expanded into multiple URLs.", "items": { "type": "string" }, "examples": [ [ "nats://127.0.0.1:4222" ], [ "nats://username:password@127.0.0.1:4222" ] ] } } }, "nsq": { "type": "object", "properties": { "auth_secret": { "type": "string", "description": "An optional secret for NSQ authentication (requires nsqd 0.2.29+)." }, "max_in_flight": { "type": "integer", "description": "The maximum number of messages to have in flight at a given time. Increase this to improve throughput. Default: 64" }, "nsqd_tcp_address": { "type": "string", "description": "The address of the target NSQD server." }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "topic": { "type": "string", "description": "The topic to publish to." }, "user_agent": { "type": "string", "description": "A user agent to assume when connecting." } } }, "opensearch": { "type": "object", "properties": { "action": { "type": "string", "description": "The action to take on the document. This field must resolve to one of the following action types: `create`, `index`, `update` or `delete`. To write to a data stream, the action must be set to `create`." }, "aws": { "type": "object", "description": "Enables and customises connectivity to Amazon Elastic Service.", "properties": { "credentials": { "type": "object", "description": "Optional manual configuration of AWS credentials to use. More information can be found [in this document](/docs/guides/cloud/aws).", "properties": { "expiry_window": { "type": "string", "description": "Allow the credentials to trigger refreshing prior to the credentials actually expiring. This is beneficial so race conditions with expiring credentials do not cause requests to fail. For example '10s' would refresh credentials ten seconds before expiration. Setting to a duration of `0` disables the expiry window. Default: " }, "from_ec2_role": { "type": "boolean", "description": "Use the credentials of a host EC2 machine configured to assume [an IAM role associated with the instance](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html). Default: false" }, "id": { "type": "string", "description": "The ID of credentials to use. Default: " }, "profile": { "type": "string", "description": "A profile from `~/.aws/credentials` to use. Default: " }, "role": { "type": "string", "description": "A role ARN to assume. Default: " }, "role_external_id": { "type": "string", "description": "An external ID to provide when assuming a role. Default: " }, "secret": { "type": "string", "description": "The secret for the credentials being used. Default: " }, "token": { "type": "string", "description": "The token for the credentials being used, required when using short term credentials. Default: " } } }, "enabled": { "type": "boolean", "description": "Whether to connect to Amazon Elastic Service. Default: false" }, "endpoint": { "type": "string", "description": "Allows you to specify a custom endpoint for the AWS API. Default: " }, "region": { "type": "string", "description": "The AWS region to target. Default: " } } }, "basic_auth": { "type": "object", "description": "Allows you to specify basic authentication.", "properties": { "enabled": { "type": "boolean", "description": "Whether to use basic authentication in requests. Default: false" }, "password": { "type": "string", "description": "A password to authenticate with. Default: " }, "username": { "type": "string", "description": "A username to authenticate as. Default: " } } }, "batching": { "type": "object", "description": "\nAllows you to configure a [batching policy](/docs/configuration/batching).", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] }, "id": { "type": "string", "description": "The ID for indexed messages. Interpolation should be used in order to create a unique ID for each message.", "examples": [ "${!counter()}-${!timestamp_unix()}" ] }, "index": { "type": "string", "description": "The index to place messages." }, "max_in_flight": { "type": "integer", "description": "The maximum number of messages to have in flight at a given time. Increase this to improve throughput. Default: 64" }, "pipeline": { "type": "string", "description": "An optional pipeline id to preprocess incoming documents. Default: " }, "routing": { "type": "string", "description": "The routing key to use for the document. Default: " }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "urls": { "type": "array", "description": "A list of URLs to connect to. If an item of the list contains commas it will be expanded into multiple URLs.", "items": { "type": "string" }, "examples": [ [ "http://localhost:9200" ] ] } } }, "pulsar": { "type": "object", "properties": { "auth": { "type": "object", "description": "Optional configuration of Pulsar authentication methods.", "properties": { "oauth2": { "type": "object", "description": "Parameters for Pulsar OAuth2 authentication.", "properties": { "audience": { "type": "string", "description": "OAuth2 audience. Default: " }, "enabled": { "type": "boolean", "description": "Whether OAuth2 is enabled. Default: false" }, "issuer_url": { "type": "string", "description": "OAuth2 issuer URL. Default: " }, "private_key_file": { "type": "string", "description": "The path to a file containing a private key. Default: " } } }, "token": { "type": "object", "description": "Parameters for Pulsar Token authentication.", "properties": { "enabled": { "type": "boolean", "description": "Whether Token Auth is enabled. Default: false" }, "token": { "type": "string", "description": "Actual base64 encoded token. Default: " } } } } }, "key": { "type": "string", "description": "The key to publish messages with. Default: " }, "max_in_flight": { "type": "integer", "description": "The maximum number of messages to have in flight at a given time. Increase this to improve throughput. Default: 64" }, "ordering_key": { "type": "string", "description": "The ordering key to publish messages with. Default: " }, "tls": { "type": "object", "description": "Specify the path to a custom CA certificate to trust broker TLS service.", "properties": { "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] } } }, "topic": { "type": "string", "description": "The topic to publish to." }, "url": { "type": "string", "description": "A URL to connect to.", "examples": [ "pulsar://localhost:6650", "pulsar://pulsar.us-west.example.com:6650", "pulsar+ssl://pulsar.us-west.example.com:6651" ] } } }, "pusher": { "type": "object", "properties": { "appId": { "type": "string", "description": "Pusher app id" }, "batching": { "type": "object", "description": "maximum batch size is 10 (limit of the pusher library)", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] }, "channel": { "type": "string", "description": "Pusher channel to publish to. Interpolation functions can also be used", "examples": [ "my_channel", "${!json(\"id\")}" ] }, "cluster": { "type": "string", "description": "Pusher cluster" }, "event": { "type": "string", "description": "Event to publish to" }, "key": { "type": "string", "description": "Pusher key" }, "max_in_flight": { "type": "integer", "description": "The maximum number of parallel message batches to have in flight at any given time. Default: 1" }, "secret": { "type": "string", "description": "Pusher secret" }, "secure": { "type": "boolean", "description": "Enable SSL encryption Default: true" } } }, "questdb": { "type": "object", "properties": { "address": { "type": "string", "description": "Address of the QuestDB server's HTTP port (excluding protocol)", "examples": [ "localhost:9000" ] }, "batching": { "type": "object", "description": "\nAllows you to configure a [batching policy](/docs/configuration/batching).", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] }, "designated_timestamp_field": { "type": "string", "description": "Name of the designated timestamp field" }, "designated_timestamp_unit": { "type": "string", "description": "Designated timestamp field units Default: auto" }, "doubles": { "type": "array", "description": "Columns that should be double type, (int is default)", "items": { "type": "string" } }, "error_on_empty_messages": { "type": "boolean", "description": "Mark a message as errored if it is empty after field validation Default: false" }, "max_in_flight": { "type": "integer", "description": "The maximum number of messages to have in flight at a given time. Increase this to improve throughput. Default: 64" }, "password": { "type": "string", "description": "Password for HTTP basic auth" }, "request_min_throughput": { "type": "integer", "description": "Minimum expected throughput in bytes per second for HTTP requests. If the throughput is lower than this value, the connection will time out. This is used to calculate an additional timeout on top of request_timeout. This is useful for large requests. You can set this value to 0 to disable this logic." }, "request_timeout": { "type": "string", "description": "The time to wait for a response from the server. This is in addition to the calculation derived from the request_min_throughput parameter." }, "retry_timeout": { "type": "string", "description": "The time to continue retrying after a failed HTTP request. The interval between retries is an exponential backoff starting at 10ms and doubling after each failed attempt up to a maximum of 1 second." }, "symbols": { "type": "array", "description": "Columns that should be the SYMBOL type (string values default to STRING)", "items": { "type": "string" } }, "table": { "type": "string", "description": "Destination table", "examples": [ "trades" ] }, "timestamp_string_fields": { "type": "array", "description": "String fields with textual timestamps", "items": { "type": "string" } }, "timestamp_string_format": { "type": "string", "description": "Timestamp format, used when parsing timestamp string fields. Specified in golang's time.Parse layout Default: Jan _2 15:04:05.000000Z0700" }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "token": { "type": "string", "description": "Bearer token for HTTP auth (takes precedence over basic auth username \u0026 password)" }, "username": { "type": "string", "description": "Username for HTTP basic auth" } } }, "redis_hash": { "type": "object", "properties": { "fields": { "type": "object", "description": "A map of key/value pairs to set as hash fields. Default: map[]" }, "key": { "type": "string", "description": "The key for each message, function interpolations should be used to create a unique key per message.", "examples": [ "${! @.kafka_key )}", "${! this.doc.id }", "${! count(\"msgs\") }" ] }, "kind": { "type": "string", "description": "Specifies a simple, cluster-aware, or failover-aware redis client. Default: simple" }, "master": { "type": "string", "description": "Name of the redis master when `kind` is `failover` Default: ", "examples": [ "mymaster" ] }, "max_in_flight": { "type": "integer", "description": "The maximum number of messages to have in flight at a given time. Increase this to improve throughput. Default: 64" }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.\n\n**Troubleshooting**\n\nSome cloud hosted instances of Redis (such as Azure Cache) might need some hand holding in order to establish stable connections. Unfortunately, it is often the case that TLS issues will manifest as generic error messages such as \"i/o timeout\". If you're using TLS and are seeing connectivity problems consider setting `enable_renegotiation` to `true`, and ensuring that the server supports at least TLS version 1.2.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "url": { "type": "string", "description": "The URL of the target Redis server. Database is optional and is supplied as the URL path.", "examples": [ "redis://:6397", "redis://localhost:6379", "redis://foousername:foopassword@redisplace:6379", "redis://:foopassword@redisplace:6379", "redis://localhost:6379/1", "redis://localhost:6379/1,redis://localhost:6380/1" ] }, "walk_json_object": { "type": "boolean", "description": "Whether to walk each message as a JSON object and add each key/value pair to the list of hash fields to set. Default: false" }, "walk_metadata": { "type": "boolean", "description": "Whether all metadata fields of messages should be walked and added to the list of hash fields to set. Default: false" } } }, "redis_list": { "type": "object", "properties": { "batching": { "type": "object", "description": "\nAllows you to configure a [batching policy](/docs/configuration/batching).", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] }, "command": { "type": "string", "description": "The command used to push elements to the Redis list Default: rpush" }, "key": { "type": "string", "description": "The key for each message, function interpolations can be optionally used to create a unique key per message.", "examples": [ "some_list", "${! @.kafka_key )}", "${! this.doc.id }", "${! count(\"msgs\") }" ] }, "kind": { "type": "string", "description": "Specifies a simple, cluster-aware, or failover-aware redis client. Default: simple" }, "master": { "type": "string", "description": "Name of the redis master when `kind` is `failover` Default: ", "examples": [ "mymaster" ] }, "max_in_flight": { "type": "integer", "description": "The maximum number of messages to have in flight at a given time. Increase this to improve throughput. Default: 64" }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.\n\n**Troubleshooting**\n\nSome cloud hosted instances of Redis (such as Azure Cache) might need some hand holding in order to establish stable connections. Unfortunately, it is often the case that TLS issues will manifest as generic error messages such as \"i/o timeout\". If you're using TLS and are seeing connectivity problems consider setting `enable_renegotiation` to `true`, and ensuring that the server supports at least TLS version 1.2.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "url": { "type": "string", "description": "The URL of the target Redis server. Database is optional and is supplied as the URL path.", "examples": [ "redis://:6397", "redis://localhost:6379", "redis://foousername:foopassword@redisplace:6379", "redis://:foopassword@redisplace:6379", "redis://localhost:6379/1", "redis://localhost:6379/1,redis://localhost:6380/1" ] } } }, "redis_pubsub": { "type": "object", "properties": { "batching": { "type": "object", "description": "\nAllows you to configure a [batching policy](/docs/configuration/batching).", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] }, "channel": { "type": "string", "description": "The channel to publish messages to." }, "kind": { "type": "string", "description": "Specifies a simple, cluster-aware, or failover-aware redis client. Default: simple" }, "master": { "type": "string", "description": "Name of the redis master when `kind` is `failover` Default: ", "examples": [ "mymaster" ] }, "max_in_flight": { "type": "integer", "description": "The maximum number of messages to have in flight at a given time. Increase this to improve throughput. Default: 64" }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.\n\n**Troubleshooting**\n\nSome cloud hosted instances of Redis (such as Azure Cache) might need some hand holding in order to establish stable connections. Unfortunately, it is often the case that TLS issues will manifest as generic error messages such as \"i/o timeout\". If you're using TLS and are seeing connectivity problems consider setting `enable_renegotiation` to `true`, and ensuring that the server supports at least TLS version 1.2.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "url": { "type": "string", "description": "The URL of the target Redis server. Database is optional and is supplied as the URL path.", "examples": [ "redis://:6397", "redis://localhost:6379", "redis://foousername:foopassword@redisplace:6379", "redis://:foopassword@redisplace:6379", "redis://localhost:6379/1", "redis://localhost:6379/1,redis://localhost:6380/1" ] } } }, "redis_streams": { "type": "object", "properties": { "batching": { "type": "object", "description": "\nAllows you to configure a [batching policy](/docs/configuration/batching).", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] }, "body_key": { "type": "string", "description": "A key to set the raw body of the message to. Default: body" }, "kind": { "type": "string", "description": "Specifies a simple, cluster-aware, or failover-aware redis client. Default: simple" }, "master": { "type": "string", "description": "Name of the redis master when `kind` is `failover` Default: ", "examples": [ "mymaster" ] }, "max_in_flight": { "type": "integer", "description": "The maximum number of messages to have in flight at a given time. Increase this to improve throughput. Default: 64" }, "max_length": { "type": "integer", "description": "When greater than zero enforces a rough cap on the length of the target stream. Default: 0" }, "metadata": { "type": "object", "description": "Specify criteria for which metadata values are included in the message body.", "properties": { "exclude_prefixes": { "type": "array", "description": "Provide a list of explicit metadata key prefixes to be excluded when adding metadata to sent messages. Default: []", "items": { "type": "string" } } } }, "stream": { "type": "string", "description": "The stream to add messages to." }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.\n\n**Troubleshooting**\n\nSome cloud hosted instances of Redis (such as Azure Cache) might need some hand holding in order to establish stable connections. Unfortunately, it is often the case that TLS issues will manifest as generic error messages such as \"i/o timeout\". If you're using TLS and are seeing connectivity problems consider setting `enable_renegotiation` to `true`, and ensuring that the server supports at least TLS version 1.2.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "url": { "type": "string", "description": "The URL of the target Redis server. Database is optional and is supplied as the URL path.", "examples": [ "redis://:6397", "redis://localhost:6379", "redis://foousername:foopassword@redisplace:6379", "redis://:foopassword@redisplace:6379", "redis://localhost:6379/1", "redis://localhost:6379/1,redis://localhost:6380/1" ] } } }, "reject": { "type": "string", "description": " Default: " }, "reject_errored": { "type": "object" }, "resource": { "type": "string", "description": " Default: " }, "retry": { "type": "object", "properties": { "backoff": { "type": "object", "description": "Control time intervals between retry attempts.", "properties": { "initial_interval": { "type": "string", "description": "The initial period to wait between retry attempts. Default: 500ms" }, "max_elapsed_time": { "type": "string", "description": "The maximum period to wait before retry attempts are abandoned. If zero then no limit is used. Default: 0s" }, "max_interval": { "type": "string", "description": "The maximum period to wait between retry attempts. Default: 3s" } } }, "max_retries": { "type": "integer", "description": "The maximum number of retries before giving up on the request. If set to zero there is no discrete limit. Default: 0" }, "output": { "type": "object", "description": "A child output." } } }, "s2": { "type": "object", "properties": { "access_token": { "type": "string", "description": "Access token for S2 account" }, "basin": { "type": "string", "description": "Basin name" }, "batching": { "type": "object", "description": "\nAllows you to configure a [batching policy](/docs/configuration/batching).", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] }, "fencing_token": { "type": "string", "description": "Enforce a fencing token (base64 encoded)", "examples": [ "aGVsbG8gczI=" ] }, "max_in_flight": { "type": "integer", "description": "The maximum number of messages to have in flight at a given time. Increase this to improve throughput. Default: 64" }, "stream": { "type": "string", "description": "Stream name" } } }, "sftp": { "type": "object", "properties": { "address": { "type": "string", "description": "The address of the server to connect to." }, "codec": { "type": "string", "description": "The way in which the bytes of messages should be written out into the output data stream. It's possible to write lines using a custom delimiter with the `delim:x` codec, where x is the character sequence custom delimiter. Default: all-bytes", "examples": [ "lines", "delim:\t", "delim:foobar" ] }, "credentials": { "type": "object", "description": "The credentials to use to log into the target server.", "properties": { "password": { "type": "string", "description": "The password for the username to connect to the SFTP server. Default: " }, "private_key_file": { "type": "string", "description": "The private key for the username to connect to the SFTP server. Default: " }, "private_key_pass": { "type": "string", "description": "Optional passphrase for private key. Default: " }, "username": { "type": "string", "description": "The username to connect to the SFTP server. Default: " } } }, "max_in_flight": { "type": "integer", "description": "The maximum number of messages to have in flight at a given time. Increase this to improve throughput. Default: 64" }, "path": { "type": "string", "description": "The file to save the messages to on the server." } } }, "slack_webhook": { "type": "object", "properties": { "timeout": { "type": "string", "description": "The maximum time to wait before abandoning a request (and trying again). Default: 5s" }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "webhook": { "type": "string", "description": "Slack webhook URL to post messages", "examples": [ "https://hooks.slack.com/services/T00000000/B00000000/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" ] } } }, "snowflake_put": { "type": "object", "properties": { "account": { "type": "string", "description": "Account name, which is the same as the Account Identifier\nas described [here](https://docs.snowflake.com/en/user-guide/admin-account-identifier.html#where-are-account-identifiers-used).\nHowever, when using an [Account Locator](https://docs.snowflake.com/en/user-guide/admin-account-identifier.html#using-an-account-locator-as-an-identifier),\nthe Account Identifier is formatted as `\u003caccount_locator\u003e.\u003cregion_id\u003e.\u003ccloud\u003e` and this field needs to be\npopulated using the `\u003caccount_locator\u003e` part.\n" }, "batching": { "type": "object", "description": "\nAllows you to configure a [batching policy](/docs/configuration/batching).", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] }, "client_session_keep_alive": { "type": "boolean", "description": "Enable Snowflake keepalive mechanism to prevent the client session from expiring after 4 hours (error 390114). Default: false" }, "cloud": { "type": "string", "description": "Optional cloud platform field which needs to be populated\nwhen using an [Account Locator](https://docs.snowflake.com/en/user-guide/admin-account-identifier.html#using-an-account-locator-as-an-identifier)\nand it must be set to the `\u003ccloud\u003e` part of the Account Identifier\n(`\u003caccount_locator\u003e.\u003cregion_id\u003e.\u003ccloud\u003e`).\n", "examples": [ "aws", "gcp", "azure" ] }, "compression": { "type": "string", "description": "Compression type. Default: AUTO" }, "database": { "type": "string", "description": "Database." }, "file_extension": { "type": "string", "description": "Stage file extension. Will be derived from the configured `compression` if not set or empty. Default: ", "examples": [ "csv", "parquet" ] }, "file_name": { "type": "string", "description": "Stage file name. Will be equal to the Request ID if not set or empty. Default: " }, "max_in_flight": { "type": "integer", "description": "The maximum number of parallel message batches to have in flight at any given time. Default: 1" }, "password": { "type": "string", "description": "An optional password." }, "path": { "type": "string", "description": "Stage path. Default: " }, "private_key_file": { "type": "string", "description": "The path to a file containing the private SSH key." }, "private_key_pass": { "type": "string", "description": "An optional private SSH key passphrase." }, "region": { "type": "string", "description": "Optional region field which needs to be populated when using\nan [Account Locator](https://docs.snowflake.com/en/user-guide/admin-account-identifier.html#using-an-account-locator-as-an-identifier)\nand it must be set to the `\u003cregion_id\u003e` part of the Account Identifier\n(`\u003caccount_locator\u003e.\u003cregion_id\u003e.\u003ccloud\u003e`).\n", "examples": [ "us-west-2" ] }, "request_id": { "type": "string", "description": "Request ID. Will be assigned a random UUID (v4) string if not set or empty. Default: " }, "role": { "type": "string", "description": "Role." }, "schema": { "type": "string", "description": "Schema." }, "snowpipe": { "type": "string", "description": "An optional Snowpipe name. Use the `\u003csnowpipe\u003e` part from `\u003cdatabase\u003e.\u003cschema\u003e.\u003csnowpipe\u003e`." }, "stage": { "type": "string", "description": "Stage name. Use either one of the\n[supported](https://docs.snowflake.com/en/user-guide/data-load-local-file-system-create-stage.html) stage types." }, "upload_parallel_threads": { "type": "integer", "description": "Specifies the number of threads to use for uploading files. Default: 4" }, "user": { "type": "string", "description": "Username." }, "warehouse": { "type": "string", "description": "Warehouse." } } }, "socket": { "type": "object", "properties": { "address": { "type": "string", "description": "The address to connect to.", "examples": [ "/tmp/bento.sock", "127.0.0.1:6000" ] }, "codec": { "type": "string", "description": "The way in which the bytes of messages should be written out into the output data stream. It's possible to write lines using a custom delimiter with the `delim:x` codec, where x is the character sequence custom delimiter. Default: lines", "examples": [ "lines", "delim:\t", "delim:foobar" ] }, "network": { "type": "string", "description": "A network type to connect as." } } }, "splunk_hec": { "type": "object", "properties": { "batching_byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If 0 disables size based batching. Splunk Cloud recommends limiting content length of HEC payload to 1 MB. Default: 1000000" }, "batching_count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If 0 disables count based batching. Default: 100" }, "batching_period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: 30s" }, "event_host": { "type": "string", "description": "Set the host value to assign to the event data. Overrides existing host field if present. Default: " }, "event_index": { "type": "string", "description": "Set the index value to assign to the event data. Overrides existing index field if present. Default: " }, "event_source": { "type": "string", "description": "Set the source value to assign to the event data. Overrides existing source field if present. Default: " }, "event_sourcetype": { "type": "string", "description": "Set the sourcetype value to assign to the event data. Overrides existing sourcetype field if present. Default: " }, "gzip": { "type": "boolean", "description": "Enable gzip compression Default: false" }, "max_in_flight": { "type": "integer", "description": "The maximum number of parallel message batches to have in flight at any given time. Default: 64" }, "rate_limit": { "type": "string", "description": "An optional rate limit resource to restrict API requests with. Default: " }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" }, "token": { "type": "string", "description": "A bot token used for authentication." }, "url": { "type": "string", "description": "Full HTTP Endpoint Collector (HEC) URL, ie. https://foobar.splunkcloud.com/services/collector/event" } } }, "sql": { "type": "object", "properties": { "args_mapping": { "type": "string", "description": "An optional [Bloblang mapping](/docs/guides/bloblang/about) which should evaluate to an array of values matching in size to the number of placeholder arguments in the field `query`.", "examples": [ "root = [ this.cat.meow, this.doc.woofs[0] ]", "root = [ metadata(\"user.id\").string() ]" ] }, "batching": { "type": "object", "description": "\nAllows you to configure a [batching policy](/docs/configuration/batching).", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] }, "data_source_name": { "type": "string", "description": "Data source name." }, "driver": { "type": "string", "description": "A database [driver](#drivers) to use." }, "max_in_flight": { "type": "integer", "description": "The maximum number of inserts to run in parallel. Default: 64" }, "query": { "type": "string", "description": "The query to execute. The style of placeholder to use depends on the driver, some drivers require question marks (`?`) whereas others expect incrementing dollar signs (`$1`, `$2`, and so on) or colons (`:1`, `:2` and so on). The style to use is outlined in this table:\n\n| Driver | Placeholder Style |\n|---|---|\n| `clickhouse` | Dollar sign |\n| `mysql` | Question mark |\n| `postgres` | Dollar sign |\n| `mssql` | Question mark |\n| `sqlite` | Question mark |\n| `oracle` | Colon |\n| `snowflake` | Question mark |\n| `spanner` | Question mark |\n| `trino` | Question mark |\n| `gocosmos` | Colon |\n", "examples": [ "INSERT INTO footable (foo, bar, baz) VALUES (?, ?, ?);" ] } } }, "sql_insert": { "type": "object", "properties": { "args_mapping": { "type": "string", "description": "A [Bloblang mapping](/docs/guides/bloblang/about) which should evaluate to an array of values matching in size to the number of columns specified.", "examples": [ "root = [ this.cat.meow, this.doc.woofs[0] ]", "root = [ metadata(\"user.id\").string() ]" ] }, "azure": { "type": "object", "description": "Optional Fields that can be set to use Azure based authentication for Azure Postgres SQL", "properties": { "entra_enabled": { "type": "boolean", "description": "An optional field used to generate an entra token to connect to 'Azure Database for PostgreSQL flexible server', This will create a new connection string with the host, user and database from the DSN field - you may need to URL encode the dsn! The [Default Azure Credential Chain](https://learn.microsoft.com/en-gb/azure/developer/go/sdk/authentication/authentication-overview#defaultazurecredential) is used from the Azure SDK. Default: false" }, "token_request_options": { "type": "object", "properties": { "claims": { "type": "string", "description": "Set additional claims for the token. Default: " }, "enable_cae": { "type": "boolean", "description": "Indicates whether to enable Continuous Access Evaluation (CAE) for the requested token Default: false" }, "scopes": { "type": "array", "description": "Scopes contains the list of permission scopes required for the token. Default: [https://ossrdbms-aad.database.windows.net/.default]", "items": { "type": "string" } }, "tenant_id": { "type": "string", "description": "tenant_id identifies the tenant from which to request the token. azure credentials authenticate in their configured default tenants when this field isn't set. Default: " } } } } }, "batching": { "type": "object", "description": "\nAllows you to configure a [batching policy](/docs/configuration/batching).", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] }, "columns": { "type": "array", "description": "A list of columns to insert.", "items": { "type": "string" }, "examples": [ [ "foo", "bar", "baz" ] ] }, "conn_max_idle": { "type": "integer", "description": "An optional maximum number of connections in the idle connection pool. If conn_max_open is greater than 0 but less than the new conn_max_idle, then the new conn_max_idle will be reduced to match the conn_max_open limit. If `value \u003c= 0`, no idle connections are retained. The default max idle connections is currently 2. This may change in a future release. Default: 2" }, "conn_max_idle_time": { "type": "string", "description": "An optional maximum amount of time a connection may be idle. Expired connections may be closed lazily before reuse. If `value \u003c= 0`, connections are not closed due to a connections idle time." }, "conn_max_life_time": { "type": "string", "description": "An optional maximum amount of time a connection may be reused. Expired connections may be closed lazily before reuse. If `value \u003c= 0`, connections are not closed due to a connections age." }, "conn_max_open": { "type": "integer", "description": "An optional maximum number of open connections to the database. If conn_max_idle is greater than 0 and the new conn_max_open is less than conn_max_idle, then conn_max_idle will be reduced to match the new conn_max_open limit. If `value \u003c= 0`, then there is no limit on the number of open connections. The default is 0 (unlimited)." }, "credentials": { "type": "object", "description": "Optional manual configuration of AWS credentials to use. More information can be found [in this document](/docs/guides/cloud/aws).", "properties": { "expiry_window": { "type": "string", "description": "Allow the credentials to trigger refreshing prior to the credentials actually expiring. This is beneficial so race conditions with expiring credentials do not cause requests to fail. For example '10s' would refresh credentials ten seconds before expiration. Setting to a duration of `0` disables the expiry window. Default: " }, "from_ec2_role": { "type": "boolean", "description": "Use the credentials of a host EC2 machine configured to assume [an IAM role associated with the instance](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html). Default: false" }, "id": { "type": "string", "description": "The ID of credentials to use. Default: " }, "profile": { "type": "string", "description": "A profile from `~/.aws/credentials` to use. Default: " }, "role": { "type": "string", "description": "A role ARN to assume. Default: " }, "role_external_id": { "type": "string", "description": "An external ID to provide when assuming a role. Default: " }, "secret": { "type": "string", "description": "The secret for the credentials being used. Default: " }, "token": { "type": "string", "description": "The token for the credentials being used, required when using short term credentials. Default: " } } }, "driver": { "type": "string", "description": "A database [driver](#drivers) to use." }, "dsn": { "type": "string", "description": "A Data Source Name to identify the target database.\n\n#### Drivers\n\nThe following is a list of supported drivers, their placeholder style, and their respective DSN formats:\n\n| Driver | Data Source Name Format |\n|---|---|\n| `clickhouse` | [`clickhouse://[username[:password]@][netloc][:port]/dbname[?param1=value1\u0026...\u0026paramN=valueN]`](https://github.com/ClickHouse/clickhouse-go#dsn) |\n| `mysql` | `[username[:password]@][protocol[(address)]]/dbname[?param1=value1\u0026...\u0026paramN=valueN]` |\n| `postgres` | `postgres://[user[:password]@][netloc][:port][/dbname][?param1=value1\u0026...]` |\n| `mssql` | `sqlserver://[user[:password]@][netloc][:port][?database=dbname\u0026param1=value1\u0026...]` |\n| `sqlite` | `file:/path/to/filename.db[?param\u0026=value1\u0026...]` |\n| `oracle` | `oracle://[username[:password]@][netloc][:port]/service_name?server=server2\u0026server=server3` |\n| `snowflake` | `username[:password]@account_identifier/dbname/schemaname[?param1=value\u0026...\u0026paramN=valueN]` |\n| `spanner` | `projects/[project]/instances/[instance]/databases/dbname` |\n| `trino` | [`http[s]://user[:pass]@host[:port][?parameters]`](https://github.com/trinodb/trino-go-client#dsn-data-source-name) |\n| `gocosmos` | [`AccountEndpoint=\u003ccosmosdb-endpoint\u003e;AccountKey=\u003ccosmosdb-account-key\u003e[;TimeoutMs=\u003ctimeout-in-ms\u003e][;Version=\u003ccosmosdb-api-version\u003e][;DefaultDb/Db=\u003cdb-name\u003e][;AutoId=\u003ctrue/false\u003e][;InsecureSkipVerify=\u003ctrue/false\u003e]`](https://pkg.go.dev/github.com/microsoft/gocosmos#readme-example-usage) |\n\nPlease note that the `postgres` driver enforces SSL by default, you can override this with the parameter `sslmode=disable` if required.\n\nThe `snowflake` driver supports multiple DSN formats. Please consult [the docs](https://pkg.go.dev/github.com/snowflakedb/gosnowflake#hdr-Connection_String) for more details. For [key pair authentication](https://docs.snowflake.com/en/user-guide/key-pair-auth.html#configuring-key-pair-authentication), the DSN has the following format: `\u003csnowflake_user\u003e@\u003csnowflake_account\u003e/\u003cdb_name\u003e/\u003cschema_name\u003e?warehouse=\u003cwarehouse\u003e\u0026role=\u003crole\u003e\u0026authenticator=snowflake_jwt\u0026privateKey=\u003cbase64_url_encoded_private_key\u003e`, where the value for the `privateKey` parameter can be constructed from an unencrypted RSA private key file `rsa_key.p8` using `openssl enc -d -base64 -in rsa_key.p8 | basenc --base64url -w0` (you can use `gbasenc` insted of `basenc` on OSX if you install `coreutils` via Homebrew). If you have a password-encrypted private key, you can decrypt it using `openssl pkcs8 -in rsa_key_encrypted.p8 -out rsa_key.p8`. Also, make sure fields such as the username are URL-encoded.\n\nThe [`gocosmos`](https://pkg.go.dev/github.com/microsoft/gocosmos) driver is still experimental, but it has support for [hierarchical partition keys](https://learn.microsoft.com/en-us/azure/cosmos-db/hierarchical-partition-keys) as well as [cross-partition queries](https://learn.microsoft.com/en-us/azure/cosmos-db/nosql/how-to-query-container#cross-partition-query). Please refer to the [SQL notes](https://github.com/microsoft/gocosmos/blob/main/SQL.md) for details.", "examples": [ "clickhouse://username:password@host1:9000,host2:9000/database?dial_timeout=200ms\u0026max_execution_time=60", "foouser:foopassword@tcp(localhost:3306)/foodb", "postgres://foouser:foopass@localhost:5432/foodb?sslmode=disable", "oracle://foouser:foopass@localhost:1521/service_name" ] }, "endpoint": { "type": "string", "description": "Allows you to specify a custom endpoint for the AWS API. Default: " }, "iam_enabled": { "type": "boolean", "description": "An optional field used to generate an IAM authentication token to connect to an Amazon Relational Database (RDS) DB instance. This will overwrite the Password in the DSN with the generated token only if the drivers are `mysql` or `postgres`. Default: false" }, "init_files": { "type": "array", "description": "\nAn optional list of file paths containing SQL statements to execute immediately upon the first connection to the target database. This is a useful way to initialise tables before processing data. Glob patterns are supported, including super globs (double star).\n\nCare should be taken to ensure that the statements are idempotent, and therefore would not cause issues when run multiple times after service restarts. If both `init_statement` and `init_files` are specified the `init_statement` is executed _after_ the `init_files`.\n\nIf a statement fails for any reason a warning log will be emitted but the operation of this component will not be stopped.\n", "items": { "type": "string" }, "examples": [ [ "./init/*.sql" ], [ "./foo.sql", "./bar.sql" ] ] }, "init_statement": { "type": "string", "description": "\nAn optional SQL statement to execute immediately upon the first connection to the target database. This is a useful way to initialise tables before processing data. Care should be taken to ensure that the statement is idempotent, and therefore would not cause issues when run multiple times after service restarts.\n\nIf both `init_statement` and `init_files` are specified the `init_statement` is executed _after_ the `init_files`.\n\nIf the statement fails for any reason a warning log will be emitted but the operation of this component will not be stopped.\n", "examples": [ "\nCREATE TABLE IF NOT EXISTS some_table (\n foo varchar(50) not null,\n bar integer,\n baz varchar(50),\n primary key (foo)\n) WITHOUT ROWID;\n" ] }, "init_verify_conn": { "type": "boolean", "description": "Whether to verify the database connection on startup by performing a simple ping, by default this is disabled. Default: false" }, "max_in_flight": { "type": "integer", "description": "The maximum number of inserts to run in parallel. Default: 64" }, "prefix": { "type": "string", "description": "An optional prefix to prepend to the insert query (before INSERT)." }, "region": { "type": "string", "description": "The AWS region to target. Default: " }, "secret_name": { "type": "string", "description": "An optional field that can be used to get the Username + Password from AWS Secrets Manager. This will overwrite the Username + Password in the DSN with the values from the Secret only if the driver is set to `postgres`." }, "suffix": { "type": "string", "description": "An optional suffix to append to the insert query.", "examples": [ "ON CONFLICT (name) DO NOTHING" ] }, "table": { "type": "string", "description": "The table to insert to.", "examples": [ "foo" ] } } }, "sql_raw": { "type": "object", "properties": { "args_mapping": { "type": "string", "description": "An optional [Bloblang mapping](/docs/guides/bloblang/about) which should evaluate to an array of values matching in size to the number of placeholder arguments in the field `query`.", "examples": [ "root = [ this.cat.meow, this.doc.woofs[0] ]", "root = [ metadata(\"user.id\").string() ]" ] }, "azure": { "type": "object", "description": "Optional Fields that can be set to use Azure based authentication for Azure Postgres SQL", "properties": { "entra_enabled": { "type": "boolean", "description": "An optional field used to generate an entra token to connect to 'Azure Database for PostgreSQL flexible server', This will create a new connection string with the host, user and database from the DSN field - you may need to URL encode the dsn! The [Default Azure Credential Chain](https://learn.microsoft.com/en-gb/azure/developer/go/sdk/authentication/authentication-overview#defaultazurecredential) is used from the Azure SDK. Default: false" }, "token_request_options": { "type": "object", "properties": { "claims": { "type": "string", "description": "Set additional claims for the token. Default: " }, "enable_cae": { "type": "boolean", "description": "Indicates whether to enable Continuous Access Evaluation (CAE) for the requested token Default: false" }, "scopes": { "type": "array", "description": "Scopes contains the list of permission scopes required for the token. Default: [https://ossrdbms-aad.database.windows.net/.default]", "items": { "type": "string" } }, "tenant_id": { "type": "string", "description": "tenant_id identifies the tenant from which to request the token. azure credentials authenticate in their configured default tenants when this field isn't set. Default: " } } } } }, "batching": { "type": "object", "description": "\nAllows you to configure a [batching policy](/docs/configuration/batching).", "properties": { "byte_size": { "type": "integer", "description": "An amount of bytes at which the batch should be flushed. If `0` disables size based batching. Default: 0" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether a message should end a batch. Default: ", "examples": [ "this.type == \"end_of_transaction\"" ] }, "count": { "type": "integer", "description": "A number of messages at which the batch should be flushed. If `0` disables count based batching. Default: 0" }, "jitter": { "type": "number", "description": "A non-negative factor that adds random delay to batch flush intervals, where delay is determined uniformly at random between `0` and `jitter * period`. For example, with `period: 100ms` and `jitter: 0.1`, each flush will be delayed by a random duration between `0-10ms`. Default: 0", "examples": [ 0.01, 0.1, 1 ] }, "period": { "type": "string", "description": "A period in which an incomplete batch should be flushed regardless of its size. Default: ", "examples": [ "1s", "1m", "500ms" ] }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about) to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op.", "items": { "type": "object" }, "examples": [ [ { "archive": { "format": "concatenate" } } ], [ { "archive": { "format": "lines" } } ], [ { "archive": { "format": "json_array" } } ] ] } }, "examples": [ { "byte_size": 5000, "count": 0, "period": "1s" }, { "count": 10, "period": "1s" }, { "check": "this.contains(\"END BATCH\")", "count": 0, "period": "1m" }, { "count": 10, "jitter": 0.1, "period": "10s" } ] }, "conn_max_idle": { "type": "integer", "description": "An optional maximum number of connections in the idle connection pool. If conn_max_open is greater than 0 but less than the new conn_max_idle, then the new conn_max_idle will be reduced to match the conn_max_open limit. If `value \u003c= 0`, no idle connections are retained. The default max idle connections is currently 2. This may change in a future release. Default: 2" }, "conn_max_idle_time": { "type": "string", "description": "An optional maximum amount of time a connection may be idle. Expired connections may be closed lazily before reuse. If `value \u003c= 0`, connections are not closed due to a connections idle time." }, "conn_max_life_time": { "type": "string", "description": "An optional maximum amount of time a connection may be reused. Expired connections may be closed lazily before reuse. If `value \u003c= 0`, connections are not closed due to a connections age." }, "conn_max_open": { "type": "integer", "description": "An optional maximum number of open connections to the database. If conn_max_idle is greater than 0 and the new conn_max_open is less than conn_max_idle, then conn_max_idle will be reduced to match the new conn_max_open limit. If `value \u003c= 0`, then there is no limit on the number of open connections. The default is 0 (unlimited)." }, "credentials": { "type": "object", "description": "Optional manual configuration of AWS credentials to use. More information can be found [in this document](/docs/guides/cloud/aws).", "properties": { "expiry_window": { "type": "string", "description": "Allow the credentials to trigger refreshing prior to the credentials actually expiring. This is beneficial so race conditions with expiring credentials do not cause requests to fail. For example '10s' would refresh credentials ten seconds before expiration. Setting to a duration of `0` disables the expiry window. Default: " }, "from_ec2_role": { "type": "boolean", "description": "Use the credentials of a host EC2 machine configured to assume [an IAM role associated with the instance](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html). Default: false" }, "id": { "type": "string", "description": "The ID of credentials to use. Default: " }, "profile": { "type": "string", "description": "A profile from `~/.aws/credentials` to use. Default: " }, "role": { "type": "string", "description": "A role ARN to assume. Default: " }, "role_external_id": { "type": "string", "description": "An external ID to provide when assuming a role. Default: " }, "secret": { "type": "string", "description": "The secret for the credentials being used. Default: " }, "token": { "type": "string", "description": "The token for the credentials being used, required when using short term credentials. Default: " } } }, "driver": { "type": "string", "description": "A database [driver](#drivers) to use." }, "dsn": { "type": "string", "description": "A Data Source Name to identify the target database.\n\n#### Drivers\n\nThe following is a list of supported drivers, their placeholder style, and their respective DSN formats:\n\n| Driver | Data Source Name Format |\n|---|---|\n| `clickhouse` | [`clickhouse://[username[:password]@][netloc][:port]/dbname[?param1=value1\u0026...\u0026paramN=valueN]`](https://github.com/ClickHouse/clickhouse-go#dsn) |\n| `mysql` | `[username[:password]@][protocol[(address)]]/dbname[?param1=value1\u0026...\u0026paramN=valueN]` |\n| `postgres` | `postgres://[user[:password]@][netloc][:port][/dbname][?param1=value1\u0026...]` |\n| `mssql` | `sqlserver://[user[:password]@][netloc][:port][?database=dbname\u0026param1=value1\u0026...]` |\n| `sqlite` | `file:/path/to/filename.db[?param\u0026=value1\u0026...]` |\n| `oracle` | `oracle://[username[:password]@][netloc][:port]/service_name?server=server2\u0026server=server3` |\n| `snowflake` | `username[:password]@account_identifier/dbname/schemaname[?param1=value\u0026...\u0026paramN=valueN]` |\n| `spanner` | `projects/[project]/instances/[instance]/databases/dbname` |\n| `trino` | [`http[s]://user[:pass]@host[:port][?parameters]`](https://github.com/trinodb/trino-go-client#dsn-data-source-name) |\n| `gocosmos` | [`AccountEndpoint=\u003ccosmosdb-endpoint\u003e;AccountKey=\u003ccosmosdb-account-key\u003e[;TimeoutMs=\u003ctimeout-in-ms\u003e][;Version=\u003ccosmosdb-api-version\u003e][;DefaultDb/Db=\u003cdb-name\u003e][;AutoId=\u003ctrue/false\u003e][;InsecureSkipVerify=\u003ctrue/false\u003e]`](https://pkg.go.dev/github.com/microsoft/gocosmos#readme-example-usage) |\n\nPlease note that the `postgres` driver enforces SSL by default, you can override this with the parameter `sslmode=disable` if required.\n\nThe `snowflake` driver supports multiple DSN formats. Please consult [the docs](https://pkg.go.dev/github.com/snowflakedb/gosnowflake#hdr-Connection_String) for more details. For [key pair authentication](https://docs.snowflake.com/en/user-guide/key-pair-auth.html#configuring-key-pair-authentication), the DSN has the following format: `\u003csnowflake_user\u003e@\u003csnowflake_account\u003e/\u003cdb_name\u003e/\u003cschema_name\u003e?warehouse=\u003cwarehouse\u003e\u0026role=\u003crole\u003e\u0026authenticator=snowflake_jwt\u0026privateKey=\u003cbase64_url_encoded_private_key\u003e`, where the value for the `privateKey` parameter can be constructed from an unencrypted RSA private key file `rsa_key.p8` using `openssl enc -d -base64 -in rsa_key.p8 | basenc --base64url -w0` (you can use `gbasenc` insted of `basenc` on OSX if you install `coreutils` via Homebrew). If you have a password-encrypted private key, you can decrypt it using `openssl pkcs8 -in rsa_key_encrypted.p8 -out rsa_key.p8`. Also, make sure fields such as the username are URL-encoded.\n\nThe [`gocosmos`](https://pkg.go.dev/github.com/microsoft/gocosmos) driver is still experimental, but it has support for [hierarchical partition keys](https://learn.microsoft.com/en-us/azure/cosmos-db/hierarchical-partition-keys) as well as [cross-partition queries](https://learn.microsoft.com/en-us/azure/cosmos-db/nosql/how-to-query-container#cross-partition-query). Please refer to the [SQL notes](https://github.com/microsoft/gocosmos/blob/main/SQL.md) for details.", "examples": [ "clickhouse://username:password@host1:9000,host2:9000/database?dial_timeout=200ms\u0026max_execution_time=60", "foouser:foopassword@tcp(localhost:3306)/foodb", "postgres://foouser:foopass@localhost:5432/foodb?sslmode=disable", "oracle://foouser:foopass@localhost:1521/service_name" ] }, "endpoint": { "type": "string", "description": "Allows you to specify a custom endpoint for the AWS API. Default: " }, "iam_enabled": { "type": "boolean", "description": "An optional field used to generate an IAM authentication token to connect to an Amazon Relational Database (RDS) DB instance. This will overwrite the Password in the DSN with the generated token only if the drivers are `mysql` or `postgres`. Default: false" }, "init_files": { "type": "array", "description": "\nAn optional list of file paths containing SQL statements to execute immediately upon the first connection to the target database. This is a useful way to initialise tables before processing data. Glob patterns are supported, including super globs (double star).\n\nCare should be taken to ensure that the statements are idempotent, and therefore would not cause issues when run multiple times after service restarts. If both `init_statement` and `init_files` are specified the `init_statement` is executed _after_ the `init_files`.\n\nIf a statement fails for any reason a warning log will be emitted but the operation of this component will not be stopped.\n", "items": { "type": "string" }, "examples": [ [ "./init/*.sql" ], [ "./foo.sql", "./bar.sql" ] ] }, "init_statement": { "type": "string", "description": "\nAn optional SQL statement to execute immediately upon the first connection to the target database. This is a useful way to initialise tables before processing data. Care should be taken to ensure that the statement is idempotent, and therefore would not cause issues when run multiple times after service restarts.\n\nIf both `init_statement` and `init_files` are specified the `init_statement` is executed _after_ the `init_files`.\n\nIf the statement fails for any reason a warning log will be emitted but the operation of this component will not be stopped.\n", "examples": [ "\nCREATE TABLE IF NOT EXISTS some_table (\n foo varchar(50) not null,\n bar integer,\n baz varchar(50),\n primary key (foo)\n) WITHOUT ROWID;\n" ] }, "init_verify_conn": { "type": "boolean", "description": "Whether to verify the database connection on startup by performing a simple ping, by default this is disabled. Default: false" }, "max_in_flight": { "type": "integer", "description": "The maximum number of inserts to run in parallel. Default: 64" }, "query": { "type": "string", "description": "The query to execute. The style of placeholder to use depends on the driver, some drivers require question marks (`?`) whereas others expect incrementing dollar signs (`$1`, `$2`, and so on) or colons (`:1`, `:2` and so on). The style to use is outlined in this table:\n\n| Driver | Placeholder Style |\n|---|---|\n| `clickhouse` | Dollar sign |\n| `mysql` | Question mark |\n| `postgres` | Dollar sign |\n| `mssql` | Question mark |\n| `sqlite` | Question mark |\n| `oracle` | Colon |\n| `snowflake` | Question mark |\n| `spanner` | Question mark |\n| `trino` | Question mark |\n| `gocosmos` | Colon |\n", "examples": [ "INSERT INTO footable (foo, bar, baz) VALUES (?, ?, ?);" ] }, "region": { "type": "string", "description": "The AWS region to target. Default: " }, "secret_name": { "type": "string", "description": "An optional field that can be used to get the Username + Password from AWS Secrets Manager. This will overwrite the Username + Password in the DSN with the values from the Secret only if the driver is set to `postgres`." }, "unsafe_dynamic_query": { "type": "boolean", "description": "Whether to enable [interpolation functions](/docs/configuration/interpolation/#bloblang-queries) in the query. Great care should be made to ensure your queries are defended against injection attacks. Default: false" } } }, "stdout": { "type": "object", "properties": { "codec": { "type": "string", "description": "The way in which the bytes of messages should be written out into the output data stream. It's possible to write lines using a custom delimiter with the `delim:x` codec, where x is the character sequence custom delimiter. Default: lines", "examples": [ "lines", "delim:\t", "delim:foobar" ] } } }, "subprocess": { "type": "object", "properties": { "args": { "type": "array", "description": "A list of arguments to provide the command. Default: []", "items": { "type": "string" } }, "codec": { "type": "string", "description": "The way in which messages should be written to the subprocess. Default: lines" }, "name": { "type": "string", "description": "The command to execute as a subprocess." } } }, "switch": { "type": "object", "properties": { "cases": { "type": "array", "description": "A list of switch cases, outlining outputs that can be routed to.", "items": { "type": "object" }, "examples": [ [ { "check": "this.urls.contains(\"https://warpstreamlabs.github.io/bento\")", "continue": true, "output": { "cache": { "key": "${!json(\"id\")}", "target": "foo" } } }, { "output": { "s3": { "bucket": "bar", "path": "${!json(\"id\")}" } } } ] ] }, "retry_until_success": { "type": "boolean", "description": "\nIf a selected output fails to send a message this field determines whether it is reattempted indefinitely. If set to false the error is instead propagated back to the input level.\n\nIf a message can be routed to \u003e1 outputs it is usually best to set this to true in order to avoid duplicate messages being routed to an output. Default: false" }, "strict_mode": { "type": "boolean", "description": "This field determines whether an error should be reported if no condition is met. If set to true, an error is propagated back to the input level. The default behavior is false, which will drop the message. Default: false" } } }, "sync_response": { "type": "object", "description": " Default: map[]" }, "websocket": { "type": "object", "properties": { "basic_auth": { "type": "object", "description": "Allows you to specify basic authentication.", "properties": { "enabled": { "type": "boolean", "description": "Whether to use basic authentication in requests. Default: false" }, "password": { "type": "string", "description": "A password to authenticate with. Default: " }, "username": { "type": "string", "description": "A username to authenticate as. Default: " } } }, "jwt": { "type": "object", "description": "BETA: Allows you to specify JWT authentication.", "properties": { "claims": { "type": "object", "description": "A value used to identify the claims that issued the JWT. Default: map[]" }, "enabled": { "type": "boolean", "description": "Whether to use JWT authentication in requests. Default: false" }, "headers": { "type": "object", "description": "Add optional key/value headers to the JWT. Default: map[]" }, "private_key_file": { "type": "string", "description": "A file with the PEM encoded via PKCS1 or PKCS8 as private key. Default: " }, "signing_method": { "type": "string", "description": "A method used to sign the token such as RS256, RS384, RS512 or EdDSA. Default: " } } }, "oauth": { "type": "object", "description": "Allows you to specify open authentication via OAuth version 1.", "properties": { "access_token": { "type": "string", "description": "A value used to gain access to the protected resources on behalf of the user. Default: " }, "access_token_secret": { "type": "string", "description": "A secret provided in order to establish ownership of a given access token. Default: " }, "consumer_key": { "type": "string", "description": "A value used to identify the client to the service provider. Default: " }, "consumer_secret": { "type": "string", "description": "A secret used to establish ownership of the consumer key. Default: " }, "enabled": { "type": "boolean", "description": "Whether to use OAuth version 1 in requests. Default: false" } } }, "proxy_url": { "type": "string", "description": "An optional HTTP proxy URL." }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "url": { "type": "string", "description": "The URL to connect to." } } }, "zmq4n": { "type": "object", "properties": { "bind": { "type": "boolean", "description": "Whether to bind to the specified URLs (otherwise they are connected to). Default: true" }, "dial_max_retries": { "type": "integer", "description": "The maximum number of dial retries (-1 for infinite retries). Default: 10" }, "dial_retry_delay": { "type": "string", "description": "The time to wait between failed dial attempts. Default: 250ms" }, "dial_timeout": { "type": "string", "description": "The maximum time to wait for a dial to complete. Default: 5m" }, "high_water_mark": { "type": "integer", "description": "The message high water mark to use. (experimental i go-zeromq) Default: 0" }, "poll_timeout": { "type": "string", "description": "The poll timeout to use. Default: 5s" }, "socket_auto_reconnect": { "type": "boolean", "description": "Whether to automatically attempt internal reconnection on connection loss.\n:::warning Important\nSince this is an internal retry, the zmq4n component will silently attempt reconnection until failure. This means that while retrying, no metric will indicate the component is in a retrying state until attempts have been exhausted.\n::: Default: true" }, "socket_type": { "type": "string", "description": "The socket type to connect as." }, "urls": { "type": "array", "description": "A list of URLs to connect to. If an item of the list contains commas it will be expanded into multiple URLs.", "items": { "type": "string" }, "examples": [ [ "tcp://localhost:5556" ] ] } } } }, "items": { "type": "object" } }, "pipeline": { "type": "object", "description": "Describes optional processing pipelines used for mutating messages.", "properties": { "processors": { "type": "array", "description": "A list of processors to apply to messages. Default: []", "items": { "type": "object", "properties": { "archive": { "type": "object", "properties": { "format": { "type": "string", "description": "The archiving format to apply." }, "path": { "type": "string", "description": "The path to set for each message in the archive (when applicable). Default: ", "examples": [ "${!count(\"files\")}-${!timestamp_unix_nano()}.txt", "${!meta(\"kafka_key\")}-${!json(\"id\")}.json" ] } } }, "avro": { "type": "object", "properties": { "encoding": { "type": "string", "description": "An Avro encoding format to use for conversions to and from a schema. Default: textual" }, "operator": { "type": "string", "description": "The [operator](#operators) to execute" }, "schema": { "type": "string", "description": "A full Avro schema to use. Default: " }, "schema_path": { "type": "string", "description": "The path of a schema document to apply. Use either this or the `schema` field. Default: ", "examples": [ "file://path/to/spec.avsc", "http://localhost:8081/path/to/spec/versions/1" ] } } }, "awk": { "type": "object", "properties": { "codec": { "type": "string", "description": "A [codec](#codecs) defines how messages should be inserted into the AWK program as variables. The codec does not change which [custom Bento functions](#awk-functions) are available. The `text` codec is the closest to a typical AWK use case." }, "program": { "type": "string", "description": "An AWK program to execute" } } }, "aws_dynamodb_partiql": { "type": "object", "properties": { "args_mapping": { "type": "string", "description": "A [Bloblang mapping](/docs/guides/bloblang/about) that, for each message, creates a list of arguments to use with the query. Default: " }, "credentials": { "type": "object", "description": "Optional manual configuration of AWS credentials to use. More information can be found [in this document](/docs/guides/cloud/aws).", "properties": { "expiry_window": { "type": "string", "description": "Allow the credentials to trigger refreshing prior to the credentials actually expiring. This is beneficial so race conditions with expiring credentials do not cause requests to fail. For example '10s' would refresh credentials ten seconds before expiration. Setting to a duration of `0` disables the expiry window. Default: " }, "from_ec2_role": { "type": "boolean", "description": "Use the credentials of a host EC2 machine configured to assume [an IAM role associated with the instance](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html). Default: false" }, "id": { "type": "string", "description": "The ID of credentials to use. Default: " }, "profile": { "type": "string", "description": "A profile from `~/.aws/credentials` to use. Default: " }, "role": { "type": "string", "description": "A role ARN to assume. Default: " }, "role_external_id": { "type": "string", "description": "An external ID to provide when assuming a role. Default: " }, "secret": { "type": "string", "description": "The secret for the credentials being used. Default: " }, "token": { "type": "string", "description": "The token for the credentials being used, required when using short term credentials. Default: " } } }, "endpoint": { "type": "string", "description": "Allows you to specify a custom endpoint for the AWS API. Default: " }, "query": { "type": "string", "description": "A PartiQL query to execute for each message." }, "region": { "type": "string", "description": "The AWS region to target. Default: " }, "unsafe_dynamic_query": { "type": "boolean", "description": "Whether to enable dynamic queries that support interpolation functions. Default: false" } } }, "aws_lambda": { "type": "object", "properties": { "credentials": { "type": "object", "description": "Optional manual configuration of AWS credentials to use. More information can be found [in this document](/docs/guides/cloud/aws).", "properties": { "expiry_window": { "type": "string", "description": "Allow the credentials to trigger refreshing prior to the credentials actually expiring. This is beneficial so race conditions with expiring credentials do not cause requests to fail. For example '10s' would refresh credentials ten seconds before expiration. Setting to a duration of `0` disables the expiry window. Default: " }, "from_ec2_role": { "type": "boolean", "description": "Use the credentials of a host EC2 machine configured to assume [an IAM role associated with the instance](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html). Default: false" }, "id": { "type": "string", "description": "The ID of credentials to use. Default: " }, "profile": { "type": "string", "description": "A profile from `~/.aws/credentials` to use. Default: " }, "role": { "type": "string", "description": "A role ARN to assume. Default: " }, "role_external_id": { "type": "string", "description": "An external ID to provide when assuming a role. Default: " }, "secret": { "type": "string", "description": "The secret for the credentials being used. Default: " }, "token": { "type": "string", "description": "The token for the credentials being used, required when using short term credentials. Default: " } } }, "endpoint": { "type": "string", "description": "Allows you to specify a custom endpoint for the AWS API. Default: " }, "function": { "type": "string", "description": "The function to invoke." }, "parallel": { "type": "boolean", "description": "Whether messages of a batch should be dispatched in parallel. Default: false" }, "rate_limit": { "type": "string", "description": "An optional [`rate_limit`](/docs/components/rate_limits/about) to throttle invocations by. Default: " }, "region": { "type": "string", "description": "The AWS region to target. Default: " }, "retries": { "type": "integer", "description": "The maximum number of retry attempts for each message. Default: 3" }, "timeout": { "type": "string", "description": "The maximum period of time to wait before abandoning an invocation. Default: 5s" } } }, "aws_s3": { "type": "object", "properties": { "bucket": { "type": "string", "description": "The bucket to perform the GetObject operation on." }, "credentials": { "type": "object", "description": "Optional manual configuration of AWS credentials to use. More information can be found [in this document](/docs/guides/cloud/aws).", "properties": { "expiry_window": { "type": "string", "description": "Allow the credentials to trigger refreshing prior to the credentials actually expiring. This is beneficial so race conditions with expiring credentials do not cause requests to fail. For example '10s' would refresh credentials ten seconds before expiration. Setting to a duration of `0` disables the expiry window. Default: " }, "from_ec2_role": { "type": "boolean", "description": "Use the credentials of a host EC2 machine configured to assume [an IAM role associated with the instance](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html). Default: false" }, "id": { "type": "string", "description": "The ID of credentials to use. Default: " }, "profile": { "type": "string", "description": "A profile from `~/.aws/credentials` to use. Default: " }, "role": { "type": "string", "description": "A role ARN to assume. Default: " }, "role_external_id": { "type": "string", "description": "An external ID to provide when assuming a role. Default: " }, "secret": { "type": "string", "description": "The secret for the credentials being used. Default: " }, "token": { "type": "string", "description": "The token for the credentials being used, required when using short term credentials. Default: " } } }, "delete_objects": { "type": "boolean", "description": "Whether to delete downloaded objects from the bucket once they are processed. Note: the S3 Object will be deleted from AWS as soon as this processor has consumed the object. Default: false" }, "endpoint": { "type": "string", "description": "Allows you to specify a custom endpoint for the AWS API. Default: " }, "force_path_style_urls": { "type": "boolean", "description": "Forces the client API to use path style URLs for downloading keys, which is often required when connecting to custom endpoints. Default: false" }, "key": { "type": "string", "description": "The key of the object you wish to retrive." }, "region": { "type": "string", "description": "The AWS region to target. Default: " }, "scanner": { "type": "object", "description": "The [scanner](/docs/components/scanners/about) by which the stream of bytes consumed will be broken out into individual messages. Scanners are useful for processing large sources of data without holding the entirety of it within memory. For example, the `csv` scanner allows you to process individual CSV rows without loading the entire CSV file in memory at once. Default: map[to_the_end:map[]]" } } }, "azure_cosmosdb": { "type": "object", "properties": { "account_key": { "type": "string", "description": "Account key.", "examples": [ "C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==" ] }, "auto_id": { "type": "boolean", "description": "Automatically set the item `id` field to a random UUID v4. If the `id` field is already set, then it will not be overwritten. Setting this to `false` can improve performance, since the messages will not have to be parsed. Default: true" }, "connection_string": { "type": "string", "description": "Connection string.", "examples": [ "AccountEndpoint=https://localhost:8081/;AccountKey=C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==;" ] }, "container": { "type": "string", "description": "Container.", "examples": [ "testcontainer" ] }, "database": { "type": "string", "description": "Database.", "examples": [ "testdb" ] }, "enable_content_response_on_write": { "type": "boolean", "description": "Enable content response on write operations. To save some bandwidth, set this to false if you don't need to receive the updated message(s) from the server, in which case the processor will not modify the content of the messages which are fed into it. Applies to every operation except Read. Default: true" }, "endpoint": { "type": "string", "description": "CosmosDB endpoint.", "examples": [ "https://localhost:8081" ] }, "item_id": { "type": "string", "description": "ID of item to replace or delete. Only used by the Replace and Delete operations", "examples": [ "${! json(\"id\") }" ] }, "operation": { "type": "string", "description": "Operation. Default: Create" }, "partition_keys_map": { "type": "string", "description": "A [Bloblang mapping](/docs/guides/bloblang/about) which should evaluate to a single partition key value or an array of partition key values of type string, integer or boolean. Currently, hierarchical partition keys are not supported so only one value may be provided.", "examples": [ "root = \"blobfish\"", "root = 41", "root = true", "root = null", "root = json(\"blobfish\").depth" ] }, "patch_condition": { "type": "string", "description": "Patch operation condition.", "examples": [ "from c where not is_defined(c.blobfish)" ] }, "patch_operations": { "type": "array", "description": "Patch operations to be performed when `operation: Patch` .", "items": { "type": "object" } } } }, "bloblang": { "type": "string", "description": " Default: " }, "bounds_check": { "type": "object", "properties": { "max_part_size": { "type": "integer", "description": "The maximum size of a message to allow (in bytes) Default: 1073741824" }, "max_parts": { "type": "integer", "description": "The maximum size of message batches to allow (in message count) Default: 100" }, "min_part_size": { "type": "integer", "description": "The minimum size of a message to allow (in bytes) Default: 1" }, "min_parts": { "type": "integer", "description": "The minimum size of message batches to allow (in message count) Default: 1" } } }, "branch": { "type": "object", "properties": { "processors": { "type": "array", "description": "A list of processors to apply to mapped requests. When processing message batches the resulting batch must match the size and ordering of the input batch, therefore filtering, grouping should not be performed within these processors.", "items": { "type": "object" } }, "request_map": { "type": "string", "description": "A [Bloblang mapping](/docs/guides/bloblang/about) that describes how to create a request payload suitable for the child processors of this branch. If left empty then the branch will begin with an exact copy of the origin message (including metadata). Default: ", "examples": [ "root = {\n\t\"id\": this.doc.id,\n\t\"content\": this.doc.body.text\n}", "root = if this.type == \"foo\" {\n\tthis.foo.request\n} else {\n\tdeleted()\n}" ] }, "result_map": { "type": "string", "description": "A [Bloblang mapping](/docs/guides/bloblang/about) that describes how the resulting messages from branched processing should be mapped back into the original payload. If left empty the origin message will remain unchanged (including metadata). Default: ", "examples": [ "meta foo_code = metadata(\"code\")\nroot.foo_result = this", "meta = metadata()\nroot.bar.body = this.body\nroot.bar.id = this.user.id", "root.raw_result = content().string()", "root.enrichments.foo = if metadata(\"request_failed\") != null {\n throw(metadata(\"request_failed\"))\n} else {\n this\n}", "# Retain only the updated metadata fields which were present in the origin message\nmeta = metadata().filter(v -\u003e @.get(v.key) != null)" ] } } }, "cache": { "type": "object", "properties": { "key": { "type": "string", "description": "A key to use with the cache." }, "operator": { "type": "string", "description": "The [operation](#operators) to perform with the cache." }, "resource": { "type": "string", "description": "The [`cache` resource](/docs/components/caches/about) to target with this processor." }, "ttl": { "type": "string", "description": "The TTL of each individual item as a duration string. After this period an item will be eligible for removal during the next compaction. Not all caches support per-key TTLs, those that do will have a configuration field `default_ttl`, and those that do not will fall back to their generally configured TTL setting.", "examples": [ "60s", "5m", "36h" ] }, "value": { "type": "string", "description": "A value to use with the cache (when applicable)." } } }, "cached": { "type": "object", "properties": { "cache": { "type": "string", "description": "The cache resource to read and write processor results from." }, "key": { "type": "string", "description": "A key to be resolved for each message, if the key already exists in the cache then the cached result is used, otherwise the processors are applied and the result is cached under this key. The key could be static and therefore apply generally to all messages or it could be an interpolated expression that is potentially unique for each message.", "examples": [ "my_foo_result", "${! this.document.id }", "${! metadata(\"kafka_key\") }", "${! metadata(\"kafka_topic\") }" ] }, "processors": { "type": "array", "description": "The list of processors whose result will be cached.", "items": { "type": "object" } }, "skip_on": { "type": "string", "description": "A condition that can be used to skip caching the results from the processors.", "examples": [ "errored()" ] }, "ttl": { "type": "string", "description": "An optional expiry period to set for each cache entry. Some caches only have a general TTL and will therefore ignore this setting." } } }, "catch": { "type": "array", "description": " Default: []", "items": { "type": "object" } }, "command": { "type": "object", "properties": { "args_mapping": { "type": "string", "description": "An optional [Bloblang mapping](/docs/guides/bloblang/about) that, when specified, should resolve into an array of arguments to pass to the command. Command arguments are expressed this way in order to support dynamic behaviour.", "examples": [ "[ \"-c\", this.script_path ]" ] }, "name": { "type": "string", "description": "The name of the command to execute.", "examples": [ "bash", "go", "${! @command }" ] } } }, "compress": { "type": "object", "properties": { "algorithm": { "type": "string", "description": "The compression algorithm to use." }, "level": { "type": "integer", "description": "The level of compression to use. May not be applicable to all algorithms. Default: -1" } } }, "couchbase": { "type": "object", "properties": { "bucket": { "type": "string", "description": "Couchbase bucket." }, "cas_enabled": { "type": "boolean", "description": "Enable CAS validation. Default: true" }, "collection": { "type": "string", "description": "Bucket collection. Default: _default" }, "content": { "type": "string", "description": "Document content." }, "id": { "type": "string", "description": "Document id.", "examples": [ "${! json(\"id\") }" ] }, "operation": { "type": "string", "description": "Couchbase operation to perform. Default: get" }, "password": { "type": "string", "description": "Password to connect to the cluster." }, "timeout": { "type": "string", "description": "Operation timeout. Default: 15s" }, "transcoder": { "type": "string", "description": "Couchbase transcoder to use. Default: legacy" }, "ttl": { "type": "string", "description": "An optional TTL to set for items." }, "url": { "type": "string", "description": "Couchbase connection string.", "examples": [ "couchbase://localhost:11210" ] }, "username": { "type": "string", "description": "Username to connect to the cluster." } } }, "decompress": { "type": "object", "properties": { "algorithm": { "type": "string", "description": "The decompression algorithm to use." } } }, "dedupe": { "type": "object", "properties": { "cache": { "type": "string", "description": "The [`cache` resource](/docs/components/caches/about) to target with this processor." }, "drop_on_err": { "type": "boolean", "description": "Whether messages should be dropped when the cache returns a general error such as a network issue. Default: true" }, "key": { "type": "string", "description": "An interpolated string yielding the key to deduplicate by for each message.", "examples": [ "${! metadata(\"kafka_key\") }", "${! content().hash(\"xxhash64\") }" ] }, "strategy": { "type": "string", "description": "Controls how to handle duplicate values. Default: FIFO" } } }, "for_each": { "type": "array", "description": " Default: []", "items": { "type": "object" } }, "gcp_bigquery_select": { "type": "object", "properties": { "args_mapping": { "type": "string", "description": "An optional [Bloblang mapping](/docs/guides/bloblang/about) which should evaluate to an array of values matching in size to the number of placeholder arguments in the field `where`.", "examples": [ "root = [ \"article\", now().ts_format(\"2006-01-02\") ]" ] }, "columns": { "type": "array", "description": "A list of columns to query.", "items": { "type": "string" } }, "columns_mapping": { "type": "string", "description": "An optional [Bloblang mapping](/docs/guides/bloblang/about) which should evaluate to an array of column names to query." }, "job_labels": { "type": "object", "description": "A list of labels to add to the query job. Default: map[]" }, "prefix": { "type": "string", "description": "An optional prefix to prepend to the select query (before SELECT)." }, "project": { "type": "string", "description": "GCP project where the query job will execute." }, "suffix": { "type": "string", "description": "An optional suffix to append to the select query." }, "table": { "type": "string", "description": "Fully-qualified BigQuery table name to query.", "examples": [ "bigquery-public-data.samples.shakespeare" ] }, "unsafe_dynamic_query": { "type": "boolean", "description": "Whether to enable [interpolation functions](/docs/configuration/interpolation/#bloblang-queries) in the columns_mapping, table \u0026 where fields. When `unsafe_dynamic_query` is set to true, you should provide a bloblang mapping via the `columns_mapping` config field, and not `columns`. Great care should be made to ensure your queries are defended against injection attacks. Default: false" }, "where": { "type": "string", "description": "An optional where clause to add. Placeholder arguments are populated with the `args_mapping` field. Placeholders should always be question marks (`?`).", "examples": [ "type = ? and created_at \u003e ?", "user_id = ?" ] } } }, "grok": { "type": "object", "properties": { "expressions": { "type": "array", "description": "One or more Grok expressions to attempt against incoming messages. The first expression to match at least one value will be used to form a result.", "items": { "type": "string" } }, "named_captures_only": { "type": "boolean", "description": "Whether to only capture values from named patterns. Default: true" }, "pattern_definitions": { "type": "object", "description": "A map of pattern definitions that can be referenced within `patterns`. Default: map[]" }, "pattern_paths": { "type": "array", "description": "A list of paths to load Grok patterns from. This field supports wildcards, including super globs (double star). Default: []", "items": { "type": "string" } }, "remove_empty_values": { "type": "boolean", "description": "Whether to remove values that are empty from the resulting structure. Default: true" }, "use_default_patterns": { "type": "boolean", "description": "Whether to use a [default set of patterns](#default-patterns). Default: true" } } }, "group_by": { "type": "array", "items": { "type": "object" } }, "group_by_value": { "type": "object", "properties": { "value": { "type": "string", "description": "The interpolated string to group based on.", "examples": [ "${! meta(\"kafka_key\") }", "${! json(\"foo.bar\") }-${! meta(\"baz\") }" ] } } }, "http": { "type": "object", "properties": { "backoff_on": { "type": "integer", "description": "A list of status codes whereby the request should be considered to have failed and retries should be attempted, but the period between them should be increased gradually. Default: [429]", "items": { "type": "integer" } }, "basic_auth": { "type": "object", "description": "Allows you to specify basic authentication.", "properties": { "enabled": { "type": "boolean", "description": "Whether to use basic authentication in requests. Default: false" }, "password": { "type": "string", "description": "A password to authenticate with. Default: " }, "username": { "type": "string", "description": "A username to authenticate as. Default: " } } }, "batch_as_multipart": { "type": "boolean", "description": "Send message batches as a single request using [RFC1341](https://www.w3.org/Protocols/rfc1341/7_2_Multipart.html). Default: false" }, "drop_on": { "type": "integer", "description": "A list of status codes whereby the request should be considered to have failed but retries should not be attempted. This is useful for preventing wasted retries for requests that will never succeed. Note that with these status codes the _request_ is dropped, but _message_ that caused the request will not be dropped. Default: []", "items": { "type": "integer" } }, "dump_request_log_level": { "type": "string", "description": "EXPERIMENTAL: Optionally set a level at which the request and response payload of each request made will be logged. Default: " }, "extract_headers": { "type": "object", "description": "Specify which response headers should be added to resulting messages as metadata. Header keys are lowercased before matching, so ensure that your patterns target lowercased versions of the header keys that you expect.", "properties": { "include_patterns": { "type": "array", "description": "Provide a list of explicit metadata key regular expression (re2) patterns to match against. Default: []", "items": { "type": "string" }, "examples": [ [ ".*" ], [ "_timestamp_unix$" ] ] }, "include_prefixes": { "type": "array", "description": "Provide a list of explicit metadata key prefixes to match against. Default: []", "items": { "type": "string" }, "examples": [ [ "foo_", "bar_" ], [ "kafka_" ], [ "content-" ] ] } } }, "headers": { "type": "object", "description": "A map of headers to add to the request. Default: map[]", "examples": [ { "Content-Type": "application/octet-stream", "traceparent": "${! tracing_span().traceparent }" } ] }, "jwt": { "type": "object", "description": "BETA: Allows you to specify JWT authentication.", "properties": { "claims": { "type": "object", "description": "A value used to identify the claims that issued the JWT. Default: map[]" }, "enabled": { "type": "boolean", "description": "Whether to use JWT authentication in requests. Default: false" }, "headers": { "type": "object", "description": "Add optional key/value headers to the JWT. Default: map[]" }, "private_key_file": { "type": "string", "description": "A file with the PEM encoded via PKCS1 or PKCS8 as private key. Default: " }, "signing_method": { "type": "string", "description": "A method used to sign the token such as RS256, RS384, RS512 or EdDSA. Default: " } } }, "max_retry_backoff": { "type": "string", "description": "The maximum period to wait between failed requests. Default: 300s" }, "metadata": { "type": "object", "description": "Specify optional matching rules to determine which metadata keys should be added to the HTTP request as headers.", "properties": { "include_patterns": { "type": "array", "description": "Provide a list of explicit metadata key regular expression (re2) patterns to match against. Default: []", "items": { "type": "string" }, "examples": [ [ ".*" ], [ "_timestamp_unix$" ] ] }, "include_prefixes": { "type": "array", "description": "Provide a list of explicit metadata key prefixes to match against. Default: []", "items": { "type": "string" }, "examples": [ [ "foo_", "bar_" ], [ "kafka_" ], [ "content-" ] ] } } }, "oauth": { "type": "object", "description": "Allows you to specify open authentication via OAuth version 1.", "properties": { "access_token": { "type": "string", "description": "A value used to gain access to the protected resources on behalf of the user. Default: " }, "access_token_secret": { "type": "string", "description": "A secret provided in order to establish ownership of a given access token. Default: " }, "consumer_key": { "type": "string", "description": "A value used to identify the client to the service provider. Default: " }, "consumer_secret": { "type": "string", "description": "A secret used to establish ownership of the consumer key. Default: " }, "enabled": { "type": "boolean", "description": "Whether to use OAuth version 1 in requests. Default: false" } } }, "oauth2": { "type": "object", "description": "Allows you to specify open authentication via OAuth version 2 using the client credentials token flow.", "properties": { "client_key": { "type": "string", "description": "A value used to identify the client to the token provider. Default: " }, "client_secret": { "type": "string", "description": "A secret used to establish ownership of the client key. Default: " }, "enabled": { "type": "boolean", "description": "Whether to use OAuth version 2 in requests. Default: false" }, "endpoint_params": { "type": "object", "description": "A list of optional endpoint parameters, values should be arrays of strings. Default: map[]", "examples": [ { "bar": [ "woof" ], "foo": [ "meow", "quack" ] } ] }, "scopes": { "type": "array", "description": "A list of optional requested permissions. Default: []", "items": { "type": "string" } }, "token_url": { "type": "string", "description": "The URL of the token provider. Default: " } } }, "parallel": { "type": "boolean", "description": "When processing batched messages, whether to send messages of the batch in parallel, otherwise they are sent serially. Default: false" }, "proxy_url": { "type": "string", "description": "An optional HTTP proxy URL." }, "rate_limit": { "type": "string", "description": "An optional [rate limit](/docs/components/rate_limits/about) to throttle requests by." }, "retries": { "type": "integer", "description": "The maximum number of retry attempts to make. Default: 3" }, "retry_period": { "type": "string", "description": "The base period to wait between failed requests. Default: 1s" }, "successful_on": { "type": "integer", "description": "A list of status codes whereby the attempt should be considered successful, this is useful for dropping requests that return non-2XX codes indicating that the message has been dealt with, such as a 303 See Other or a 409 Conflict. All 2XX codes are considered successful unless they are present within `backoff_on` or `drop_on`, regardless of this field. Default: []", "items": { "type": "integer" } }, "timeout": { "type": "string", "description": "A static timeout to apply to requests. Default: 5s" }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "transport": { "type": "object", "description": "Custom transport options.", "properties": { "dial_context": { "type": "object", "description": "Settings for the dialer used to create new connections.", "properties": { "keep_alive": { "type": "string", "description": "Keep-alive period for active network connections used by the dialer. Default: 30s" }, "timeout": { "type": "string", "description": "Timeout for establishing new network connections. Default: 30s" } } }, "expect_continue_timeout": { "type": "string", "description": "Time to wait for a server's first response headers after sending request headers when 'Expect: 100-continue' is used. Zero means send body immediately. Default: 1s" }, "force_http2": { "type": "boolean", "description": "If true, the transport will attempt to use HTTP/2. Default: true" }, "idle_connection_timeout": { "type": "string", "description": "Maximum time an idle keep-alive connection remains open before closing itself. Default: 90s" }, "max_idle_connections": { "type": "integer", "description": "Maximum number of idle keep-alive connections. Zero = unlimited. Default: 100" }, "tls_handshake_timeout": { "type": "string", "description": "Maximum time allowed for TLS handshake to complete. Default: 10s" } } }, "url": { "type": "string", "description": "The URL to connect to." }, "verb": { "type": "string", "description": "A verb to connect with Default: POST", "examples": [ "POST", "GET", "DELETE" ] } } }, "insert_part": { "type": "object", "properties": { "content": { "type": "string", "description": "The content of the message being inserted. Default: " }, "index": { "type": "integer", "description": "The index within the batch to insert the message at. Default: -1" } } }, "javascript": { "type": "object", "properties": { "code": { "type": "string", "description": "An inline JavaScript program to run. One of `code` or `file` must be defined." }, "file": { "type": "string", "description": "A file containing a JavaScript program to run. One of `code` or `file` must be defined." }, "global_folders": { "type": "array", "description": "List of folders that will be used to load modules from if the requested JS module is not found elsewhere. Default: []", "items": { "type": "string" } } } }, "jmespath": { "type": "object", "properties": { "query": { "type": "string", "description": "The JMESPath query to apply to messages." } } }, "jq": { "type": "object", "properties": { "output_raw": { "type": "boolean", "description": "Whether to output raw text (unquoted) instead of JSON strings when the emitted values are string types. Default: false" }, "query": { "type": "string", "description": "The jq query to filter and transform messages with." }, "raw": { "type": "boolean", "description": "Whether to process the input as a raw string instead of as JSON. Default: false" } } }, "json_schema": { "type": "object", "properties": { "schema": { "type": "string", "description": "A schema to apply. Use either this or the `schema_path` field." }, "schema_path": { "type": "string", "description": "The path of a schema document to apply. Use either this or the `schema` field." } } }, "log": { "type": "object", "properties": { "fields": { "type": "object", "description": "A map of fields to print along with the log message." }, "fields_mapping": { "type": "string", "description": "An optional [Bloblang mapping](/docs/guides/bloblang/about) that can be used to specify extra fields to add to the log. If log fields are also added with the `fields` field then those values will override matching keys from this mapping.", "examples": [ "root.reason = \"cus I wana\"\nroot.id = this.id\nroot.age = this.user.age.number()\nroot.kafka_topic = metadata(\"kafka_topic\")" ] }, "level": { "type": "string", "description": "The log level to use. Default: INFO" }, "message": { "type": "string", "description": "The message to print. Default: " } } }, "mapping": { "type": "string" }, "metric": { "type": "object", "properties": { "labels": { "type": "object", "description": "A map of label names and values that can be used to enrich metrics. Labels are not supported by some metric destinations, in which case the metrics series are combined.", "examples": [ { "topic": "${! meta(\"kafka_topic\") }", "type": "${! json(\"doc.type\") }" } ] }, "name": { "type": "string", "description": "The name of the metric to create, this must be unique across all Bento components otherwise it will overwrite those other metrics." }, "type": { "type": "string", "description": "The metric [type](#types) to create." }, "value": { "type": "string", "description": "For some metric types specifies a value to set, increment. Certain metrics exporters such as Prometheus support floating point values, but those that do not will cast a floating point value into an integer. Default: " } } }, "mongodb": { "type": "object", "properties": { "backoff": { "type": "object", "description": "Control time intervals between retry attempts.", "properties": { "initial_interval": { "type": "string", "description": "The initial period to wait between retry attempts. Default: 1s" }, "max_elapsed_time": { "type": "string", "description": "The maximum period to wait before retry attempts are abandoned. If zero then no limit is used. Default: 30s" }, "max_interval": { "type": "string", "description": "The maximum period to wait between retry attempts. Default: 5s" } } }, "collection": { "type": "string", "description": "The name of the target collection." }, "database": { "type": "string", "description": "The name of the target MongoDB database." }, "document_map": { "type": "string", "description": "A bloblang map representing a document to store within MongoDB, expressed as [extended JSON in canonical form](https://www.mongodb.com/docs/manual/reference/mongodb-extended-json/). The document map is required for the operations insert-one, replace-one and update-one. Default: ", "examples": [ "root.a = this.foo\nroot.b = this.bar" ] }, "filter_map": { "type": "string", "description": "A bloblang map representing a filter for a MongoDB command, expressed as [extended JSON in canonical form](https://www.mongodb.com/docs/manual/reference/mongodb-extended-json/). The filter map is required for all operations except insert-one. It is used to find the document(s) for the operation. For example in a delete-one case, the filter map should have the fields required to locate the document to delete. Default: ", "examples": [ "root.a = this.foo\nroot.b = this.bar" ] }, "hint_map": { "type": "string", "description": "A bloblang map representing the hint for the MongoDB command, expressed as [extended JSON in canonical form](https://www.mongodb.com/docs/manual/reference/mongodb-extended-json/). This map is optional and is used with all operations except insert-one. It is used to improve performance of finding the documents in the mongodb. Default: ", "examples": [ "root.a = this.foo\nroot.b = this.bar" ] }, "json_marshal_mode": { "type": "string", "description": "The json_marshal_mode setting is optional and controls the format of the output message. Default: canonical" }, "max_retries": { "type": "integer", "description": "The maximum number of retries before giving up on the request. If set to zero there is no discrete limit. Default: 3" }, "operation": { "type": "string", "description": "The mongodb operation to perform. Default: insert-one" }, "password": { "type": "string", "description": "The password to connect to the database. Default: " }, "upsert": { "type": "boolean", "description": "The upsert setting is optional and only applies for update-one and replace-one operations. If the filter specified in filter_map matches, the document is updated or replaced accordingly, otherwise it is created. Default: false" }, "url": { "type": "string", "description": "The URL of the target MongoDB server.", "examples": [ "mongodb://localhost:27017" ] }, "username": { "type": "string", "description": "The username to connect to the database. Default: " }, "write_concern": { "type": "object", "description": "The write concern settings for the mongo connection.", "properties": { "j": { "type": "boolean", "description": "J requests acknowledgement from MongoDB that write operations are written to the journal. Default: false" }, "w": { "type": "string", "description": "W requests acknowledgement that write operations propagate to the specified number of mongodb instances. Default: " }, "w_timeout": { "type": "string", "description": "The write concern timeout. Default: " } } } } }, "msgpack": { "type": "object", "properties": { "operator": { "type": "string", "description": "The operation to perform on messages." } } }, "mutation": { "type": "string" }, "nats_kv": { "type": "object", "properties": { "auth": { "type": "object", "description": "Optional configuration of NATS authentication parameters.", "properties": { "nkey_file": { "type": "string", "description": "An optional file containing a NKey seed.", "examples": [ "./seed.nk" ] }, "user_credentials_file": { "type": "string", "description": "An optional file containing user credentials which consist of an user JWT and corresponding NKey seed.", "examples": [ "./user.creds" ] }, "user_jwt": { "type": "string", "description": "An optional plain text user JWT (given along with the corresponding user NKey Seed)." }, "user_nkey_seed": { "type": "string", "description": "An optional plain text user NKey Seed (given along with the corresponding user JWT)." } } }, "bucket": { "type": "string", "description": "The name of the KV bucket.", "examples": [ "my_kv_bucket" ] }, "key": { "type": "string", "description": "The key for each message. Supports [wildcards](https://docs.nats.io/nats-concepts/subjects#wildcards) for the `history` and `keys` operations.", "examples": [ "foo", "foo.bar.baz", "foo.*", "foo.\u003e", "foo.${! json(\"meta.type\") }" ] }, "operation": { "type": "string", "description": "The operation to perform on the KV bucket." }, "revision": { "type": "string", "description": "The revision of the key to operate on. Used for `get_revision` and `update` operations.", "examples": [ "42", "${! @nats_kv_revision }" ] }, "timeout": { "type": "string", "description": "The maximum period to wait on an operation before aborting and returning an error. Default: 5s" }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "urls": { "type": "array", "description": "A list of URLs to connect to. If an item of the list contains commas it will be expanded into multiple URLs.", "items": { "type": "string" }, "examples": [ [ "nats://127.0.0.1:4222" ], [ "nats://username:password@127.0.0.1:4222" ] ] } } }, "nats_object_store": { "type": "object", "properties": { "auth": { "type": "object", "description": "Optional configuration of NATS authentication parameters.", "properties": { "nkey_file": { "type": "string", "description": "An optional file containing a NKey seed.", "examples": [ "./seed.nk" ] }, "user_credentials_file": { "type": "string", "description": "An optional file containing user credentials which consist of an user JWT and corresponding NKey seed.", "examples": [ "./user.creds" ] }, "user_jwt": { "type": "string", "description": "An optional plain text user JWT (given along with the corresponding user NKey Seed)." }, "user_nkey_seed": { "type": "string", "description": "An optional plain text user NKey Seed (given along with the corresponding user JWT)." } } }, "bucket": { "type": "string", "description": "The name of the object store bucket.", "examples": [ "my_bucket" ] }, "object_name": { "type": "string", "description": "The name of the object in the object store to operate on." }, "operation": { "type": "string", "description": "The operation to perform on the Object Store bucket." }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "urls": { "type": "array", "description": "A list of URLs to connect to. If an item of the list contains commas it will be expanded into multiple URLs.", "items": { "type": "string" }, "examples": [ [ "nats://127.0.0.1:4222" ], [ "nats://username:password@127.0.0.1:4222" ] ] } } }, "nats_request_reply": { "type": "object", "properties": { "auth": { "type": "object", "description": "Optional configuration of NATS authentication parameters.", "properties": { "nkey_file": { "type": "string", "description": "An optional file containing a NKey seed.", "examples": [ "./seed.nk" ] }, "user_credentials_file": { "type": "string", "description": "An optional file containing user credentials which consist of an user JWT and corresponding NKey seed.", "examples": [ "./user.creds" ] }, "user_jwt": { "type": "string", "description": "An optional plain text user JWT (given along with the corresponding user NKey Seed)." }, "user_nkey_seed": { "type": "string", "description": "An optional plain text user NKey Seed (given along with the corresponding user JWT)." } } }, "headers": { "type": "object", "description": "Explicit message headers to add to messages. Default: map[]", "examples": [ { "Content-Type": "application/json", "Timestamp": "${!metadata(\"Timestamp\").string()}" } ] }, "inbox_prefix": { "type": "string", "description": "Set an explicit inbox prefix for the response subject", "examples": [ "_INBOX_joe" ] }, "metadata": { "type": "object", "description": "Determine which (if any) metadata values should be added to messages as headers.", "properties": { "include_patterns": { "type": "array", "description": "Provide a list of explicit metadata key regular expression (re2) patterns to match against. Default: []", "items": { "type": "string" }, "examples": [ [ ".*" ], [ "_timestamp_unix$" ] ] }, "include_prefixes": { "type": "array", "description": "Provide a list of explicit metadata key prefixes to match against. Default: []", "items": { "type": "string" }, "examples": [ [ "foo_", "bar_" ], [ "kafka_" ], [ "content-" ] ] } } }, "subject": { "type": "string", "description": "A subject to write to.", "examples": [ "foo.bar.baz", "${! metadata(\"kafka_topic\") }", "foo.${! json(\"meta.type\") }" ] }, "timeout": { "type": "string", "description": "A duration string is a possibly signed sequence of decimal numbers, each with optional fraction and a unit suffix, such as 300ms, -1.5h or 2h45m. Valid time units are ns, us (or µs), ms, s, m, h. Default: 3s" }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "urls": { "type": "array", "description": "A list of URLs to connect to. If an item of the list contains commas it will be expanded into multiple URLs.", "items": { "type": "string" }, "examples": [ [ "nats://127.0.0.1:4222" ], [ "nats://username:password@127.0.0.1:4222" ] ] } } }, "nlp_classify_text": { "type": "object", "properties": { "aggregation_function": { "type": "string", "description": "The aggregation function to use for the text classification pipeline. Default: SOFTMAX" }, "download_options": { "type": "object", "description": "Options used to download a model directly from HuggingFace. Before the model is downloaded, validation occurs to ensure the remote repository contains both an`.onnx` and `tokenizers.json` file.", "properties": { "onnx_filepath": { "type": "string", "description": "Filepath of the ONNX model within the repository. Only needed when multiple `.onnx` files exist. Default: model.onnx", "examples": [ "onnx/model.onnx", "onnx/model_quantized.onnx", "onnx/model_fp16.onnx" ] }, "repository": { "type": "string", "description": "The name of the huggingface model repository.", "examples": [ "KnightsAnalytics/distilbert-NER", "KnightsAnalytics/distilbert-base-uncased-finetuned-sst-2-english", "sentence-transformers/all-MiniLM-L6-v2" ] } } }, "enable_download": { "type": "boolean", "description": "When enabled, attempts to download an ONNX Runtime compatible model from HuggingFace specified in `repository`. Default: false" }, "multi_label": { "type": "boolean", "description": "Whether a text classification pipeline should return multiple labels. If false, only the label-pair with the highest score is returned. Default: false" }, "name": { "type": "string", "description": "Name of the hugot pipeline. Defaults to a random UUID if not set." }, "path": { "type": "string", "description": "Path to the ONNX model file, or directory containing the model. When downloading (`enable_download: true`), this becomes the destination and must be a directory.", "examples": [ "/path/to/models/my_model.onnx", "/path/to/models/" ] } } }, "nlp_classify_tokens": { "type": "object", "properties": { "aggregation_strategy": { "type": "string", "description": "The aggregation strategy to use for the token classification pipeline. Default: SIMPLE" }, "download_options": { "type": "object", "description": "Options used to download a model directly from HuggingFace. Before the model is downloaded, validation occurs to ensure the remote repository contains both an`.onnx` and `tokenizers.json` file.", "properties": { "onnx_filepath": { "type": "string", "description": "Filepath of the ONNX model within the repository. Only needed when multiple `.onnx` files exist. Default: model.onnx", "examples": [ "onnx/model.onnx", "onnx/model_quantized.onnx", "onnx/model_fp16.onnx" ] }, "repository": { "type": "string", "description": "The name of the huggingface model repository.", "examples": [ "KnightsAnalytics/distilbert-NER", "KnightsAnalytics/distilbert-base-uncased-finetuned-sst-2-english", "sentence-transformers/all-MiniLM-L6-v2" ] } } }, "enable_download": { "type": "boolean", "description": "When enabled, attempts to download an ONNX Runtime compatible model from HuggingFace specified in `repository`. Default: false" }, "ignore_labels": { "type": "array", "description": "Labels to ignore in the token classification pipeline. Default: []", "items": { "type": "string" }, "examples": [ [ "O", "MISC" ] ] }, "name": { "type": "string", "description": "Name of the hugot pipeline. Defaults to a random UUID if not set." }, "path": { "type": "string", "description": "Path to the ONNX model file, or directory containing the model. When downloading (`enable_download: true`), this becomes the destination and must be a directory.", "examples": [ "/path/to/models/my_model.onnx", "/path/to/models/" ] } } }, "nlp_extract_features": { "type": "object", "properties": { "download_options": { "type": "object", "description": "Options used to download a model directly from HuggingFace. Before the model is downloaded, validation occurs to ensure the remote repository contains both an`.onnx` and `tokenizers.json` file.", "properties": { "onnx_filepath": { "type": "string", "description": "Filepath of the ONNX model within the repository. Only needed when multiple `.onnx` files exist. Default: model.onnx", "examples": [ "onnx/model.onnx", "onnx/model_quantized.onnx", "onnx/model_fp16.onnx" ] }, "repository": { "type": "string", "description": "The name of the huggingface model repository.", "examples": [ "KnightsAnalytics/distilbert-NER", "KnightsAnalytics/distilbert-base-uncased-finetuned-sst-2-english", "sentence-transformers/all-MiniLM-L6-v2" ] } } }, "enable_download": { "type": "boolean", "description": "When enabled, attempts to download an ONNX Runtime compatible model from HuggingFace specified in `repository`. Default: false" }, "name": { "type": "string", "description": "Name of the hugot pipeline. Defaults to a random UUID if not set." }, "normalization": { "type": "boolean", "description": "Whether to apply normalization in the feature extraction pipeline. Default: false" }, "path": { "type": "string", "description": "Path to the ONNX model file, or directory containing the model. When downloading (`enable_download: true`), this becomes the destination and must be a directory.", "examples": [ "/path/to/models/my_model.onnx", "/path/to/models/" ] } } }, "nlp_zero_shot_classify": { "type": "object", "properties": { "download_options": { "type": "object", "description": "Options used to download a model directly from HuggingFace. Before the model is downloaded, validation occurs to ensure the remote repository contains both an`.onnx` and `tokenizers.json` file.", "properties": { "onnx_filepath": { "type": "string", "description": "Filepath of the ONNX model within the repository. Only needed when multiple `.onnx` files exist. Default: model.onnx", "examples": [ "onnx/model.onnx", "onnx/model_quantized.onnx", "onnx/model_fp16.onnx" ] }, "repository": { "type": "string", "description": "The name of the huggingface model repository.", "examples": [ "KnightsAnalytics/distilbert-NER", "KnightsAnalytics/distilbert-base-uncased-finetuned-sst-2-english", "sentence-transformers/all-MiniLM-L6-v2" ] } } }, "enable_download": { "type": "boolean", "description": "When enabled, attempts to download an ONNX Runtime compatible model from HuggingFace specified in `repository`. Default: false" }, "hypothesis_template": { "type": "string", "description": "Template to turn each label into an NLI-style hypothesis. Must include {} where the label will be inserted. Default: This example is {}." }, "labels": { "type": "array", "description": "The set of possible class labels to classify each sequence into.", "items": { "type": "string" }, "examples": [ [ "positive", "negative", "neutral" ] ] }, "multi_label": { "type": "boolean", "description": "Whether multiple labels can be true. If false, scores sum to 1. If true, each label is scored independently. Default: false" }, "name": { "type": "string", "description": "Name of the hugot pipeline. Defaults to a random UUID if not set." }, "path": { "type": "string", "description": "Path to the ONNX model file, or directory containing the model. When downloading (`enable_download: true`), this becomes the destination and must be a directory.", "examples": [ "/path/to/models/my_model.onnx", "/path/to/models/" ] } } }, "noop": { "type": "object", "description": " Default: map[]" }, "opensnowcat": { "type": "object", "properties": { "filters": { "type": "object", "description": "Filter and transformation configurations", "properties": { "drop": { "type": "object", "description": "Map of field names to filter criteria. Events matching ANY criteria will be dropped (OR logic). Supports both regular TSV columns (e.g., `user_ipaddress`, `useragent`) and schema property paths (e.g., `com.snowplowanalytics.snowplow.ua_parser_context.useragentFamily`). Each filter uses 'contains' for substring matching." }, "transform": { "type": "object", "description": "Field transformation configuration for anonymization, hashing, and redaction", "properties": { "fields": { "type": "object", "description": "Map of field names to transformation configurations. Each field must specify:\n- **strategy** (required): Transformation type - \"hash\", \"redact\", or \"anonymize_ip\"\n- **hash_algo** (optional): Algorithm for hash strategy - \"MD5\", \"SHA-1\", \"SHA-256\", \"SHA-384\", \"SHA-512\" (overrides global default)\n- **salt** (optional): Salt for hash strategy (overrides global default)\n- **redact_value** (optional): Replacement value for redact strategy (default: \"[REDACTED]\")\n- **anon_octets** (optional): Number of IPv4 octets to mask for anonymize_ip strategy (default: 0)\n- **anon_segments** (optional): Number of IPv6 segments to mask for anonymize_ip strategy (default: 0)\n\nSupports both TSV columns (e.g., user_id, user_ipaddress) and schema property paths (e.g., com.vendor.schema.field)." }, "hash_algo": { "type": "string", "description": "Global default hash algorithm. Can be overridden per field. Default: SHA-256" }, "salt": { "type": "string", "description": "Global default salt for hashing operations. Can be overridden per field." } } } } }, "output_format": { "type": "string", "description": "Output format for processed events. Default: tsv" }, "schema_discovery": { "type": "object", "description": "Schema discovery configuration", "properties": { "enabled": { "type": "boolean", "description": "Enable schema discovery feature Default: false" }, "endpoint": { "type": "string", "description": "HTTP endpoint to send schema discovery data Default: https://api.snowcatcloud.com/internal/schema-discovery" }, "flush_interval": { "type": "string", "description": "Interval between schema discovery flushes Default: 5m" }, "template": { "type": "string", "description": "Template for schema discovery payload. Use `{{SCHEMAS}}` variable for schema list Default: {\"schemas\": {{SCHEMAS}}}" } } }, "set_metadata": { "type": "object", "description": "Map metadata keys to OpenSnowcat canonical event model field names. Supports direct TSV column names (e.g., 'event_fingerprint', 'app_id') and schema property paths (e.g., 'com.vendor.schema.field'). Metadata is set before any filters or transformations are applied.", "examples": [ { "app_id": "app_id", "collector_tstamp": "collector_tstamp", "eid": "event_id", "fingerprint": "event_fingerprint" } ] } } }, "parallel": { "type": "object", "properties": { "cap": { "type": "integer", "description": "The maximum number of messages to have processing at a given time. Default: 0" }, "processors": { "type": "array", "description": "A list of child processors to apply.", "items": { "type": "object" } } } }, "parquet_decode": { "type": "object", "properties": { "byte_array_as_string": { "type": "boolean", "description": "Whether to extract BYTE_ARRAY and FIXED_LEN_BYTE_ARRAY values as strings rather than byte slices in all cases. Values with a logical type of UTF8 will automatically be extracted as strings irrespective of this field. Enabling this field makes serialising the data as JSON more intuitive as `[]byte` values are serialised as base64 encoded strings by default. Default: false" }, "strict_schema": { "type": "boolean", "description": "Whether to enforce strict Parquet schema validation. When set to false, allows reading files with non-standard schema structures (such as non-standard LIST formats). Disabling strict mode may reduce validation but increases compatibility. Default: true" }, "use_parquet_list_format": { "type": "boolean", "description": "Whether to decode`LIST`type columns into their Parquet logical type format `{\"list\": [{\"element\": value_1}, {\"element\": value_2}, ...]}` instead of a Go slice `[value_1, value_2, ...]`.\n\n:::caution\nThis flag will be disabled (set to `false`) by default and deprecated in future versions, with the logical format being deprecated in favour of the Go slice.\n::: Default: true" } } }, "parquet_encode": { "type": "object", "properties": { "default_compression": { "type": "string", "description": "The default compression type to use for fields. Default: uncompressed" }, "default_encoding": { "type": "string", "description": "The default encoding type to use for fields. A custom default encoding is only necessary when consuming data with libraries that do not support `DELTA_LENGTH_BYTE_ARRAY` and is therefore best left unset where possible. Default: DELTA_LENGTH_BYTE_ARRAY" }, "schema": { "type": "array", "description": "Parquet schema.", "items": { "type": "object" } } } }, "parse_log": { "type": "object", "properties": { "allow_rfc3339": { "type": "boolean", "description": "Also accept timestamps in rfc3339 format while parsing. Applicable to format `syslog_rfc3164`. Default: true" }, "best_effort": { "type": "boolean", "description": "Still returns partially parsed messages even if an error occurs. Default: true" }, "codec": { "type": "string" }, "default_timezone": { "type": "string", "description": "Sets the strategy to decide the timezone for rfc3164 timestamps. Applicable to format `syslog_rfc3164`. This value should follow the [time.LoadLocation](https://golang.org/pkg/time/#LoadLocation) format. Default: UTC" }, "default_year": { "type": "string", "description": "Sets the strategy used to set the year for rfc3164 timestamps. Applicable to format `syslog_rfc3164`. When set to `current` the current year will be set, when set to an integer that value will be used. Leave this field empty to not set a default year at all. Default: current" }, "format": { "type": "string", "description": "A common log [format](#formats) to parse." } } }, "processors": { "type": "array", "description": " Default: []", "items": { "type": "object" } }, "protobuf": { "type": "object", "properties": { "bsr": { "type": "array", "description": "Buf Schema Registry configuration. Either this field or `import_paths` must be populated. Note that this field is an array, and multiple BSR configurations can be provided. Default: []", "items": { "type": "object" } }, "discard_unknown": { "type": "boolean", "description": "If `true`, the `from_json` operator discards fields that are unknown to the schema. Default: false" }, "import_paths": { "type": "array", "description": "A list of directories containing .proto files or list of file paths, including all definitions required for parsing the target message. If left empty the current directory is used. Each directory listed will be walked with all found .proto files imported. Either this field or `bsr` must be populated. Default: []", "items": { "type": "string" } }, "message": { "type": "string", "description": "The fully qualified name of the protobuf message to convert to/from." }, "operator": { "type": "string", "description": "The [operator](#operators) to execute" }, "use_proto_names": { "type": "boolean", "description": "If `true`, the `to_json` operator deserializes fields exactly as named in schema file. Default: false" } } }, "rate_limit": { "type": "object", "properties": { "resource": { "type": "string", "description": "The target [`rate_limit` resource](/docs/components/rate_limits/about)." } } }, "redis": { "type": "object", "properties": { "args_mapping": { "type": "string", "description": "A [Bloblang mapping](/docs/guides/bloblang/about) which should evaluate to an array of values matching in size to the number of arguments required for the specified Redis command.", "examples": [ "root = [ this.key ]", "root = [ metadata(\"kafka_key\"), this.count ]" ] }, "command": { "type": "string", "description": "The command to execute.", "examples": [ "scard", "incrby", "${! metadata(\"command\") }" ] }, "key": { "type": "string", "description": "A key to use for the target operator." }, "kind": { "type": "string", "description": "Specifies a simple, cluster-aware, or failover-aware redis client. Default: simple" }, "master": { "type": "string", "description": "Name of the redis master when `kind` is `failover` Default: ", "examples": [ "mymaster" ] }, "operator": { "type": "string", "description": "The operator to apply." }, "retries": { "type": "integer", "description": "The maximum number of retries before abandoning a request. Default: 3" }, "retry_period": { "type": "string", "description": "The time to wait before consecutive retry attempts. Default: 500ms" }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.\n\n**Troubleshooting**\n\nSome cloud hosted instances of Redis (such as Azure Cache) might need some hand holding in order to establish stable connections. Unfortunately, it is often the case that TLS issues will manifest as generic error messages such as \"i/o timeout\". If you're using TLS and are seeing connectivity problems consider setting `enable_renegotiation` to `true`, and ensuring that the server supports at least TLS version 1.2.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "url": { "type": "string", "description": "The URL of the target Redis server. Database is optional and is supplied as the URL path.", "examples": [ "redis://:6397", "redis://localhost:6379", "redis://foousername:foopassword@redisplace:6379", "redis://:foopassword@redisplace:6379", "redis://localhost:6379/1", "redis://localhost:6379/1,redis://localhost:6380/1" ] } } }, "redis_script": { "type": "object", "properties": { "args_mapping": { "type": "string", "description": "A [Bloblang mapping](/docs/guides/bloblang/about) which should evaluate to an array of values matching in size to the number of arguments required for the specified Redis script.", "examples": [ "root = [ this.key ]", "root = [ metadata(\"kafka_key\"), \"hardcoded_value\" ]" ] }, "keys_mapping": { "type": "string", "description": "A [Bloblang mapping](/docs/guides/bloblang/about) which should evaluate to an array of keys matching in size to the number of arguments required for the specified Redis script.", "examples": [ "root = [ this.key ]", "root = [ metadata(\"kafka_key\"), this.count ]" ] }, "kind": { "type": "string", "description": "Specifies a simple, cluster-aware, or failover-aware redis client. Default: simple" }, "master": { "type": "string", "description": "Name of the redis master when `kind` is `failover` Default: ", "examples": [ "mymaster" ] }, "retries": { "type": "integer", "description": "The maximum number of retries before abandoning a request. Default: 3" }, "retry_period": { "type": "string", "description": "The time to wait before consecutive retry attempts. Default: 500ms" }, "script": { "type": "string", "description": "A script to use for the target operator. It has precedence over the 'command' field.", "examples": [ "return redis.call('set', KEYS[1], ARGV[1])" ] }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.\n\n**Troubleshooting**\n\nSome cloud hosted instances of Redis (such as Azure Cache) might need some hand holding in order to establish stable connections. Unfortunately, it is often the case that TLS issues will manifest as generic error messages such as \"i/o timeout\". If you're using TLS and are seeing connectivity problems consider setting `enable_renegotiation` to `true`, and ensuring that the server supports at least TLS version 1.2.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "url": { "type": "string", "description": "The URL of the target Redis server. Database is optional and is supplied as the URL path.", "examples": [ "redis://:6397", "redis://localhost:6379", "redis://foousername:foopassword@redisplace:6379", "redis://:foopassword@redisplace:6379", "redis://localhost:6379/1", "redis://localhost:6379/1,redis://localhost:6380/1" ] } } }, "resource": { "type": "string", "description": " Default: " }, "retry": { "type": "object", "properties": { "backoff": { "type": "object", "description": "Determine time intervals and cut offs for retry attempts.", "properties": { "initial_interval": { "type": "string", "description": "The initial period to wait between retry attempts. Default: 500ms", "examples": [ "50ms", "1s" ] }, "max_elapsed_time": { "type": "string", "description": "The maximum overall period of time to spend on retry attempts before the request is aborted. Setting this value to a zeroed duration (such as `0s`) will result in unbounded retries. Default: 1m", "examples": [ "1m", "1h" ] }, "max_interval": { "type": "string", "description": "The maximum period to wait between retry attempts Default: 10s", "examples": [ "5s", "1m" ] } } }, "max_retries": { "type": "integer", "description": "The maximum number of retry attempts before the request is aborted. Setting this value to `0` will result in unbounded number of retries. Default: 0" }, "parallel": { "type": "boolean", "description": "When processing batches of messages these batches are ignored and the processors apply to each message sequentially. However, when this field is set to `true` each message will be processed in parallel. Caution should be made to ensure that batch sizes do not surpass a point where this would cause resource (CPU, memory, API limits) contention. Default: false" }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about/) to execute on each message.", "items": { "type": "object" } } } }, "schema_registry_decode": { "type": "object", "properties": { "avro_nested_schemas": { "type": "boolean", "description": "Whether Avro Schemas are nested. If true bento will resolve schema references. (Up to a maximum depth of 100) Default: false" }, "avro_raw_json": { "type": "boolean", "description": "Whether Avro messages should be decoded into normal JSON (\"json that meets the expectations of regular internet json\") rather than [Avro JSON](https://avro.apache.org/docs/current/specification/_print/#json-encoding). If `true` the schema returned from the subject should be decoded as [standard json](https://pkg.go.dev/github.com/linkedin/goavro/v2#NewCodecForStandardJSONFull) instead of as [avro json](https://pkg.go.dev/github.com/linkedin/goavro/v2#NewCodec). There is a [comment in goavro](https://github.com/linkedin/goavro/blob/5ec5a5ee7ec82e16e6e2b438d610e1cab2588393/union.go#L224-L249), the [underlining library used for avro serialization](https://github.com/linkedin/goavro), that explains in more detail the difference between the standard json and avro json. Default: false" }, "basic_auth": { "type": "object", "description": "Allows you to specify basic authentication.", "properties": { "enabled": { "type": "boolean", "description": "Whether to use basic authentication in requests. Default: false" }, "password": { "type": "string", "description": "A password to authenticate with. Default: " }, "username": { "type": "string", "description": "A username to authenticate as. Default: " } } }, "jwt": { "type": "object", "description": "BETA: Allows you to specify JWT authentication.", "properties": { "claims": { "type": "object", "description": "A value used to identify the claims that issued the JWT. Default: map[]" }, "enabled": { "type": "boolean", "description": "Whether to use JWT authentication in requests. Default: false" }, "headers": { "type": "object", "description": "Add optional key/value headers to the JWT. Default: map[]" }, "private_key_file": { "type": "string", "description": "A file with the PEM encoded via PKCS1 or PKCS8 as private key. Default: " }, "signing_method": { "type": "string", "description": "A method used to sign the token such as RS256, RS384, RS512 or EdDSA. Default: " } } }, "oauth": { "type": "object", "description": "Allows you to specify open authentication via OAuth version 1.", "properties": { "access_token": { "type": "string", "description": "A value used to gain access to the protected resources on behalf of the user. Default: " }, "access_token_secret": { "type": "string", "description": "A secret provided in order to establish ownership of a given access token. Default: " }, "consumer_key": { "type": "string", "description": "A value used to identify the client to the service provider. Default: " }, "consumer_secret": { "type": "string", "description": "A secret used to establish ownership of the consumer key. Default: " }, "enabled": { "type": "boolean", "description": "Whether to use OAuth version 1 in requests. Default: false" } } }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "transport": { "type": "object", "description": "Custom transport options.", "properties": { "dial_context": { "type": "object", "description": "Settings for the dialer used to create new connections.", "properties": { "keep_alive": { "type": "string", "description": "Keep-alive period for active network connections used by the dialer. Default: 30s" }, "timeout": { "type": "string", "description": "Timeout for establishing new network connections. Default: 30s" } } }, "expect_continue_timeout": { "type": "string", "description": "Time to wait for a server's first response headers after sending request headers when 'Expect: 100-continue' is used. Zero means send body immediately. Default: 1s" }, "force_http2": { "type": "boolean", "description": "If true, the transport will attempt to use HTTP/2. Default: true" }, "idle_connection_timeout": { "type": "string", "description": "Maximum time an idle keep-alive connection remains open before closing itself. Default: 90s" }, "max_idle_connections": { "type": "integer", "description": "Maximum number of idle keep-alive connections. Zero = unlimited. Default: 100" }, "tls_handshake_timeout": { "type": "string", "description": "Maximum time allowed for TLS handshake to complete. Default: 10s" } } }, "url": { "type": "string", "description": "The base URL of the schema registry service." } } }, "schema_registry_encode": { "type": "object", "properties": { "avro_nested_schemas": { "type": "boolean", "description": "Whether Avro Schemas are nested. If true bento will resolve schema references. (Up to a maximum depth of 100) Default: false" }, "avro_raw_json": { "type": "boolean", "description": "Whether messages encoded in Avro format should be parsed as normal JSON (\"json that meets the expectations of regular internet json\") rather than [Avro JSON](https://avro.apache.org/docs/current/specification/_print/#json-encoding). If `true` the schema returned from the subject should be parsed as [standard json](https://pkg.go.dev/github.com/linkedin/goavro/v2#NewCodecForStandardJSONFull) instead of as [avro json](https://pkg.go.dev/github.com/linkedin/goavro/v2#NewCodec). There is a [comment in goavro](https://github.com/linkedin/goavro/blob/5ec5a5ee7ec82e16e6e2b438d610e1cab2588393/union.go#L224-L249), the [underlining library used for avro serialization](https://github.com/linkedin/goavro), that explains in more detail the difference between standard json and avro json. Default: false" }, "basic_auth": { "type": "object", "description": "Allows you to specify basic authentication.", "properties": { "enabled": { "type": "boolean", "description": "Whether to use basic authentication in requests. Default: false" }, "password": { "type": "string", "description": "A password to authenticate with. Default: " }, "username": { "type": "string", "description": "A username to authenticate as. Default: " } } }, "jwt": { "type": "object", "description": "BETA: Allows you to specify JWT authentication.", "properties": { "claims": { "type": "object", "description": "A value used to identify the claims that issued the JWT. Default: map[]" }, "enabled": { "type": "boolean", "description": "Whether to use JWT authentication in requests. Default: false" }, "headers": { "type": "object", "description": "Add optional key/value headers to the JWT. Default: map[]" }, "private_key_file": { "type": "string", "description": "A file with the PEM encoded via PKCS1 or PKCS8 as private key. Default: " }, "signing_method": { "type": "string", "description": "A method used to sign the token such as RS256, RS384, RS512 or EdDSA. Default: " } } }, "oauth": { "type": "object", "description": "Allows you to specify open authentication via OAuth version 1.", "properties": { "access_token": { "type": "string", "description": "A value used to gain access to the protected resources on behalf of the user. Default: " }, "access_token_secret": { "type": "string", "description": "A secret provided in order to establish ownership of a given access token. Default: " }, "consumer_key": { "type": "string", "description": "A value used to identify the client to the service provider. Default: " }, "consumer_secret": { "type": "string", "description": "A secret used to establish ownership of the consumer key. Default: " }, "enabled": { "type": "boolean", "description": "Whether to use OAuth version 1 in requests. Default: false" } } }, "refresh_period": { "type": "string", "description": "The period after which a schema is refreshed for each subject, this is done by polling the schema registry service. Default: 10m", "examples": [ "60s", "1h" ] }, "subject": { "type": "string", "description": "The schema subject to derive schemas from.", "examples": [ "foo", "${! metadata(\"kafka_topic\") }" ] }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "transport": { "type": "object", "description": "Custom transport options.", "properties": { "dial_context": { "type": "object", "description": "Settings for the dialer used to create new connections.", "properties": { "keep_alive": { "type": "string", "description": "Keep-alive period for active network connections used by the dialer. Default: 30s" }, "timeout": { "type": "string", "description": "Timeout for establishing new network connections. Default: 30s" } } }, "expect_continue_timeout": { "type": "string", "description": "Time to wait for a server's first response headers after sending request headers when 'Expect: 100-continue' is used. Zero means send body immediately. Default: 1s" }, "force_http2": { "type": "boolean", "description": "If true, the transport will attempt to use HTTP/2. Default: true" }, "idle_connection_timeout": { "type": "string", "description": "Maximum time an idle keep-alive connection remains open before closing itself. Default: 90s" }, "max_idle_connections": { "type": "integer", "description": "Maximum number of idle keep-alive connections. Zero = unlimited. Default: 100" }, "tls_handshake_timeout": { "type": "string", "description": "Maximum time allowed for TLS handshake to complete. Default: 10s" } } }, "url": { "type": "string", "description": "The base URL of the schema registry service." } } }, "select_parts": { "type": "object", "properties": { "parts": { "type": "integer", "description": "An array of message indexes of a batch. Indexes can be negative, and if so the part will be selected from the end counting backwards starting from -1. Default: []", "items": { "type": "integer" } } } }, "sentry_capture": { "type": "object", "properties": { "context": { "type": "string", "description": "A mapping that must evaluate to an object-of-objects or `deleted()`. If this mapping produces a value, then it is set on a sentry event as additional context.", "examples": [ "root = {\"order\": {\"product_id\": \"P93174\", \"quantity\": 5}}", "root = deleted()" ] }, "dsn": { "type": "string", "description": "The DSN address to send sentry events to. If left empty, then SENTRY_DSN is used. Default: " }, "environment": { "type": "string", "description": "The environment to be sent with events. If left empty, then SENTRY_ENVIRONMENT is used. Default: " }, "flush_timeout": { "type": "string", "description": "The duration to wait when closing the processor to flush any remaining enqueued events. Default: 5s" }, "level": { "type": "string", "description": "Sets the level on sentry events similar to logging levels. Default: INFO" }, "message": { "type": "string", "description": "A message to set on the sentry event", "examples": [ "webhook event received", "failed to find product in database: ${! error() }" ] }, "release": { "type": "string", "description": "The version of the code deployed to an environment. If left empty, then the Sentry client will attempt to detect the release from the environment. Default: " }, "sampling_rate": { "type": "number", "description": "The rate at which events are sent to the server. A value of 0 disables capturing sentry events entirely. A value of 1 results in sending all events to Sentry. Any value in between results sending some percentage of events. Default: 1" }, "tags": { "type": "object", "description": "Sets key/value string tags on an event. Unlike context, these are indexed and searchable on Sentry but have length limitations." }, "transport_mode": { "type": "string", "description": "Determines how events are sent. A sync transport will block when sending each event until a response is received from the Sentry server. The recommended async transport will enqueue events in a buffer and send them in the background. Default: async" } } }, "sleep": { "type": "object", "properties": { "duration": { "type": "string", "description": "The duration of time to sleep for each execution." } } }, "split": { "type": "object", "properties": { "byte_size": { "type": "integer", "description": "An optional target of total message bytes. Default: 0" }, "size": { "type": "integer", "description": "The target number of messages. Default: 1" } } }, "sql": { "type": "object", "properties": { "args_mapping": { "type": "string", "description": "An optional [Bloblang mapping](/docs/guides/bloblang/about) which should evaluate to an array of values matching in size to the number of placeholder arguments in the field `query`.", "examples": [ "root = [ this.cat.meow, this.doc.woofs[0] ]", "root = [ metadata(\"user.id\").string() ]" ] }, "data_source_name": { "type": "string", "description": "Data source name." }, "driver": { "type": "string", "description": "A database [driver](#drivers) to use." }, "query": { "type": "string", "description": "The query to execute. The style of placeholder to use depends on the driver, some drivers require question marks (`?`) whereas others expect incrementing dollar signs (`$1`, `$2`, and so on) or colons (`:1`, `:2` and so on). The style to use is outlined in this table:\n\n| Driver | Placeholder Style |\n|---|---|\n| `clickhouse` | Dollar sign |\n| `mysql` | Question mark |\n| `postgres` | Dollar sign |\n| `mssql` | Question mark |\n| `sqlite` | Question mark |\n| `oracle` | Colon |\n| `snowflake` | Question mark |\n| `spanner` | Question mark |\n| `trino` | Question mark |\n| `gocosmos` | Colon |\n", "examples": [ "INSERT INTO footable (foo, bar, baz) VALUES (?, ?, ?);" ] }, "result_codec": { "type": "string", "description": "Result codec. Default: none" }, "unsafe_dynamic_query": { "type": "boolean", "description": "Whether to enable [interpolation functions](/docs/configuration/interpolation/#bloblang-queries) in the query. Great care should be made to ensure your queries are defended against injection attacks. Default: false" } } }, "sql_insert": { "type": "object", "properties": { "args_mapping": { "type": "string", "description": "A [Bloblang mapping](/docs/guides/bloblang/about) which should evaluate to an array of values matching in size to the number of columns specified.", "examples": [ "root = [ this.cat.meow, this.doc.woofs[0] ]", "root = [ metadata(\"user.id\").string() ]" ] }, "azure": { "type": "object", "description": "Optional Fields that can be set to use Azure based authentication for Azure Postgres SQL", "properties": { "entra_enabled": { "type": "boolean", "description": "An optional field used to generate an entra token to connect to 'Azure Database for PostgreSQL flexible server', This will create a new connection string with the host, user and database from the DSN field - you may need to URL encode the dsn! The [Default Azure Credential Chain](https://learn.microsoft.com/en-gb/azure/developer/go/sdk/authentication/authentication-overview#defaultazurecredential) is used from the Azure SDK. Default: false" }, "token_request_options": { "type": "object", "properties": { "claims": { "type": "string", "description": "Set additional claims for the token. Default: " }, "enable_cae": { "type": "boolean", "description": "Indicates whether to enable Continuous Access Evaluation (CAE) for the requested token Default: false" }, "scopes": { "type": "array", "description": "Scopes contains the list of permission scopes required for the token. Default: [https://ossrdbms-aad.database.windows.net/.default]", "items": { "type": "string" } }, "tenant_id": { "type": "string", "description": "tenant_id identifies the tenant from which to request the token. azure credentials authenticate in their configured default tenants when this field isn't set. Default: " } } } } }, "columns": { "type": "array", "description": "A list of columns to insert.", "items": { "type": "string" }, "examples": [ [ "foo", "bar", "baz" ] ] }, "conn_max_idle": { "type": "integer", "description": "An optional maximum number of connections in the idle connection pool. If conn_max_open is greater than 0 but less than the new conn_max_idle, then the new conn_max_idle will be reduced to match the conn_max_open limit. If `value \u003c= 0`, no idle connections are retained. The default max idle connections is currently 2. This may change in a future release. Default: 2" }, "conn_max_idle_time": { "type": "string", "description": "An optional maximum amount of time a connection may be idle. Expired connections may be closed lazily before reuse. If `value \u003c= 0`, connections are not closed due to a connections idle time." }, "conn_max_life_time": { "type": "string", "description": "An optional maximum amount of time a connection may be reused. Expired connections may be closed lazily before reuse. If `value \u003c= 0`, connections are not closed due to a connections age." }, "conn_max_open": { "type": "integer", "description": "An optional maximum number of open connections to the database. If conn_max_idle is greater than 0 and the new conn_max_open is less than conn_max_idle, then conn_max_idle will be reduced to match the new conn_max_open limit. If `value \u003c= 0`, then there is no limit on the number of open connections. The default is 0 (unlimited)." }, "credentials": { "type": "object", "description": "Optional manual configuration of AWS credentials to use. More information can be found [in this document](/docs/guides/cloud/aws).", "properties": { "expiry_window": { "type": "string", "description": "Allow the credentials to trigger refreshing prior to the credentials actually expiring. This is beneficial so race conditions with expiring credentials do not cause requests to fail. For example '10s' would refresh credentials ten seconds before expiration. Setting to a duration of `0` disables the expiry window. Default: " }, "from_ec2_role": { "type": "boolean", "description": "Use the credentials of a host EC2 machine configured to assume [an IAM role associated with the instance](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html). Default: false" }, "id": { "type": "string", "description": "The ID of credentials to use. Default: " }, "profile": { "type": "string", "description": "A profile from `~/.aws/credentials` to use. Default: " }, "role": { "type": "string", "description": "A role ARN to assume. Default: " }, "role_external_id": { "type": "string", "description": "An external ID to provide when assuming a role. Default: " }, "secret": { "type": "string", "description": "The secret for the credentials being used. Default: " }, "token": { "type": "string", "description": "The token for the credentials being used, required when using short term credentials. Default: " } } }, "driver": { "type": "string", "description": "A database [driver](#drivers) to use." }, "dsn": { "type": "string", "description": "A Data Source Name to identify the target database.\n\n#### Drivers\n\nThe following is a list of supported drivers, their placeholder style, and their respective DSN formats:\n\n| Driver | Data Source Name Format |\n|---|---|\n| `clickhouse` | [`clickhouse://[username[:password]@][netloc][:port]/dbname[?param1=value1\u0026...\u0026paramN=valueN]`](https://github.com/ClickHouse/clickhouse-go#dsn) |\n| `mysql` | `[username[:password]@][protocol[(address)]]/dbname[?param1=value1\u0026...\u0026paramN=valueN]` |\n| `postgres` | `postgres://[user[:password]@][netloc][:port][/dbname][?param1=value1\u0026...]` |\n| `mssql` | `sqlserver://[user[:password]@][netloc][:port][?database=dbname\u0026param1=value1\u0026...]` |\n| `sqlite` | `file:/path/to/filename.db[?param\u0026=value1\u0026...]` |\n| `oracle` | `oracle://[username[:password]@][netloc][:port]/service_name?server=server2\u0026server=server3` |\n| `snowflake` | `username[:password]@account_identifier/dbname/schemaname[?param1=value\u0026...\u0026paramN=valueN]` |\n| `spanner` | `projects/[project]/instances/[instance]/databases/dbname` |\n| `trino` | [`http[s]://user[:pass]@host[:port][?parameters]`](https://github.com/trinodb/trino-go-client#dsn-data-source-name) |\n| `gocosmos` | [`AccountEndpoint=\u003ccosmosdb-endpoint\u003e;AccountKey=\u003ccosmosdb-account-key\u003e[;TimeoutMs=\u003ctimeout-in-ms\u003e][;Version=\u003ccosmosdb-api-version\u003e][;DefaultDb/Db=\u003cdb-name\u003e][;AutoId=\u003ctrue/false\u003e][;InsecureSkipVerify=\u003ctrue/false\u003e]`](https://pkg.go.dev/github.com/microsoft/gocosmos#readme-example-usage) |\n\nPlease note that the `postgres` driver enforces SSL by default, you can override this with the parameter `sslmode=disable` if required.\n\nThe `snowflake` driver supports multiple DSN formats. Please consult [the docs](https://pkg.go.dev/github.com/snowflakedb/gosnowflake#hdr-Connection_String) for more details. For [key pair authentication](https://docs.snowflake.com/en/user-guide/key-pair-auth.html#configuring-key-pair-authentication), the DSN has the following format: `\u003csnowflake_user\u003e@\u003csnowflake_account\u003e/\u003cdb_name\u003e/\u003cschema_name\u003e?warehouse=\u003cwarehouse\u003e\u0026role=\u003crole\u003e\u0026authenticator=snowflake_jwt\u0026privateKey=\u003cbase64_url_encoded_private_key\u003e`, where the value for the `privateKey` parameter can be constructed from an unencrypted RSA private key file `rsa_key.p8` using `openssl enc -d -base64 -in rsa_key.p8 | basenc --base64url -w0` (you can use `gbasenc` insted of `basenc` on OSX if you install `coreutils` via Homebrew). If you have a password-encrypted private key, you can decrypt it using `openssl pkcs8 -in rsa_key_encrypted.p8 -out rsa_key.p8`. Also, make sure fields such as the username are URL-encoded.\n\nThe [`gocosmos`](https://pkg.go.dev/github.com/microsoft/gocosmos) driver is still experimental, but it has support for [hierarchical partition keys](https://learn.microsoft.com/en-us/azure/cosmos-db/hierarchical-partition-keys) as well as [cross-partition queries](https://learn.microsoft.com/en-us/azure/cosmos-db/nosql/how-to-query-container#cross-partition-query). Please refer to the [SQL notes](https://github.com/microsoft/gocosmos/blob/main/SQL.md) for details.", "examples": [ "clickhouse://username:password@host1:9000,host2:9000/database?dial_timeout=200ms\u0026max_execution_time=60", "foouser:foopassword@tcp(localhost:3306)/foodb", "postgres://foouser:foopass@localhost:5432/foodb?sslmode=disable", "oracle://foouser:foopass@localhost:1521/service_name" ] }, "endpoint": { "type": "string", "description": "Allows you to specify a custom endpoint for the AWS API. Default: " }, "iam_enabled": { "type": "boolean", "description": "An optional field used to generate an IAM authentication token to connect to an Amazon Relational Database (RDS) DB instance. This will overwrite the Password in the DSN with the generated token only if the drivers are `mysql` or `postgres`. Default: false" }, "init_files": { "type": "array", "description": "\nAn optional list of file paths containing SQL statements to execute immediately upon the first connection to the target database. This is a useful way to initialise tables before processing data. Glob patterns are supported, including super globs (double star).\n\nCare should be taken to ensure that the statements are idempotent, and therefore would not cause issues when run multiple times after service restarts. If both `init_statement` and `init_files` are specified the `init_statement` is executed _after_ the `init_files`.\n\nIf a statement fails for any reason a warning log will be emitted but the operation of this component will not be stopped.\n", "items": { "type": "string" }, "examples": [ [ "./init/*.sql" ], [ "./foo.sql", "./bar.sql" ] ] }, "init_statement": { "type": "string", "description": "\nAn optional SQL statement to execute immediately upon the first connection to the target database. This is a useful way to initialise tables before processing data. Care should be taken to ensure that the statement is idempotent, and therefore would not cause issues when run multiple times after service restarts.\n\nIf both `init_statement` and `init_files` are specified the `init_statement` is executed _after_ the `init_files`.\n\nIf the statement fails for any reason a warning log will be emitted but the operation of this component will not be stopped.\n", "examples": [ "\nCREATE TABLE IF NOT EXISTS some_table (\n foo varchar(50) not null,\n bar integer,\n baz varchar(50),\n primary key (foo)\n) WITHOUT ROWID;\n" ] }, "init_verify_conn": { "type": "boolean", "description": "Whether to verify the database connection on startup by performing a simple ping, by default this is disabled. Default: false" }, "prefix": { "type": "string", "description": "An optional prefix to prepend to the insert query (before INSERT)." }, "region": { "type": "string", "description": "The AWS region to target. Default: " }, "secret_name": { "type": "string", "description": "An optional field that can be used to get the Username + Password from AWS Secrets Manager. This will overwrite the Username + Password in the DSN with the values from the Secret only if the driver is set to `postgres`." }, "suffix": { "type": "string", "description": "An optional suffix to append to the insert query.", "examples": [ "ON CONFLICT (name) DO NOTHING" ] }, "table": { "type": "string", "description": "The table to insert to.", "examples": [ "foo" ] } } }, "sql_raw": { "type": "object", "properties": { "args_mapping": { "type": "string", "description": "An optional [Bloblang mapping](/docs/guides/bloblang/about) which should evaluate to an array of values matching in size to the number of placeholder arguments in the field `query`.", "examples": [ "root = [ this.cat.meow, this.doc.woofs[0] ]", "root = [ metadata(\"user.id\").string() ]" ] }, "azure": { "type": "object", "description": "Optional Fields that can be set to use Azure based authentication for Azure Postgres SQL", "properties": { "entra_enabled": { "type": "boolean", "description": "An optional field used to generate an entra token to connect to 'Azure Database for PostgreSQL flexible server', This will create a new connection string with the host, user and database from the DSN field - you may need to URL encode the dsn! The [Default Azure Credential Chain](https://learn.microsoft.com/en-gb/azure/developer/go/sdk/authentication/authentication-overview#defaultazurecredential) is used from the Azure SDK. Default: false" }, "token_request_options": { "type": "object", "properties": { "claims": { "type": "string", "description": "Set additional claims for the token. Default: " }, "enable_cae": { "type": "boolean", "description": "Indicates whether to enable Continuous Access Evaluation (CAE) for the requested token Default: false" }, "scopes": { "type": "array", "description": "Scopes contains the list of permission scopes required for the token. Default: [https://ossrdbms-aad.database.windows.net/.default]", "items": { "type": "string" } }, "tenant_id": { "type": "string", "description": "tenant_id identifies the tenant from which to request the token. azure credentials authenticate in their configured default tenants when this field isn't set. Default: " } } } } }, "conn_max_idle": { "type": "integer", "description": "An optional maximum number of connections in the idle connection pool. If conn_max_open is greater than 0 but less than the new conn_max_idle, then the new conn_max_idle will be reduced to match the conn_max_open limit. If `value \u003c= 0`, no idle connections are retained. The default max idle connections is currently 2. This may change in a future release. Default: 2" }, "conn_max_idle_time": { "type": "string", "description": "An optional maximum amount of time a connection may be idle. Expired connections may be closed lazily before reuse. If `value \u003c= 0`, connections are not closed due to a connections idle time." }, "conn_max_life_time": { "type": "string", "description": "An optional maximum amount of time a connection may be reused. Expired connections may be closed lazily before reuse. If `value \u003c= 0`, connections are not closed due to a connections age." }, "conn_max_open": { "type": "integer", "description": "An optional maximum number of open connections to the database. If conn_max_idle is greater than 0 and the new conn_max_open is less than conn_max_idle, then conn_max_idle will be reduced to match the new conn_max_open limit. If `value \u003c= 0`, then there is no limit on the number of open connections. The default is 0 (unlimited)." }, "credentials": { "type": "object", "description": "Optional manual configuration of AWS credentials to use. More information can be found [in this document](/docs/guides/cloud/aws).", "properties": { "expiry_window": { "type": "string", "description": "Allow the credentials to trigger refreshing prior to the credentials actually expiring. This is beneficial so race conditions with expiring credentials do not cause requests to fail. For example '10s' would refresh credentials ten seconds before expiration. Setting to a duration of `0` disables the expiry window. Default: " }, "from_ec2_role": { "type": "boolean", "description": "Use the credentials of a host EC2 machine configured to assume [an IAM role associated with the instance](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html). Default: false" }, "id": { "type": "string", "description": "The ID of credentials to use. Default: " }, "profile": { "type": "string", "description": "A profile from `~/.aws/credentials` to use. Default: " }, "role": { "type": "string", "description": "A role ARN to assume. Default: " }, "role_external_id": { "type": "string", "description": "An external ID to provide when assuming a role. Default: " }, "secret": { "type": "string", "description": "The secret for the credentials being used. Default: " }, "token": { "type": "string", "description": "The token for the credentials being used, required when using short term credentials. Default: " } } }, "driver": { "type": "string", "description": "A database [driver](#drivers) to use." }, "dsn": { "type": "string", "description": "A Data Source Name to identify the target database.\n\n#### Drivers\n\nThe following is a list of supported drivers, their placeholder style, and their respective DSN formats:\n\n| Driver | Data Source Name Format |\n|---|---|\n| `clickhouse` | [`clickhouse://[username[:password]@][netloc][:port]/dbname[?param1=value1\u0026...\u0026paramN=valueN]`](https://github.com/ClickHouse/clickhouse-go#dsn) |\n| `mysql` | `[username[:password]@][protocol[(address)]]/dbname[?param1=value1\u0026...\u0026paramN=valueN]` |\n| `postgres` | `postgres://[user[:password]@][netloc][:port][/dbname][?param1=value1\u0026...]` |\n| `mssql` | `sqlserver://[user[:password]@][netloc][:port][?database=dbname\u0026param1=value1\u0026...]` |\n| `sqlite` | `file:/path/to/filename.db[?param\u0026=value1\u0026...]` |\n| `oracle` | `oracle://[username[:password]@][netloc][:port]/service_name?server=server2\u0026server=server3` |\n| `snowflake` | `username[:password]@account_identifier/dbname/schemaname[?param1=value\u0026...\u0026paramN=valueN]` |\n| `spanner` | `projects/[project]/instances/[instance]/databases/dbname` |\n| `trino` | [`http[s]://user[:pass]@host[:port][?parameters]`](https://github.com/trinodb/trino-go-client#dsn-data-source-name) |\n| `gocosmos` | [`AccountEndpoint=\u003ccosmosdb-endpoint\u003e;AccountKey=\u003ccosmosdb-account-key\u003e[;TimeoutMs=\u003ctimeout-in-ms\u003e][;Version=\u003ccosmosdb-api-version\u003e][;DefaultDb/Db=\u003cdb-name\u003e][;AutoId=\u003ctrue/false\u003e][;InsecureSkipVerify=\u003ctrue/false\u003e]`](https://pkg.go.dev/github.com/microsoft/gocosmos#readme-example-usage) |\n\nPlease note that the `postgres` driver enforces SSL by default, you can override this with the parameter `sslmode=disable` if required.\n\nThe `snowflake` driver supports multiple DSN formats. Please consult [the docs](https://pkg.go.dev/github.com/snowflakedb/gosnowflake#hdr-Connection_String) for more details. For [key pair authentication](https://docs.snowflake.com/en/user-guide/key-pair-auth.html#configuring-key-pair-authentication), the DSN has the following format: `\u003csnowflake_user\u003e@\u003csnowflake_account\u003e/\u003cdb_name\u003e/\u003cschema_name\u003e?warehouse=\u003cwarehouse\u003e\u0026role=\u003crole\u003e\u0026authenticator=snowflake_jwt\u0026privateKey=\u003cbase64_url_encoded_private_key\u003e`, where the value for the `privateKey` parameter can be constructed from an unencrypted RSA private key file `rsa_key.p8` using `openssl enc -d -base64 -in rsa_key.p8 | basenc --base64url -w0` (you can use `gbasenc` insted of `basenc` on OSX if you install `coreutils` via Homebrew). If you have a password-encrypted private key, you can decrypt it using `openssl pkcs8 -in rsa_key_encrypted.p8 -out rsa_key.p8`. Also, make sure fields such as the username are URL-encoded.\n\nThe [`gocosmos`](https://pkg.go.dev/github.com/microsoft/gocosmos) driver is still experimental, but it has support for [hierarchical partition keys](https://learn.microsoft.com/en-us/azure/cosmos-db/hierarchical-partition-keys) as well as [cross-partition queries](https://learn.microsoft.com/en-us/azure/cosmos-db/nosql/how-to-query-container#cross-partition-query). Please refer to the [SQL notes](https://github.com/microsoft/gocosmos/blob/main/SQL.md) for details.", "examples": [ "clickhouse://username:password@host1:9000,host2:9000/database?dial_timeout=200ms\u0026max_execution_time=60", "foouser:foopassword@tcp(localhost:3306)/foodb", "postgres://foouser:foopass@localhost:5432/foodb?sslmode=disable", "oracle://foouser:foopass@localhost:1521/service_name" ] }, "endpoint": { "type": "string", "description": "Allows you to specify a custom endpoint for the AWS API. Default: " }, "exec_only": { "type": "boolean", "description": "Whether the query result should be discarded. When set to `true` the message contents will remain unchanged, which is useful in cases where you are executing inserts, updates, etc. Default: false" }, "iam_enabled": { "type": "boolean", "description": "An optional field used to generate an IAM authentication token to connect to an Amazon Relational Database (RDS) DB instance. This will overwrite the Password in the DSN with the generated token only if the drivers are `mysql` or `postgres`. Default: false" }, "init_files": { "type": "array", "description": "\nAn optional list of file paths containing SQL statements to execute immediately upon the first connection to the target database. This is a useful way to initialise tables before processing data. Glob patterns are supported, including super globs (double star).\n\nCare should be taken to ensure that the statements are idempotent, and therefore would not cause issues when run multiple times after service restarts. If both `init_statement` and `init_files` are specified the `init_statement` is executed _after_ the `init_files`.\n\nIf a statement fails for any reason a warning log will be emitted but the operation of this component will not be stopped.\n", "items": { "type": "string" }, "examples": [ [ "./init/*.sql" ], [ "./foo.sql", "./bar.sql" ] ] }, "init_statement": { "type": "string", "description": "\nAn optional SQL statement to execute immediately upon the first connection to the target database. This is a useful way to initialise tables before processing data. Care should be taken to ensure that the statement is idempotent, and therefore would not cause issues when run multiple times after service restarts.\n\nIf both `init_statement` and `init_files` are specified the `init_statement` is executed _after_ the `init_files`.\n\nIf the statement fails for any reason a warning log will be emitted but the operation of this component will not be stopped.\n", "examples": [ "\nCREATE TABLE IF NOT EXISTS some_table (\n foo varchar(50) not null,\n bar integer,\n baz varchar(50),\n primary key (foo)\n) WITHOUT ROWID;\n" ] }, "init_verify_conn": { "type": "boolean", "description": "Whether to verify the database connection on startup by performing a simple ping, by default this is disabled. Default: false" }, "query": { "type": "string", "description": "The query to execute. The style of placeholder to use depends on the driver, some drivers require question marks (`?`) whereas others expect incrementing dollar signs (`$1`, `$2`, and so on) or colons (`:1`, `:2` and so on). The style to use is outlined in this table:\n\n| Driver | Placeholder Style |\n|---|---|\n| `clickhouse` | Dollar sign |\n| `mysql` | Question mark |\n| `postgres` | Dollar sign |\n| `mssql` | Question mark |\n| `sqlite` | Question mark |\n| `oracle` | Colon |\n| `snowflake` | Question mark |\n| `spanner` | Question mark |\n| `trino` | Question mark |\n| `gocosmos` | Colon |\n", "examples": [ "INSERT INTO footable (foo, bar, baz) VALUES (?, ?, ?);", "SELECT * FROM footable WHERE user_id = $1;" ] }, "region": { "type": "string", "description": "The AWS region to target. Default: " }, "secret_name": { "type": "string", "description": "An optional field that can be used to get the Username + Password from AWS Secrets Manager. This will overwrite the Username + Password in the DSN with the values from the Secret only if the driver is set to `postgres`." }, "unsafe_dynamic_query": { "type": "boolean", "description": "Whether to enable [interpolation functions](/docs/configuration/interpolation/#bloblang-queries) in the query. Great care should be made to ensure your queries are defended against injection attacks. Default: false" } } }, "sql_select": { "type": "object", "properties": { "args_mapping": { "type": "string", "description": "An optional [Bloblang mapping](/docs/guides/bloblang/about) which should evaluate to an array of values matching in size to the number of placeholder arguments in the field `where`.", "examples": [ "root = [ this.cat.meow, this.doc.woofs[0] ]", "root = [ metadata(\"user.id\").string() ]" ] }, "azure": { "type": "object", "description": "Optional Fields that can be set to use Azure based authentication for Azure Postgres SQL", "properties": { "entra_enabled": { "type": "boolean", "description": "An optional field used to generate an entra token to connect to 'Azure Database for PostgreSQL flexible server', This will create a new connection string with the host, user and database from the DSN field - you may need to URL encode the dsn! The [Default Azure Credential Chain](https://learn.microsoft.com/en-gb/azure/developer/go/sdk/authentication/authentication-overview#defaultazurecredential) is used from the Azure SDK. Default: false" }, "token_request_options": { "type": "object", "properties": { "claims": { "type": "string", "description": "Set additional claims for the token. Default: " }, "enable_cae": { "type": "boolean", "description": "Indicates whether to enable Continuous Access Evaluation (CAE) for the requested token Default: false" }, "scopes": { "type": "array", "description": "Scopes contains the list of permission scopes required for the token. Default: [https://ossrdbms-aad.database.windows.net/.default]", "items": { "type": "string" } }, "tenant_id": { "type": "string", "description": "tenant_id identifies the tenant from which to request the token. azure credentials authenticate in their configured default tenants when this field isn't set. Default: " } } } } }, "columns": { "type": "array", "description": "A list of columns to query.", "items": { "type": "string" }, "examples": [ [ "*" ], [ "foo", "bar", "baz" ] ] }, "conn_max_idle": { "type": "integer", "description": "An optional maximum number of connections in the idle connection pool. If conn_max_open is greater than 0 but less than the new conn_max_idle, then the new conn_max_idle will be reduced to match the conn_max_open limit. If `value \u003c= 0`, no idle connections are retained. The default max idle connections is currently 2. This may change in a future release. Default: 2" }, "conn_max_idle_time": { "type": "string", "description": "An optional maximum amount of time a connection may be idle. Expired connections may be closed lazily before reuse. If `value \u003c= 0`, connections are not closed due to a connections idle time." }, "conn_max_life_time": { "type": "string", "description": "An optional maximum amount of time a connection may be reused. Expired connections may be closed lazily before reuse. If `value \u003c= 0`, connections are not closed due to a connections age." }, "conn_max_open": { "type": "integer", "description": "An optional maximum number of open connections to the database. If conn_max_idle is greater than 0 and the new conn_max_open is less than conn_max_idle, then conn_max_idle will be reduced to match the new conn_max_open limit. If `value \u003c= 0`, then there is no limit on the number of open connections. The default is 0 (unlimited)." }, "credentials": { "type": "object", "description": "Optional manual configuration of AWS credentials to use. More information can be found [in this document](/docs/guides/cloud/aws).", "properties": { "expiry_window": { "type": "string", "description": "Allow the credentials to trigger refreshing prior to the credentials actually expiring. This is beneficial so race conditions with expiring credentials do not cause requests to fail. For example '10s' would refresh credentials ten seconds before expiration. Setting to a duration of `0` disables the expiry window. Default: " }, "from_ec2_role": { "type": "boolean", "description": "Use the credentials of a host EC2 machine configured to assume [an IAM role associated with the instance](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html). Default: false" }, "id": { "type": "string", "description": "The ID of credentials to use. Default: " }, "profile": { "type": "string", "description": "A profile from `~/.aws/credentials` to use. Default: " }, "role": { "type": "string", "description": "A role ARN to assume. Default: " }, "role_external_id": { "type": "string", "description": "An external ID to provide when assuming a role. Default: " }, "secret": { "type": "string", "description": "The secret for the credentials being used. Default: " }, "token": { "type": "string", "description": "The token for the credentials being used, required when using short term credentials. Default: " } } }, "driver": { "type": "string", "description": "A database [driver](#drivers) to use." }, "dsn": { "type": "string", "description": "A Data Source Name to identify the target database.\n\n#### Drivers\n\nThe following is a list of supported drivers, their placeholder style, and their respective DSN formats:\n\n| Driver | Data Source Name Format |\n|---|---|\n| `clickhouse` | [`clickhouse://[username[:password]@][netloc][:port]/dbname[?param1=value1\u0026...\u0026paramN=valueN]`](https://github.com/ClickHouse/clickhouse-go#dsn) |\n| `mysql` | `[username[:password]@][protocol[(address)]]/dbname[?param1=value1\u0026...\u0026paramN=valueN]` |\n| `postgres` | `postgres://[user[:password]@][netloc][:port][/dbname][?param1=value1\u0026...]` |\n| `mssql` | `sqlserver://[user[:password]@][netloc][:port][?database=dbname\u0026param1=value1\u0026...]` |\n| `sqlite` | `file:/path/to/filename.db[?param\u0026=value1\u0026...]` |\n| `oracle` | `oracle://[username[:password]@][netloc][:port]/service_name?server=server2\u0026server=server3` |\n| `snowflake` | `username[:password]@account_identifier/dbname/schemaname[?param1=value\u0026...\u0026paramN=valueN]` |\n| `spanner` | `projects/[project]/instances/[instance]/databases/dbname` |\n| `trino` | [`http[s]://user[:pass]@host[:port][?parameters]`](https://github.com/trinodb/trino-go-client#dsn-data-source-name) |\n| `gocosmos` | [`AccountEndpoint=\u003ccosmosdb-endpoint\u003e;AccountKey=\u003ccosmosdb-account-key\u003e[;TimeoutMs=\u003ctimeout-in-ms\u003e][;Version=\u003ccosmosdb-api-version\u003e][;DefaultDb/Db=\u003cdb-name\u003e][;AutoId=\u003ctrue/false\u003e][;InsecureSkipVerify=\u003ctrue/false\u003e]`](https://pkg.go.dev/github.com/microsoft/gocosmos#readme-example-usage) |\n\nPlease note that the `postgres` driver enforces SSL by default, you can override this with the parameter `sslmode=disable` if required.\n\nThe `snowflake` driver supports multiple DSN formats. Please consult [the docs](https://pkg.go.dev/github.com/snowflakedb/gosnowflake#hdr-Connection_String) for more details. For [key pair authentication](https://docs.snowflake.com/en/user-guide/key-pair-auth.html#configuring-key-pair-authentication), the DSN has the following format: `\u003csnowflake_user\u003e@\u003csnowflake_account\u003e/\u003cdb_name\u003e/\u003cschema_name\u003e?warehouse=\u003cwarehouse\u003e\u0026role=\u003crole\u003e\u0026authenticator=snowflake_jwt\u0026privateKey=\u003cbase64_url_encoded_private_key\u003e`, where the value for the `privateKey` parameter can be constructed from an unencrypted RSA private key file `rsa_key.p8` using `openssl enc -d -base64 -in rsa_key.p8 | basenc --base64url -w0` (you can use `gbasenc` insted of `basenc` on OSX if you install `coreutils` via Homebrew). If you have a password-encrypted private key, you can decrypt it using `openssl pkcs8 -in rsa_key_encrypted.p8 -out rsa_key.p8`. Also, make sure fields such as the username are URL-encoded.\n\nThe [`gocosmos`](https://pkg.go.dev/github.com/microsoft/gocosmos) driver is still experimental, but it has support for [hierarchical partition keys](https://learn.microsoft.com/en-us/azure/cosmos-db/hierarchical-partition-keys) as well as [cross-partition queries](https://learn.microsoft.com/en-us/azure/cosmos-db/nosql/how-to-query-container#cross-partition-query). Please refer to the [SQL notes](https://github.com/microsoft/gocosmos/blob/main/SQL.md) for details.", "examples": [ "clickhouse://username:password@host1:9000,host2:9000/database?dial_timeout=200ms\u0026max_execution_time=60", "foouser:foopassword@tcp(localhost:3306)/foodb", "postgres://foouser:foopass@localhost:5432/foodb?sslmode=disable", "oracle://foouser:foopass@localhost:1521/service_name" ] }, "endpoint": { "type": "string", "description": "Allows you to specify a custom endpoint for the AWS API. Default: " }, "iam_enabled": { "type": "boolean", "description": "An optional field used to generate an IAM authentication token to connect to an Amazon Relational Database (RDS) DB instance. This will overwrite the Password in the DSN with the generated token only if the drivers are `mysql` or `postgres`. Default: false" }, "init_files": { "type": "array", "description": "\nAn optional list of file paths containing SQL statements to execute immediately upon the first connection to the target database. This is a useful way to initialise tables before processing data. Glob patterns are supported, including super globs (double star).\n\nCare should be taken to ensure that the statements are idempotent, and therefore would not cause issues when run multiple times after service restarts. If both `init_statement` and `init_files` are specified the `init_statement` is executed _after_ the `init_files`.\n\nIf a statement fails for any reason a warning log will be emitted but the operation of this component will not be stopped.\n", "items": { "type": "string" }, "examples": [ [ "./init/*.sql" ], [ "./foo.sql", "./bar.sql" ] ] }, "init_statement": { "type": "string", "description": "\nAn optional SQL statement to execute immediately upon the first connection to the target database. This is a useful way to initialise tables before processing data. Care should be taken to ensure that the statement is idempotent, and therefore would not cause issues when run multiple times after service restarts.\n\nIf both `init_statement` and `init_files` are specified the `init_statement` is executed _after_ the `init_files`.\n\nIf the statement fails for any reason a warning log will be emitted but the operation of this component will not be stopped.\n", "examples": [ "\nCREATE TABLE IF NOT EXISTS some_table (\n foo varchar(50) not null,\n bar integer,\n baz varchar(50),\n primary key (foo)\n) WITHOUT ROWID;\n" ] }, "init_verify_conn": { "type": "boolean", "description": "Whether to verify the database connection on startup by performing a simple ping, by default this is disabled. Default: false" }, "prefix": { "type": "string", "description": "An optional prefix to prepend to the query (before SELECT)." }, "region": { "type": "string", "description": "The AWS region to target. Default: " }, "secret_name": { "type": "string", "description": "An optional field that can be used to get the Username + Password from AWS Secrets Manager. This will overwrite the Username + Password in the DSN with the values from the Secret only if the driver is set to `postgres`." }, "suffix": { "type": "string", "description": "An optional suffix to append to the select query." }, "table": { "type": "string", "description": "The table to query.", "examples": [ "foo" ] }, "where": { "type": "string", "description": "An optional where clause to add. Placeholder arguments are populated with the `args_mapping` field. Placeholders should always be question marks, and will automatically be converted to dollar syntax when the postgres or clickhouse drivers are used.", "examples": [ "meow = ? and woof = ?", "user_id = ?" ] } } }, "subprocess": { "type": "object", "properties": { "args": { "type": "array", "description": "A list of arguments to provide the command. Default: []", "items": { "type": "string" } }, "codec_recv": { "type": "string", "description": "Determines how messages read from the subprocess are decoded, which allows them to be logically separated. Default: lines" }, "codec_send": { "type": "string", "description": "Determines how messages written to the subprocess are encoded, which allows them to be logically separated. Default: lines" }, "max_buffer": { "type": "integer", "description": "The maximum expected response size. Default: 65536" }, "name": { "type": "string", "description": "The command to execute as a subprocess.", "examples": [ "cat", "sed", "awk" ] } } }, "switch": { "type": "array", "items": { "type": "object" } }, "sync_response": { "type": "object", "description": " Default: map[]" }, "try": { "type": "array", "description": " Default: []", "items": { "type": "object" } }, "unarchive": { "type": "object", "properties": { "format": { "type": "string", "description": "The unarchiving format to apply." } } }, "wasm": { "type": "object", "properties": { "function": { "type": "string", "description": "The name of the function exported by the target WASM module to run for each message. Default: process" }, "module_path": { "type": "string", "description": "The path of the target WASM module to execute." } } }, "while": { "type": "object", "properties": { "at_least_once": { "type": "boolean", "description": "Whether to always run the child processors at least one time. Default: false" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether the while loop should execute again. Default: ", "examples": [ "errored()", "this.urls.unprocessed.length() \u003e 0" ] }, "max_loops": { "type": "integer", "description": "An optional maximum number of loops to execute. Helps protect against accidentally creating infinite loops. Default: 0" }, "processors": { "type": "array", "description": "A list of child processors to execute on each loop.", "items": { "type": "object" } } } }, "workflow": { "type": "object", "properties": { "branch_resources": { "type": "array", "description": "An optional list of [`branch` processor](/docs/components/processors/branch) names that are configured as [resources](#resources). These resources will be included in the workflow with any branches configured inline within the [`branches`](#branches) field. The order and parallelism in which branches are executed is automatically resolved based on the mappings of each branch. When using resources with an explicit order it is not necessary to list resources in this field. Default: []", "items": { "type": "string" } }, "branches": { "type": "object", "description": "An object of named [`branch` processors](/docs/components/processors/branch) that make up the workflow. The order and parallelism in which branches are executed can either be made explicit with the field `order`, or if omitted an attempt is made to automatically resolve an ordering based on the mappings of each branch. Default: map[]", "properties": { "processors": { "type": "array", "description": "A list of processors to apply to mapped requests. When processing message batches the resulting batch must match the size and ordering of the input batch, therefore filtering, grouping should not be performed within these processors.", "items": { "type": "object" } }, "request_map": { "type": "string", "description": "A [Bloblang mapping](/docs/guides/bloblang/about) that describes how to create a request payload suitable for the child processors of this branch. If left empty then the branch will begin with an exact copy of the origin message (including metadata). Default: ", "examples": [ "root = {\n\t\"id\": this.doc.id,\n\t\"content\": this.doc.body.text\n}", "root = if this.type == \"foo\" {\n\tthis.foo.request\n} else {\n\tdeleted()\n}" ] }, "result_map": { "type": "string", "description": "A [Bloblang mapping](/docs/guides/bloblang/about) that describes how the resulting messages from branched processing should be mapped back into the original payload. If left empty the origin message will remain unchanged (including metadata). Default: ", "examples": [ "meta foo_code = metadata(\"code\")\nroot.foo_result = this", "meta = metadata()\nroot.bar.body = this.body\nroot.bar.id = this.user.id", "root.raw_result = content().string()", "root.enrichments.foo = if metadata(\"request_failed\") != null {\n throw(metadata(\"request_failed\"))\n} else {\n this\n}", "# Retain only the updated metadata fields which were present in the origin message\nmeta = metadata().filter(v -\u003e @.get(v.key) != null)" ] } } }, "meta_path": { "type": "string", "description": "A [dot path](/docs/configuration/field_paths) indicating where to store and reference [structured metadata](#structured-metadata) about the workflow execution. Default: meta.workflow" }, "order": { "type": "array", "description": "An explicit declaration of branch ordered tiers, which describes the order in which parallel tiers of branches should be executed. Branches should be identified by the name as they are configured in the field `branches`. It's also possible to specify branch processors configured [as a resource](#resources). Default: []", "items": { "items": { "type": "array" } }, "examples": [ [ [ "foo", "bar" ], [ "baz" ] ], [ [ "foo" ], [ "bar" ], [ "baz" ] ] ] } } }, "xml": { "type": "object", "properties": { "cast": { "type": "boolean", "description": "Whether to try to cast values that are numbers and booleans to the right type. Default: all values are strings. Default: false" }, "operator": { "type": "string", "description": "An XML [operation](#operators) to apply to messages. Default: " } } } } } }, "threads": { "type": "integer", "description": "The number of threads to execute processing pipelines across. Default: -1" } } }, "processor_resources": { "type": "array", "description": "A list of processor resources, each must have a unique label. Default: []", "items": { "type": "object", "properties": { "archive": { "type": "object", "properties": { "format": { "type": "string", "description": "The archiving format to apply." }, "path": { "type": "string", "description": "The path to set for each message in the archive (when applicable). Default: ", "examples": [ "${!count(\"files\")}-${!timestamp_unix_nano()}.txt", "${!meta(\"kafka_key\")}-${!json(\"id\")}.json" ] } } }, "avro": { "type": "object", "properties": { "encoding": { "type": "string", "description": "An Avro encoding format to use for conversions to and from a schema. Default: textual" }, "operator": { "type": "string", "description": "The [operator](#operators) to execute" }, "schema": { "type": "string", "description": "A full Avro schema to use. Default: " }, "schema_path": { "type": "string", "description": "The path of a schema document to apply. Use either this or the `schema` field. Default: ", "examples": [ "file://path/to/spec.avsc", "http://localhost:8081/path/to/spec/versions/1" ] } } }, "awk": { "type": "object", "properties": { "codec": { "type": "string", "description": "A [codec](#codecs) defines how messages should be inserted into the AWK program as variables. The codec does not change which [custom Bento functions](#awk-functions) are available. The `text` codec is the closest to a typical AWK use case." }, "program": { "type": "string", "description": "An AWK program to execute" } } }, "aws_dynamodb_partiql": { "type": "object", "properties": { "args_mapping": { "type": "string", "description": "A [Bloblang mapping](/docs/guides/bloblang/about) that, for each message, creates a list of arguments to use with the query. Default: " }, "credentials": { "type": "object", "description": "Optional manual configuration of AWS credentials to use. More information can be found [in this document](/docs/guides/cloud/aws).", "properties": { "expiry_window": { "type": "string", "description": "Allow the credentials to trigger refreshing prior to the credentials actually expiring. This is beneficial so race conditions with expiring credentials do not cause requests to fail. For example '10s' would refresh credentials ten seconds before expiration. Setting to a duration of `0` disables the expiry window. Default: " }, "from_ec2_role": { "type": "boolean", "description": "Use the credentials of a host EC2 machine configured to assume [an IAM role associated with the instance](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html). Default: false" }, "id": { "type": "string", "description": "The ID of credentials to use. Default: " }, "profile": { "type": "string", "description": "A profile from `~/.aws/credentials` to use. Default: " }, "role": { "type": "string", "description": "A role ARN to assume. Default: " }, "role_external_id": { "type": "string", "description": "An external ID to provide when assuming a role. Default: " }, "secret": { "type": "string", "description": "The secret for the credentials being used. Default: " }, "token": { "type": "string", "description": "The token for the credentials being used, required when using short term credentials. Default: " } } }, "endpoint": { "type": "string", "description": "Allows you to specify a custom endpoint for the AWS API. Default: " }, "query": { "type": "string", "description": "A PartiQL query to execute for each message." }, "region": { "type": "string", "description": "The AWS region to target. Default: " }, "unsafe_dynamic_query": { "type": "boolean", "description": "Whether to enable dynamic queries that support interpolation functions. Default: false" } } }, "aws_lambda": { "type": "object", "properties": { "credentials": { "type": "object", "description": "Optional manual configuration of AWS credentials to use. More information can be found [in this document](/docs/guides/cloud/aws).", "properties": { "expiry_window": { "type": "string", "description": "Allow the credentials to trigger refreshing prior to the credentials actually expiring. This is beneficial so race conditions with expiring credentials do not cause requests to fail. For example '10s' would refresh credentials ten seconds before expiration. Setting to a duration of `0` disables the expiry window. Default: " }, "from_ec2_role": { "type": "boolean", "description": "Use the credentials of a host EC2 machine configured to assume [an IAM role associated with the instance](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html). Default: false" }, "id": { "type": "string", "description": "The ID of credentials to use. Default: " }, "profile": { "type": "string", "description": "A profile from `~/.aws/credentials` to use. Default: " }, "role": { "type": "string", "description": "A role ARN to assume. Default: " }, "role_external_id": { "type": "string", "description": "An external ID to provide when assuming a role. Default: " }, "secret": { "type": "string", "description": "The secret for the credentials being used. Default: " }, "token": { "type": "string", "description": "The token for the credentials being used, required when using short term credentials. Default: " } } }, "endpoint": { "type": "string", "description": "Allows you to specify a custom endpoint for the AWS API. Default: " }, "function": { "type": "string", "description": "The function to invoke." }, "parallel": { "type": "boolean", "description": "Whether messages of a batch should be dispatched in parallel. Default: false" }, "rate_limit": { "type": "string", "description": "An optional [`rate_limit`](/docs/components/rate_limits/about) to throttle invocations by. Default: " }, "region": { "type": "string", "description": "The AWS region to target. Default: " }, "retries": { "type": "integer", "description": "The maximum number of retry attempts for each message. Default: 3" }, "timeout": { "type": "string", "description": "The maximum period of time to wait before abandoning an invocation. Default: 5s" } } }, "aws_s3": { "type": "object", "properties": { "bucket": { "type": "string", "description": "The bucket to perform the GetObject operation on." }, "credentials": { "type": "object", "description": "Optional manual configuration of AWS credentials to use. More information can be found [in this document](/docs/guides/cloud/aws).", "properties": { "expiry_window": { "type": "string", "description": "Allow the credentials to trigger refreshing prior to the credentials actually expiring. This is beneficial so race conditions with expiring credentials do not cause requests to fail. For example '10s' would refresh credentials ten seconds before expiration. Setting to a duration of `0` disables the expiry window. Default: " }, "from_ec2_role": { "type": "boolean", "description": "Use the credentials of a host EC2 machine configured to assume [an IAM role associated with the instance](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html). Default: false" }, "id": { "type": "string", "description": "The ID of credentials to use. Default: " }, "profile": { "type": "string", "description": "A profile from `~/.aws/credentials` to use. Default: " }, "role": { "type": "string", "description": "A role ARN to assume. Default: " }, "role_external_id": { "type": "string", "description": "An external ID to provide when assuming a role. Default: " }, "secret": { "type": "string", "description": "The secret for the credentials being used. Default: " }, "token": { "type": "string", "description": "The token for the credentials being used, required when using short term credentials. Default: " } } }, "delete_objects": { "type": "boolean", "description": "Whether to delete downloaded objects from the bucket once they are processed. Note: the S3 Object will be deleted from AWS as soon as this processor has consumed the object. Default: false" }, "endpoint": { "type": "string", "description": "Allows you to specify a custom endpoint for the AWS API. Default: " }, "force_path_style_urls": { "type": "boolean", "description": "Forces the client API to use path style URLs for downloading keys, which is often required when connecting to custom endpoints. Default: false" }, "key": { "type": "string", "description": "The key of the object you wish to retrive." }, "region": { "type": "string", "description": "The AWS region to target. Default: " }, "scanner": { "type": "object", "description": "The [scanner](/docs/components/scanners/about) by which the stream of bytes consumed will be broken out into individual messages. Scanners are useful for processing large sources of data without holding the entirety of it within memory. For example, the `csv` scanner allows you to process individual CSV rows without loading the entire CSV file in memory at once. Default: map[to_the_end:map[]]" } } }, "azure_cosmosdb": { "type": "object", "properties": { "account_key": { "type": "string", "description": "Account key.", "examples": [ "C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==" ] }, "auto_id": { "type": "boolean", "description": "Automatically set the item `id` field to a random UUID v4. If the `id` field is already set, then it will not be overwritten. Setting this to `false` can improve performance, since the messages will not have to be parsed. Default: true" }, "connection_string": { "type": "string", "description": "Connection string.", "examples": [ "AccountEndpoint=https://localhost:8081/;AccountKey=C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==;" ] }, "container": { "type": "string", "description": "Container.", "examples": [ "testcontainer" ] }, "database": { "type": "string", "description": "Database.", "examples": [ "testdb" ] }, "enable_content_response_on_write": { "type": "boolean", "description": "Enable content response on write operations. To save some bandwidth, set this to false if you don't need to receive the updated message(s) from the server, in which case the processor will not modify the content of the messages which are fed into it. Applies to every operation except Read. Default: true" }, "endpoint": { "type": "string", "description": "CosmosDB endpoint.", "examples": [ "https://localhost:8081" ] }, "item_id": { "type": "string", "description": "ID of item to replace or delete. Only used by the Replace and Delete operations", "examples": [ "${! json(\"id\") }" ] }, "operation": { "type": "string", "description": "Operation. Default: Create" }, "partition_keys_map": { "type": "string", "description": "A [Bloblang mapping](/docs/guides/bloblang/about) which should evaluate to a single partition key value or an array of partition key values of type string, integer or boolean. Currently, hierarchical partition keys are not supported so only one value may be provided.", "examples": [ "root = \"blobfish\"", "root = 41", "root = true", "root = null", "root = json(\"blobfish\").depth" ] }, "patch_condition": { "type": "string", "description": "Patch operation condition.", "examples": [ "from c where not is_defined(c.blobfish)" ] }, "patch_operations": { "type": "array", "description": "Patch operations to be performed when `operation: Patch` .", "items": { "type": "object" } } } }, "bloblang": { "type": "string", "description": " Default: " }, "bounds_check": { "type": "object", "properties": { "max_part_size": { "type": "integer", "description": "The maximum size of a message to allow (in bytes) Default: 1073741824" }, "max_parts": { "type": "integer", "description": "The maximum size of message batches to allow (in message count) Default: 100" }, "min_part_size": { "type": "integer", "description": "The minimum size of a message to allow (in bytes) Default: 1" }, "min_parts": { "type": "integer", "description": "The minimum size of message batches to allow (in message count) Default: 1" } } }, "branch": { "type": "object", "properties": { "processors": { "type": "array", "description": "A list of processors to apply to mapped requests. When processing message batches the resulting batch must match the size and ordering of the input batch, therefore filtering, grouping should not be performed within these processors.", "items": { "type": "object" } }, "request_map": { "type": "string", "description": "A [Bloblang mapping](/docs/guides/bloblang/about) that describes how to create a request payload suitable for the child processors of this branch. If left empty then the branch will begin with an exact copy of the origin message (including metadata). Default: ", "examples": [ "root = {\n\t\"id\": this.doc.id,\n\t\"content\": this.doc.body.text\n}", "root = if this.type == \"foo\" {\n\tthis.foo.request\n} else {\n\tdeleted()\n}" ] }, "result_map": { "type": "string", "description": "A [Bloblang mapping](/docs/guides/bloblang/about) that describes how the resulting messages from branched processing should be mapped back into the original payload. If left empty the origin message will remain unchanged (including metadata). Default: ", "examples": [ "meta foo_code = metadata(\"code\")\nroot.foo_result = this", "meta = metadata()\nroot.bar.body = this.body\nroot.bar.id = this.user.id", "root.raw_result = content().string()", "root.enrichments.foo = if metadata(\"request_failed\") != null {\n throw(metadata(\"request_failed\"))\n} else {\n this\n}", "# Retain only the updated metadata fields which were present in the origin message\nmeta = metadata().filter(v -\u003e @.get(v.key) != null)" ] } } }, "cache": { "type": "object", "properties": { "key": { "type": "string", "description": "A key to use with the cache." }, "operator": { "type": "string", "description": "The [operation](#operators) to perform with the cache." }, "resource": { "type": "string", "description": "The [`cache` resource](/docs/components/caches/about) to target with this processor." }, "ttl": { "type": "string", "description": "The TTL of each individual item as a duration string. After this period an item will be eligible for removal during the next compaction. Not all caches support per-key TTLs, those that do will have a configuration field `default_ttl`, and those that do not will fall back to their generally configured TTL setting.", "examples": [ "60s", "5m", "36h" ] }, "value": { "type": "string", "description": "A value to use with the cache (when applicable)." } } }, "cached": { "type": "object", "properties": { "cache": { "type": "string", "description": "The cache resource to read and write processor results from." }, "key": { "type": "string", "description": "A key to be resolved for each message, if the key already exists in the cache then the cached result is used, otherwise the processors are applied and the result is cached under this key. The key could be static and therefore apply generally to all messages or it could be an interpolated expression that is potentially unique for each message.", "examples": [ "my_foo_result", "${! this.document.id }", "${! metadata(\"kafka_key\") }", "${! metadata(\"kafka_topic\") }" ] }, "processors": { "type": "array", "description": "The list of processors whose result will be cached.", "items": { "type": "object" } }, "skip_on": { "type": "string", "description": "A condition that can be used to skip caching the results from the processors.", "examples": [ "errored()" ] }, "ttl": { "type": "string", "description": "An optional expiry period to set for each cache entry. Some caches only have a general TTL and will therefore ignore this setting." } } }, "catch": { "type": "array", "description": " Default: []", "items": { "type": "object" } }, "command": { "type": "object", "properties": { "args_mapping": { "type": "string", "description": "An optional [Bloblang mapping](/docs/guides/bloblang/about) that, when specified, should resolve into an array of arguments to pass to the command. Command arguments are expressed this way in order to support dynamic behaviour.", "examples": [ "[ \"-c\", this.script_path ]" ] }, "name": { "type": "string", "description": "The name of the command to execute.", "examples": [ "bash", "go", "${! @command }" ] } } }, "compress": { "type": "object", "properties": { "algorithm": { "type": "string", "description": "The compression algorithm to use." }, "level": { "type": "integer", "description": "The level of compression to use. May not be applicable to all algorithms. Default: -1" } } }, "couchbase": { "type": "object", "properties": { "bucket": { "type": "string", "description": "Couchbase bucket." }, "cas_enabled": { "type": "boolean", "description": "Enable CAS validation. Default: true" }, "collection": { "type": "string", "description": "Bucket collection. Default: _default" }, "content": { "type": "string", "description": "Document content." }, "id": { "type": "string", "description": "Document id.", "examples": [ "${! json(\"id\") }" ] }, "operation": { "type": "string", "description": "Couchbase operation to perform. Default: get" }, "password": { "type": "string", "description": "Password to connect to the cluster." }, "timeout": { "type": "string", "description": "Operation timeout. Default: 15s" }, "transcoder": { "type": "string", "description": "Couchbase transcoder to use. Default: legacy" }, "ttl": { "type": "string", "description": "An optional TTL to set for items." }, "url": { "type": "string", "description": "Couchbase connection string.", "examples": [ "couchbase://localhost:11210" ] }, "username": { "type": "string", "description": "Username to connect to the cluster." } } }, "decompress": { "type": "object", "properties": { "algorithm": { "type": "string", "description": "The decompression algorithm to use." } } }, "dedupe": { "type": "object", "properties": { "cache": { "type": "string", "description": "The [`cache` resource](/docs/components/caches/about) to target with this processor." }, "drop_on_err": { "type": "boolean", "description": "Whether messages should be dropped when the cache returns a general error such as a network issue. Default: true" }, "key": { "type": "string", "description": "An interpolated string yielding the key to deduplicate by for each message.", "examples": [ "${! metadata(\"kafka_key\") }", "${! content().hash(\"xxhash64\") }" ] }, "strategy": { "type": "string", "description": "Controls how to handle duplicate values. Default: FIFO" } } }, "for_each": { "type": "array", "description": " Default: []", "items": { "type": "object" } }, "gcp_bigquery_select": { "type": "object", "properties": { "args_mapping": { "type": "string", "description": "An optional [Bloblang mapping](/docs/guides/bloblang/about) which should evaluate to an array of values matching in size to the number of placeholder arguments in the field `where`.", "examples": [ "root = [ \"article\", now().ts_format(\"2006-01-02\") ]" ] }, "columns": { "type": "array", "description": "A list of columns to query.", "items": { "type": "string" } }, "columns_mapping": { "type": "string", "description": "An optional [Bloblang mapping](/docs/guides/bloblang/about) which should evaluate to an array of column names to query." }, "job_labels": { "type": "object", "description": "A list of labels to add to the query job. Default: map[]" }, "prefix": { "type": "string", "description": "An optional prefix to prepend to the select query (before SELECT)." }, "project": { "type": "string", "description": "GCP project where the query job will execute." }, "suffix": { "type": "string", "description": "An optional suffix to append to the select query." }, "table": { "type": "string", "description": "Fully-qualified BigQuery table name to query.", "examples": [ "bigquery-public-data.samples.shakespeare" ] }, "unsafe_dynamic_query": { "type": "boolean", "description": "Whether to enable [interpolation functions](/docs/configuration/interpolation/#bloblang-queries) in the columns_mapping, table \u0026 where fields. When `unsafe_dynamic_query` is set to true, you should provide a bloblang mapping via the `columns_mapping` config field, and not `columns`. Great care should be made to ensure your queries are defended against injection attacks. Default: false" }, "where": { "type": "string", "description": "An optional where clause to add. Placeholder arguments are populated with the `args_mapping` field. Placeholders should always be question marks (`?`).", "examples": [ "type = ? and created_at \u003e ?", "user_id = ?" ] } } }, "grok": { "type": "object", "properties": { "expressions": { "type": "array", "description": "One or more Grok expressions to attempt against incoming messages. The first expression to match at least one value will be used to form a result.", "items": { "type": "string" } }, "named_captures_only": { "type": "boolean", "description": "Whether to only capture values from named patterns. Default: true" }, "pattern_definitions": { "type": "object", "description": "A map of pattern definitions that can be referenced within `patterns`. Default: map[]" }, "pattern_paths": { "type": "array", "description": "A list of paths to load Grok patterns from. This field supports wildcards, including super globs (double star). Default: []", "items": { "type": "string" } }, "remove_empty_values": { "type": "boolean", "description": "Whether to remove values that are empty from the resulting structure. Default: true" }, "use_default_patterns": { "type": "boolean", "description": "Whether to use a [default set of patterns](#default-patterns). Default: true" } } }, "group_by": { "type": "array", "items": { "type": "object" } }, "group_by_value": { "type": "object", "properties": { "value": { "type": "string", "description": "The interpolated string to group based on.", "examples": [ "${! meta(\"kafka_key\") }", "${! json(\"foo.bar\") }-${! meta(\"baz\") }" ] } } }, "http": { "type": "object", "properties": { "backoff_on": { "type": "integer", "description": "A list of status codes whereby the request should be considered to have failed and retries should be attempted, but the period between them should be increased gradually. Default: [429]", "items": { "type": "integer" } }, "basic_auth": { "type": "object", "description": "Allows you to specify basic authentication.", "properties": { "enabled": { "type": "boolean", "description": "Whether to use basic authentication in requests. Default: false" }, "password": { "type": "string", "description": "A password to authenticate with. Default: " }, "username": { "type": "string", "description": "A username to authenticate as. Default: " } } }, "batch_as_multipart": { "type": "boolean", "description": "Send message batches as a single request using [RFC1341](https://www.w3.org/Protocols/rfc1341/7_2_Multipart.html). Default: false" }, "drop_on": { "type": "integer", "description": "A list of status codes whereby the request should be considered to have failed but retries should not be attempted. This is useful for preventing wasted retries for requests that will never succeed. Note that with these status codes the _request_ is dropped, but _message_ that caused the request will not be dropped. Default: []", "items": { "type": "integer" } }, "dump_request_log_level": { "type": "string", "description": "EXPERIMENTAL: Optionally set a level at which the request and response payload of each request made will be logged. Default: " }, "extract_headers": { "type": "object", "description": "Specify which response headers should be added to resulting messages as metadata. Header keys are lowercased before matching, so ensure that your patterns target lowercased versions of the header keys that you expect.", "properties": { "include_patterns": { "type": "array", "description": "Provide a list of explicit metadata key regular expression (re2) patterns to match against. Default: []", "items": { "type": "string" }, "examples": [ [ ".*" ], [ "_timestamp_unix$" ] ] }, "include_prefixes": { "type": "array", "description": "Provide a list of explicit metadata key prefixes to match against. Default: []", "items": { "type": "string" }, "examples": [ [ "foo_", "bar_" ], [ "kafka_" ], [ "content-" ] ] } } }, "headers": { "type": "object", "description": "A map of headers to add to the request. Default: map[]", "examples": [ { "Content-Type": "application/octet-stream", "traceparent": "${! tracing_span().traceparent }" } ] }, "jwt": { "type": "object", "description": "BETA: Allows you to specify JWT authentication.", "properties": { "claims": { "type": "object", "description": "A value used to identify the claims that issued the JWT. Default: map[]" }, "enabled": { "type": "boolean", "description": "Whether to use JWT authentication in requests. Default: false" }, "headers": { "type": "object", "description": "Add optional key/value headers to the JWT. Default: map[]" }, "private_key_file": { "type": "string", "description": "A file with the PEM encoded via PKCS1 or PKCS8 as private key. Default: " }, "signing_method": { "type": "string", "description": "A method used to sign the token such as RS256, RS384, RS512 or EdDSA. Default: " } } }, "max_retry_backoff": { "type": "string", "description": "The maximum period to wait between failed requests. Default: 300s" }, "metadata": { "type": "object", "description": "Specify optional matching rules to determine which metadata keys should be added to the HTTP request as headers.", "properties": { "include_patterns": { "type": "array", "description": "Provide a list of explicit metadata key regular expression (re2) patterns to match against. Default: []", "items": { "type": "string" }, "examples": [ [ ".*" ], [ "_timestamp_unix$" ] ] }, "include_prefixes": { "type": "array", "description": "Provide a list of explicit metadata key prefixes to match against. Default: []", "items": { "type": "string" }, "examples": [ [ "foo_", "bar_" ], [ "kafka_" ], [ "content-" ] ] } } }, "oauth": { "type": "object", "description": "Allows you to specify open authentication via OAuth version 1.", "properties": { "access_token": { "type": "string", "description": "A value used to gain access to the protected resources on behalf of the user. Default: " }, "access_token_secret": { "type": "string", "description": "A secret provided in order to establish ownership of a given access token. Default: " }, "consumer_key": { "type": "string", "description": "A value used to identify the client to the service provider. Default: " }, "consumer_secret": { "type": "string", "description": "A secret used to establish ownership of the consumer key. Default: " }, "enabled": { "type": "boolean", "description": "Whether to use OAuth version 1 in requests. Default: false" } } }, "oauth2": { "type": "object", "description": "Allows you to specify open authentication via OAuth version 2 using the client credentials token flow.", "properties": { "client_key": { "type": "string", "description": "A value used to identify the client to the token provider. Default: " }, "client_secret": { "type": "string", "description": "A secret used to establish ownership of the client key. Default: " }, "enabled": { "type": "boolean", "description": "Whether to use OAuth version 2 in requests. Default: false" }, "endpoint_params": { "type": "object", "description": "A list of optional endpoint parameters, values should be arrays of strings. Default: map[]", "examples": [ { "bar": [ "woof" ], "foo": [ "meow", "quack" ] } ] }, "scopes": { "type": "array", "description": "A list of optional requested permissions. Default: []", "items": { "type": "string" } }, "token_url": { "type": "string", "description": "The URL of the token provider. Default: " } } }, "parallel": { "type": "boolean", "description": "When processing batched messages, whether to send messages of the batch in parallel, otherwise they are sent serially. Default: false" }, "proxy_url": { "type": "string", "description": "An optional HTTP proxy URL." }, "rate_limit": { "type": "string", "description": "An optional [rate limit](/docs/components/rate_limits/about) to throttle requests by." }, "retries": { "type": "integer", "description": "The maximum number of retry attempts to make. Default: 3" }, "retry_period": { "type": "string", "description": "The base period to wait between failed requests. Default: 1s" }, "successful_on": { "type": "integer", "description": "A list of status codes whereby the attempt should be considered successful, this is useful for dropping requests that return non-2XX codes indicating that the message has been dealt with, such as a 303 See Other or a 409 Conflict. All 2XX codes are considered successful unless they are present within `backoff_on` or `drop_on`, regardless of this field. Default: []", "items": { "type": "integer" } }, "timeout": { "type": "string", "description": "A static timeout to apply to requests. Default: 5s" }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "transport": { "type": "object", "description": "Custom transport options.", "properties": { "dial_context": { "type": "object", "description": "Settings for the dialer used to create new connections.", "properties": { "keep_alive": { "type": "string", "description": "Keep-alive period for active network connections used by the dialer. Default: 30s" }, "timeout": { "type": "string", "description": "Timeout for establishing new network connections. Default: 30s" } } }, "expect_continue_timeout": { "type": "string", "description": "Time to wait for a server's first response headers after sending request headers when 'Expect: 100-continue' is used. Zero means send body immediately. Default: 1s" }, "force_http2": { "type": "boolean", "description": "If true, the transport will attempt to use HTTP/2. Default: true" }, "idle_connection_timeout": { "type": "string", "description": "Maximum time an idle keep-alive connection remains open before closing itself. Default: 90s" }, "max_idle_connections": { "type": "integer", "description": "Maximum number of idle keep-alive connections. Zero = unlimited. Default: 100" }, "tls_handshake_timeout": { "type": "string", "description": "Maximum time allowed for TLS handshake to complete. Default: 10s" } } }, "url": { "type": "string", "description": "The URL to connect to." }, "verb": { "type": "string", "description": "A verb to connect with Default: POST", "examples": [ "POST", "GET", "DELETE" ] } } }, "insert_part": { "type": "object", "properties": { "content": { "type": "string", "description": "The content of the message being inserted. Default: " }, "index": { "type": "integer", "description": "The index within the batch to insert the message at. Default: -1" } } }, "javascript": { "type": "object", "properties": { "code": { "type": "string", "description": "An inline JavaScript program to run. One of `code` or `file` must be defined." }, "file": { "type": "string", "description": "A file containing a JavaScript program to run. One of `code` or `file` must be defined." }, "global_folders": { "type": "array", "description": "List of folders that will be used to load modules from if the requested JS module is not found elsewhere. Default: []", "items": { "type": "string" } } } }, "jmespath": { "type": "object", "properties": { "query": { "type": "string", "description": "The JMESPath query to apply to messages." } } }, "jq": { "type": "object", "properties": { "output_raw": { "type": "boolean", "description": "Whether to output raw text (unquoted) instead of JSON strings when the emitted values are string types. Default: false" }, "query": { "type": "string", "description": "The jq query to filter and transform messages with." }, "raw": { "type": "boolean", "description": "Whether to process the input as a raw string instead of as JSON. Default: false" } } }, "json_schema": { "type": "object", "properties": { "schema": { "type": "string", "description": "A schema to apply. Use either this or the `schema_path` field." }, "schema_path": { "type": "string", "description": "The path of a schema document to apply. Use either this or the `schema` field." } } }, "log": { "type": "object", "properties": { "fields": { "type": "object", "description": "A map of fields to print along with the log message." }, "fields_mapping": { "type": "string", "description": "An optional [Bloblang mapping](/docs/guides/bloblang/about) that can be used to specify extra fields to add to the log. If log fields are also added with the `fields` field then those values will override matching keys from this mapping.", "examples": [ "root.reason = \"cus I wana\"\nroot.id = this.id\nroot.age = this.user.age.number()\nroot.kafka_topic = metadata(\"kafka_topic\")" ] }, "level": { "type": "string", "description": "The log level to use. Default: INFO" }, "message": { "type": "string", "description": "The message to print. Default: " } } }, "mapping": { "type": "string" }, "metric": { "type": "object", "properties": { "labels": { "type": "object", "description": "A map of label names and values that can be used to enrich metrics. Labels are not supported by some metric destinations, in which case the metrics series are combined.", "examples": [ { "topic": "${! meta(\"kafka_topic\") }", "type": "${! json(\"doc.type\") }" } ] }, "name": { "type": "string", "description": "The name of the metric to create, this must be unique across all Bento components otherwise it will overwrite those other metrics." }, "type": { "type": "string", "description": "The metric [type](#types) to create." }, "value": { "type": "string", "description": "For some metric types specifies a value to set, increment. Certain metrics exporters such as Prometheus support floating point values, but those that do not will cast a floating point value into an integer. Default: " } } }, "mongodb": { "type": "object", "properties": { "backoff": { "type": "object", "description": "Control time intervals between retry attempts.", "properties": { "initial_interval": { "type": "string", "description": "The initial period to wait between retry attempts. Default: 1s" }, "max_elapsed_time": { "type": "string", "description": "The maximum period to wait before retry attempts are abandoned. If zero then no limit is used. Default: 30s" }, "max_interval": { "type": "string", "description": "The maximum period to wait between retry attempts. Default: 5s" } } }, "collection": { "type": "string", "description": "The name of the target collection." }, "database": { "type": "string", "description": "The name of the target MongoDB database." }, "document_map": { "type": "string", "description": "A bloblang map representing a document to store within MongoDB, expressed as [extended JSON in canonical form](https://www.mongodb.com/docs/manual/reference/mongodb-extended-json/). The document map is required for the operations insert-one, replace-one and update-one. Default: ", "examples": [ "root.a = this.foo\nroot.b = this.bar" ] }, "filter_map": { "type": "string", "description": "A bloblang map representing a filter for a MongoDB command, expressed as [extended JSON in canonical form](https://www.mongodb.com/docs/manual/reference/mongodb-extended-json/). The filter map is required for all operations except insert-one. It is used to find the document(s) for the operation. For example in a delete-one case, the filter map should have the fields required to locate the document to delete. Default: ", "examples": [ "root.a = this.foo\nroot.b = this.bar" ] }, "hint_map": { "type": "string", "description": "A bloblang map representing the hint for the MongoDB command, expressed as [extended JSON in canonical form](https://www.mongodb.com/docs/manual/reference/mongodb-extended-json/). This map is optional and is used with all operations except insert-one. It is used to improve performance of finding the documents in the mongodb. Default: ", "examples": [ "root.a = this.foo\nroot.b = this.bar" ] }, "json_marshal_mode": { "type": "string", "description": "The json_marshal_mode setting is optional and controls the format of the output message. Default: canonical" }, "max_retries": { "type": "integer", "description": "The maximum number of retries before giving up on the request. If set to zero there is no discrete limit. Default: 3" }, "operation": { "type": "string", "description": "The mongodb operation to perform. Default: insert-one" }, "password": { "type": "string", "description": "The password to connect to the database. Default: " }, "upsert": { "type": "boolean", "description": "The upsert setting is optional and only applies for update-one and replace-one operations. If the filter specified in filter_map matches, the document is updated or replaced accordingly, otherwise it is created. Default: false" }, "url": { "type": "string", "description": "The URL of the target MongoDB server.", "examples": [ "mongodb://localhost:27017" ] }, "username": { "type": "string", "description": "The username to connect to the database. Default: " }, "write_concern": { "type": "object", "description": "The write concern settings for the mongo connection.", "properties": { "j": { "type": "boolean", "description": "J requests acknowledgement from MongoDB that write operations are written to the journal. Default: false" }, "w": { "type": "string", "description": "W requests acknowledgement that write operations propagate to the specified number of mongodb instances. Default: " }, "w_timeout": { "type": "string", "description": "The write concern timeout. Default: " } } } } }, "msgpack": { "type": "object", "properties": { "operator": { "type": "string", "description": "The operation to perform on messages." } } }, "mutation": { "type": "string" }, "nats_kv": { "type": "object", "properties": { "auth": { "type": "object", "description": "Optional configuration of NATS authentication parameters.", "properties": { "nkey_file": { "type": "string", "description": "An optional file containing a NKey seed.", "examples": [ "./seed.nk" ] }, "user_credentials_file": { "type": "string", "description": "An optional file containing user credentials which consist of an user JWT and corresponding NKey seed.", "examples": [ "./user.creds" ] }, "user_jwt": { "type": "string", "description": "An optional plain text user JWT (given along with the corresponding user NKey Seed)." }, "user_nkey_seed": { "type": "string", "description": "An optional plain text user NKey Seed (given along with the corresponding user JWT)." } } }, "bucket": { "type": "string", "description": "The name of the KV bucket.", "examples": [ "my_kv_bucket" ] }, "key": { "type": "string", "description": "The key for each message. Supports [wildcards](https://docs.nats.io/nats-concepts/subjects#wildcards) for the `history` and `keys` operations.", "examples": [ "foo", "foo.bar.baz", "foo.*", "foo.\u003e", "foo.${! json(\"meta.type\") }" ] }, "operation": { "type": "string", "description": "The operation to perform on the KV bucket." }, "revision": { "type": "string", "description": "The revision of the key to operate on. Used for `get_revision` and `update` operations.", "examples": [ "42", "${! @nats_kv_revision }" ] }, "timeout": { "type": "string", "description": "The maximum period to wait on an operation before aborting and returning an error. Default: 5s" }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "urls": { "type": "array", "description": "A list of URLs to connect to. If an item of the list contains commas it will be expanded into multiple URLs.", "items": { "type": "string" }, "examples": [ [ "nats://127.0.0.1:4222" ], [ "nats://username:password@127.0.0.1:4222" ] ] } } }, "nats_object_store": { "type": "object", "properties": { "auth": { "type": "object", "description": "Optional configuration of NATS authentication parameters.", "properties": { "nkey_file": { "type": "string", "description": "An optional file containing a NKey seed.", "examples": [ "./seed.nk" ] }, "user_credentials_file": { "type": "string", "description": "An optional file containing user credentials which consist of an user JWT and corresponding NKey seed.", "examples": [ "./user.creds" ] }, "user_jwt": { "type": "string", "description": "An optional plain text user JWT (given along with the corresponding user NKey Seed)." }, "user_nkey_seed": { "type": "string", "description": "An optional plain text user NKey Seed (given along with the corresponding user JWT)." } } }, "bucket": { "type": "string", "description": "The name of the object store bucket.", "examples": [ "my_bucket" ] }, "object_name": { "type": "string", "description": "The name of the object in the object store to operate on." }, "operation": { "type": "string", "description": "The operation to perform on the Object Store bucket." }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "urls": { "type": "array", "description": "A list of URLs to connect to. If an item of the list contains commas it will be expanded into multiple URLs.", "items": { "type": "string" }, "examples": [ [ "nats://127.0.0.1:4222" ], [ "nats://username:password@127.0.0.1:4222" ] ] } } }, "nats_request_reply": { "type": "object", "properties": { "auth": { "type": "object", "description": "Optional configuration of NATS authentication parameters.", "properties": { "nkey_file": { "type": "string", "description": "An optional file containing a NKey seed.", "examples": [ "./seed.nk" ] }, "user_credentials_file": { "type": "string", "description": "An optional file containing user credentials which consist of an user JWT and corresponding NKey seed.", "examples": [ "./user.creds" ] }, "user_jwt": { "type": "string", "description": "An optional plain text user JWT (given along with the corresponding user NKey Seed)." }, "user_nkey_seed": { "type": "string", "description": "An optional plain text user NKey Seed (given along with the corresponding user JWT)." } } }, "headers": { "type": "object", "description": "Explicit message headers to add to messages. Default: map[]", "examples": [ { "Content-Type": "application/json", "Timestamp": "${!metadata(\"Timestamp\").string()}" } ] }, "inbox_prefix": { "type": "string", "description": "Set an explicit inbox prefix for the response subject", "examples": [ "_INBOX_joe" ] }, "metadata": { "type": "object", "description": "Determine which (if any) metadata values should be added to messages as headers.", "properties": { "include_patterns": { "type": "array", "description": "Provide a list of explicit metadata key regular expression (re2) patterns to match against. Default: []", "items": { "type": "string" }, "examples": [ [ ".*" ], [ "_timestamp_unix$" ] ] }, "include_prefixes": { "type": "array", "description": "Provide a list of explicit metadata key prefixes to match against. Default: []", "items": { "type": "string" }, "examples": [ [ "foo_", "bar_" ], [ "kafka_" ], [ "content-" ] ] } } }, "subject": { "type": "string", "description": "A subject to write to.", "examples": [ "foo.bar.baz", "${! metadata(\"kafka_topic\") }", "foo.${! json(\"meta.type\") }" ] }, "timeout": { "type": "string", "description": "A duration string is a possibly signed sequence of decimal numbers, each with optional fraction and a unit suffix, such as 300ms, -1.5h or 2h45m. Valid time units are ns, us (or µs), ms, s, m, h. Default: 3s" }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "urls": { "type": "array", "description": "A list of URLs to connect to. If an item of the list contains commas it will be expanded into multiple URLs.", "items": { "type": "string" }, "examples": [ [ "nats://127.0.0.1:4222" ], [ "nats://username:password@127.0.0.1:4222" ] ] } } }, "nlp_classify_text": { "type": "object", "properties": { "aggregation_function": { "type": "string", "description": "The aggregation function to use for the text classification pipeline. Default: SOFTMAX" }, "download_options": { "type": "object", "description": "Options used to download a model directly from HuggingFace. Before the model is downloaded, validation occurs to ensure the remote repository contains both an`.onnx` and `tokenizers.json` file.", "properties": { "onnx_filepath": { "type": "string", "description": "Filepath of the ONNX model within the repository. Only needed when multiple `.onnx` files exist. Default: model.onnx", "examples": [ "onnx/model.onnx", "onnx/model_quantized.onnx", "onnx/model_fp16.onnx" ] }, "repository": { "type": "string", "description": "The name of the huggingface model repository.", "examples": [ "KnightsAnalytics/distilbert-NER", "KnightsAnalytics/distilbert-base-uncased-finetuned-sst-2-english", "sentence-transformers/all-MiniLM-L6-v2" ] } } }, "enable_download": { "type": "boolean", "description": "When enabled, attempts to download an ONNX Runtime compatible model from HuggingFace specified in `repository`. Default: false" }, "multi_label": { "type": "boolean", "description": "Whether a text classification pipeline should return multiple labels. If false, only the label-pair with the highest score is returned. Default: false" }, "name": { "type": "string", "description": "Name of the hugot pipeline. Defaults to a random UUID if not set." }, "path": { "type": "string", "description": "Path to the ONNX model file, or directory containing the model. When downloading (`enable_download: true`), this becomes the destination and must be a directory.", "examples": [ "/path/to/models/my_model.onnx", "/path/to/models/" ] } } }, "nlp_classify_tokens": { "type": "object", "properties": { "aggregation_strategy": { "type": "string", "description": "The aggregation strategy to use for the token classification pipeline. Default: SIMPLE" }, "download_options": { "type": "object", "description": "Options used to download a model directly from HuggingFace. Before the model is downloaded, validation occurs to ensure the remote repository contains both an`.onnx` and `tokenizers.json` file.", "properties": { "onnx_filepath": { "type": "string", "description": "Filepath of the ONNX model within the repository. Only needed when multiple `.onnx` files exist. Default: model.onnx", "examples": [ "onnx/model.onnx", "onnx/model_quantized.onnx", "onnx/model_fp16.onnx" ] }, "repository": { "type": "string", "description": "The name of the huggingface model repository.", "examples": [ "KnightsAnalytics/distilbert-NER", "KnightsAnalytics/distilbert-base-uncased-finetuned-sst-2-english", "sentence-transformers/all-MiniLM-L6-v2" ] } } }, "enable_download": { "type": "boolean", "description": "When enabled, attempts to download an ONNX Runtime compatible model from HuggingFace specified in `repository`. Default: false" }, "ignore_labels": { "type": "array", "description": "Labels to ignore in the token classification pipeline. Default: []", "items": { "type": "string" }, "examples": [ [ "O", "MISC" ] ] }, "name": { "type": "string", "description": "Name of the hugot pipeline. Defaults to a random UUID if not set." }, "path": { "type": "string", "description": "Path to the ONNX model file, or directory containing the model. When downloading (`enable_download: true`), this becomes the destination and must be a directory.", "examples": [ "/path/to/models/my_model.onnx", "/path/to/models/" ] } } }, "nlp_extract_features": { "type": "object", "properties": { "download_options": { "type": "object", "description": "Options used to download a model directly from HuggingFace. Before the model is downloaded, validation occurs to ensure the remote repository contains both an`.onnx` and `tokenizers.json` file.", "properties": { "onnx_filepath": { "type": "string", "description": "Filepath of the ONNX model within the repository. Only needed when multiple `.onnx` files exist. Default: model.onnx", "examples": [ "onnx/model.onnx", "onnx/model_quantized.onnx", "onnx/model_fp16.onnx" ] }, "repository": { "type": "string", "description": "The name of the huggingface model repository.", "examples": [ "KnightsAnalytics/distilbert-NER", "KnightsAnalytics/distilbert-base-uncased-finetuned-sst-2-english", "sentence-transformers/all-MiniLM-L6-v2" ] } } }, "enable_download": { "type": "boolean", "description": "When enabled, attempts to download an ONNX Runtime compatible model from HuggingFace specified in `repository`. Default: false" }, "name": { "type": "string", "description": "Name of the hugot pipeline. Defaults to a random UUID if not set." }, "normalization": { "type": "boolean", "description": "Whether to apply normalization in the feature extraction pipeline. Default: false" }, "path": { "type": "string", "description": "Path to the ONNX model file, or directory containing the model. When downloading (`enable_download: true`), this becomes the destination and must be a directory.", "examples": [ "/path/to/models/my_model.onnx", "/path/to/models/" ] } } }, "nlp_zero_shot_classify": { "type": "object", "properties": { "download_options": { "type": "object", "description": "Options used to download a model directly from HuggingFace. Before the model is downloaded, validation occurs to ensure the remote repository contains both an`.onnx` and `tokenizers.json` file.", "properties": { "onnx_filepath": { "type": "string", "description": "Filepath of the ONNX model within the repository. Only needed when multiple `.onnx` files exist. Default: model.onnx", "examples": [ "onnx/model.onnx", "onnx/model_quantized.onnx", "onnx/model_fp16.onnx" ] }, "repository": { "type": "string", "description": "The name of the huggingface model repository.", "examples": [ "KnightsAnalytics/distilbert-NER", "KnightsAnalytics/distilbert-base-uncased-finetuned-sst-2-english", "sentence-transformers/all-MiniLM-L6-v2" ] } } }, "enable_download": { "type": "boolean", "description": "When enabled, attempts to download an ONNX Runtime compatible model from HuggingFace specified in `repository`. Default: false" }, "hypothesis_template": { "type": "string", "description": "Template to turn each label into an NLI-style hypothesis. Must include {} where the label will be inserted. Default: This example is {}." }, "labels": { "type": "array", "description": "The set of possible class labels to classify each sequence into.", "items": { "type": "string" }, "examples": [ [ "positive", "negative", "neutral" ] ] }, "multi_label": { "type": "boolean", "description": "Whether multiple labels can be true. If false, scores sum to 1. If true, each label is scored independently. Default: false" }, "name": { "type": "string", "description": "Name of the hugot pipeline. Defaults to a random UUID if not set." }, "path": { "type": "string", "description": "Path to the ONNX model file, or directory containing the model. When downloading (`enable_download: true`), this becomes the destination and must be a directory.", "examples": [ "/path/to/models/my_model.onnx", "/path/to/models/" ] } } }, "noop": { "type": "object", "description": " Default: map[]" }, "opensnowcat": { "type": "object", "properties": { "filters": { "type": "object", "description": "Filter and transformation configurations", "properties": { "drop": { "type": "object", "description": "Map of field names to filter criteria. Events matching ANY criteria will be dropped (OR logic). Supports both regular TSV columns (e.g., `user_ipaddress`, `useragent`) and schema property paths (e.g., `com.snowplowanalytics.snowplow.ua_parser_context.useragentFamily`). Each filter uses 'contains' for substring matching." }, "transform": { "type": "object", "description": "Field transformation configuration for anonymization, hashing, and redaction", "properties": { "fields": { "type": "object", "description": "Map of field names to transformation configurations. Each field must specify:\n- **strategy** (required): Transformation type - \"hash\", \"redact\", or \"anonymize_ip\"\n- **hash_algo** (optional): Algorithm for hash strategy - \"MD5\", \"SHA-1\", \"SHA-256\", \"SHA-384\", \"SHA-512\" (overrides global default)\n- **salt** (optional): Salt for hash strategy (overrides global default)\n- **redact_value** (optional): Replacement value for redact strategy (default: \"[REDACTED]\")\n- **anon_octets** (optional): Number of IPv4 octets to mask for anonymize_ip strategy (default: 0)\n- **anon_segments** (optional): Number of IPv6 segments to mask for anonymize_ip strategy (default: 0)\n\nSupports both TSV columns (e.g., user_id, user_ipaddress) and schema property paths (e.g., com.vendor.schema.field)." }, "hash_algo": { "type": "string", "description": "Global default hash algorithm. Can be overridden per field. Default: SHA-256" }, "salt": { "type": "string", "description": "Global default salt for hashing operations. Can be overridden per field." } } } } }, "output_format": { "type": "string", "description": "Output format for processed events. Default: tsv" }, "schema_discovery": { "type": "object", "description": "Schema discovery configuration", "properties": { "enabled": { "type": "boolean", "description": "Enable schema discovery feature Default: false" }, "endpoint": { "type": "string", "description": "HTTP endpoint to send schema discovery data Default: https://api.snowcatcloud.com/internal/schema-discovery" }, "flush_interval": { "type": "string", "description": "Interval between schema discovery flushes Default: 5m" }, "template": { "type": "string", "description": "Template for schema discovery payload. Use `{{SCHEMAS}}` variable for schema list Default: {\"schemas\": {{SCHEMAS}}}" } } }, "set_metadata": { "type": "object", "description": "Map metadata keys to OpenSnowcat canonical event model field names. Supports direct TSV column names (e.g., 'event_fingerprint', 'app_id') and schema property paths (e.g., 'com.vendor.schema.field'). Metadata is set before any filters or transformations are applied.", "examples": [ { "app_id": "app_id", "collector_tstamp": "collector_tstamp", "eid": "event_id", "fingerprint": "event_fingerprint" } ] } } }, "parallel": { "type": "object", "properties": { "cap": { "type": "integer", "description": "The maximum number of messages to have processing at a given time. Default: 0" }, "processors": { "type": "array", "description": "A list of child processors to apply.", "items": { "type": "object" } } } }, "parquet_decode": { "type": "object", "properties": { "byte_array_as_string": { "type": "boolean", "description": "Whether to extract BYTE_ARRAY and FIXED_LEN_BYTE_ARRAY values as strings rather than byte slices in all cases. Values with a logical type of UTF8 will automatically be extracted as strings irrespective of this field. Enabling this field makes serialising the data as JSON more intuitive as `[]byte` values are serialised as base64 encoded strings by default. Default: false" }, "strict_schema": { "type": "boolean", "description": "Whether to enforce strict Parquet schema validation. When set to false, allows reading files with non-standard schema structures (such as non-standard LIST formats). Disabling strict mode may reduce validation but increases compatibility. Default: true" }, "use_parquet_list_format": { "type": "boolean", "description": "Whether to decode`LIST`type columns into their Parquet logical type format `{\"list\": [{\"element\": value_1}, {\"element\": value_2}, ...]}` instead of a Go slice `[value_1, value_2, ...]`.\n\n:::caution\nThis flag will be disabled (set to `false`) by default and deprecated in future versions, with the logical format being deprecated in favour of the Go slice.\n::: Default: true" } } }, "parquet_encode": { "type": "object", "properties": { "default_compression": { "type": "string", "description": "The default compression type to use for fields. Default: uncompressed" }, "default_encoding": { "type": "string", "description": "The default encoding type to use for fields. A custom default encoding is only necessary when consuming data with libraries that do not support `DELTA_LENGTH_BYTE_ARRAY` and is therefore best left unset where possible. Default: DELTA_LENGTH_BYTE_ARRAY" }, "schema": { "type": "array", "description": "Parquet schema.", "items": { "type": "object" } } } }, "parse_log": { "type": "object", "properties": { "allow_rfc3339": { "type": "boolean", "description": "Also accept timestamps in rfc3339 format while parsing. Applicable to format `syslog_rfc3164`. Default: true" }, "best_effort": { "type": "boolean", "description": "Still returns partially parsed messages even if an error occurs. Default: true" }, "codec": { "type": "string" }, "default_timezone": { "type": "string", "description": "Sets the strategy to decide the timezone for rfc3164 timestamps. Applicable to format `syslog_rfc3164`. This value should follow the [time.LoadLocation](https://golang.org/pkg/time/#LoadLocation) format. Default: UTC" }, "default_year": { "type": "string", "description": "Sets the strategy used to set the year for rfc3164 timestamps. Applicable to format `syslog_rfc3164`. When set to `current` the current year will be set, when set to an integer that value will be used. Leave this field empty to not set a default year at all. Default: current" }, "format": { "type": "string", "description": "A common log [format](#formats) to parse." } } }, "processors": { "type": "array", "description": " Default: []", "items": { "type": "object" } }, "protobuf": { "type": "object", "properties": { "bsr": { "type": "array", "description": "Buf Schema Registry configuration. Either this field or `import_paths` must be populated. Note that this field is an array, and multiple BSR configurations can be provided. Default: []", "items": { "type": "object" } }, "discard_unknown": { "type": "boolean", "description": "If `true`, the `from_json` operator discards fields that are unknown to the schema. Default: false" }, "import_paths": { "type": "array", "description": "A list of directories containing .proto files or list of file paths, including all definitions required for parsing the target message. If left empty the current directory is used. Each directory listed will be walked with all found .proto files imported. Either this field or `bsr` must be populated. Default: []", "items": { "type": "string" } }, "message": { "type": "string", "description": "The fully qualified name of the protobuf message to convert to/from." }, "operator": { "type": "string", "description": "The [operator](#operators) to execute" }, "use_proto_names": { "type": "boolean", "description": "If `true`, the `to_json` operator deserializes fields exactly as named in schema file. Default: false" } } }, "rate_limit": { "type": "object", "properties": { "resource": { "type": "string", "description": "The target [`rate_limit` resource](/docs/components/rate_limits/about)." } } }, "redis": { "type": "object", "properties": { "args_mapping": { "type": "string", "description": "A [Bloblang mapping](/docs/guides/bloblang/about) which should evaluate to an array of values matching in size to the number of arguments required for the specified Redis command.", "examples": [ "root = [ this.key ]", "root = [ metadata(\"kafka_key\"), this.count ]" ] }, "command": { "type": "string", "description": "The command to execute.", "examples": [ "scard", "incrby", "${! metadata(\"command\") }" ] }, "key": { "type": "string", "description": "A key to use for the target operator." }, "kind": { "type": "string", "description": "Specifies a simple, cluster-aware, or failover-aware redis client. Default: simple" }, "master": { "type": "string", "description": "Name of the redis master when `kind` is `failover` Default: ", "examples": [ "mymaster" ] }, "operator": { "type": "string", "description": "The operator to apply." }, "retries": { "type": "integer", "description": "The maximum number of retries before abandoning a request. Default: 3" }, "retry_period": { "type": "string", "description": "The time to wait before consecutive retry attempts. Default: 500ms" }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.\n\n**Troubleshooting**\n\nSome cloud hosted instances of Redis (such as Azure Cache) might need some hand holding in order to establish stable connections. Unfortunately, it is often the case that TLS issues will manifest as generic error messages such as \"i/o timeout\". If you're using TLS and are seeing connectivity problems consider setting `enable_renegotiation` to `true`, and ensuring that the server supports at least TLS version 1.2.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "url": { "type": "string", "description": "The URL of the target Redis server. Database is optional and is supplied as the URL path.", "examples": [ "redis://:6397", "redis://localhost:6379", "redis://foousername:foopassword@redisplace:6379", "redis://:foopassword@redisplace:6379", "redis://localhost:6379/1", "redis://localhost:6379/1,redis://localhost:6380/1" ] } } }, "redis_script": { "type": "object", "properties": { "args_mapping": { "type": "string", "description": "A [Bloblang mapping](/docs/guides/bloblang/about) which should evaluate to an array of values matching in size to the number of arguments required for the specified Redis script.", "examples": [ "root = [ this.key ]", "root = [ metadata(\"kafka_key\"), \"hardcoded_value\" ]" ] }, "keys_mapping": { "type": "string", "description": "A [Bloblang mapping](/docs/guides/bloblang/about) which should evaluate to an array of keys matching in size to the number of arguments required for the specified Redis script.", "examples": [ "root = [ this.key ]", "root = [ metadata(\"kafka_key\"), this.count ]" ] }, "kind": { "type": "string", "description": "Specifies a simple, cluster-aware, or failover-aware redis client. Default: simple" }, "master": { "type": "string", "description": "Name of the redis master when `kind` is `failover` Default: ", "examples": [ "mymaster" ] }, "retries": { "type": "integer", "description": "The maximum number of retries before abandoning a request. Default: 3" }, "retry_period": { "type": "string", "description": "The time to wait before consecutive retry attempts. Default: 500ms" }, "script": { "type": "string", "description": "A script to use for the target operator. It has precedence over the 'command' field.", "examples": [ "return redis.call('set', KEYS[1], ARGV[1])" ] }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.\n\n**Troubleshooting**\n\nSome cloud hosted instances of Redis (such as Azure Cache) might need some hand holding in order to establish stable connections. Unfortunately, it is often the case that TLS issues will manifest as generic error messages such as \"i/o timeout\". If you're using TLS and are seeing connectivity problems consider setting `enable_renegotiation` to `true`, and ensuring that the server supports at least TLS version 1.2.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "url": { "type": "string", "description": "The URL of the target Redis server. Database is optional and is supplied as the URL path.", "examples": [ "redis://:6397", "redis://localhost:6379", "redis://foousername:foopassword@redisplace:6379", "redis://:foopassword@redisplace:6379", "redis://localhost:6379/1", "redis://localhost:6379/1,redis://localhost:6380/1" ] } } }, "resource": { "type": "string", "description": " Default: " }, "retry": { "type": "object", "properties": { "backoff": { "type": "object", "description": "Determine time intervals and cut offs for retry attempts.", "properties": { "initial_interval": { "type": "string", "description": "The initial period to wait between retry attempts. Default: 500ms", "examples": [ "50ms", "1s" ] }, "max_elapsed_time": { "type": "string", "description": "The maximum overall period of time to spend on retry attempts before the request is aborted. Setting this value to a zeroed duration (such as `0s`) will result in unbounded retries. Default: 1m", "examples": [ "1m", "1h" ] }, "max_interval": { "type": "string", "description": "The maximum period to wait between retry attempts Default: 10s", "examples": [ "5s", "1m" ] } } }, "max_retries": { "type": "integer", "description": "The maximum number of retry attempts before the request is aborted. Setting this value to `0` will result in unbounded number of retries. Default: 0" }, "parallel": { "type": "boolean", "description": "When processing batches of messages these batches are ignored and the processors apply to each message sequentially. However, when this field is set to `true` each message will be processed in parallel. Caution should be made to ensure that batch sizes do not surpass a point where this would cause resource (CPU, memory, API limits) contention. Default: false" }, "processors": { "type": "array", "description": "A list of [processors](/docs/components/processors/about/) to execute on each message.", "items": { "type": "object" } } } }, "schema_registry_decode": { "type": "object", "properties": { "avro_nested_schemas": { "type": "boolean", "description": "Whether Avro Schemas are nested. If true bento will resolve schema references. (Up to a maximum depth of 100) Default: false" }, "avro_raw_json": { "type": "boolean", "description": "Whether Avro messages should be decoded into normal JSON (\"json that meets the expectations of regular internet json\") rather than [Avro JSON](https://avro.apache.org/docs/current/specification/_print/#json-encoding). If `true` the schema returned from the subject should be decoded as [standard json](https://pkg.go.dev/github.com/linkedin/goavro/v2#NewCodecForStandardJSONFull) instead of as [avro json](https://pkg.go.dev/github.com/linkedin/goavro/v2#NewCodec). There is a [comment in goavro](https://github.com/linkedin/goavro/blob/5ec5a5ee7ec82e16e6e2b438d610e1cab2588393/union.go#L224-L249), the [underlining library used for avro serialization](https://github.com/linkedin/goavro), that explains in more detail the difference between the standard json and avro json. Default: false" }, "basic_auth": { "type": "object", "description": "Allows you to specify basic authentication.", "properties": { "enabled": { "type": "boolean", "description": "Whether to use basic authentication in requests. Default: false" }, "password": { "type": "string", "description": "A password to authenticate with. Default: " }, "username": { "type": "string", "description": "A username to authenticate as. Default: " } } }, "jwt": { "type": "object", "description": "BETA: Allows you to specify JWT authentication.", "properties": { "claims": { "type": "object", "description": "A value used to identify the claims that issued the JWT. Default: map[]" }, "enabled": { "type": "boolean", "description": "Whether to use JWT authentication in requests. Default: false" }, "headers": { "type": "object", "description": "Add optional key/value headers to the JWT. Default: map[]" }, "private_key_file": { "type": "string", "description": "A file with the PEM encoded via PKCS1 or PKCS8 as private key. Default: " }, "signing_method": { "type": "string", "description": "A method used to sign the token such as RS256, RS384, RS512 or EdDSA. Default: " } } }, "oauth": { "type": "object", "description": "Allows you to specify open authentication via OAuth version 1.", "properties": { "access_token": { "type": "string", "description": "A value used to gain access to the protected resources on behalf of the user. Default: " }, "access_token_secret": { "type": "string", "description": "A secret provided in order to establish ownership of a given access token. Default: " }, "consumer_key": { "type": "string", "description": "A value used to identify the client to the service provider. Default: " }, "consumer_secret": { "type": "string", "description": "A secret used to establish ownership of the consumer key. Default: " }, "enabled": { "type": "boolean", "description": "Whether to use OAuth version 1 in requests. Default: false" } } }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "transport": { "type": "object", "description": "Custom transport options.", "properties": { "dial_context": { "type": "object", "description": "Settings for the dialer used to create new connections.", "properties": { "keep_alive": { "type": "string", "description": "Keep-alive period for active network connections used by the dialer. Default: 30s" }, "timeout": { "type": "string", "description": "Timeout for establishing new network connections. Default: 30s" } } }, "expect_continue_timeout": { "type": "string", "description": "Time to wait for a server's first response headers after sending request headers when 'Expect: 100-continue' is used. Zero means send body immediately. Default: 1s" }, "force_http2": { "type": "boolean", "description": "If true, the transport will attempt to use HTTP/2. Default: true" }, "idle_connection_timeout": { "type": "string", "description": "Maximum time an idle keep-alive connection remains open before closing itself. Default: 90s" }, "max_idle_connections": { "type": "integer", "description": "Maximum number of idle keep-alive connections. Zero = unlimited. Default: 100" }, "tls_handshake_timeout": { "type": "string", "description": "Maximum time allowed for TLS handshake to complete. Default: 10s" } } }, "url": { "type": "string", "description": "The base URL of the schema registry service." } } }, "schema_registry_encode": { "type": "object", "properties": { "avro_nested_schemas": { "type": "boolean", "description": "Whether Avro Schemas are nested. If true bento will resolve schema references. (Up to a maximum depth of 100) Default: false" }, "avro_raw_json": { "type": "boolean", "description": "Whether messages encoded in Avro format should be parsed as normal JSON (\"json that meets the expectations of regular internet json\") rather than [Avro JSON](https://avro.apache.org/docs/current/specification/_print/#json-encoding). If `true` the schema returned from the subject should be parsed as [standard json](https://pkg.go.dev/github.com/linkedin/goavro/v2#NewCodecForStandardJSONFull) instead of as [avro json](https://pkg.go.dev/github.com/linkedin/goavro/v2#NewCodec). There is a [comment in goavro](https://github.com/linkedin/goavro/blob/5ec5a5ee7ec82e16e6e2b438d610e1cab2588393/union.go#L224-L249), the [underlining library used for avro serialization](https://github.com/linkedin/goavro), that explains in more detail the difference between standard json and avro json. Default: false" }, "basic_auth": { "type": "object", "description": "Allows you to specify basic authentication.", "properties": { "enabled": { "type": "boolean", "description": "Whether to use basic authentication in requests. Default: false" }, "password": { "type": "string", "description": "A password to authenticate with. Default: " }, "username": { "type": "string", "description": "A username to authenticate as. Default: " } } }, "jwt": { "type": "object", "description": "BETA: Allows you to specify JWT authentication.", "properties": { "claims": { "type": "object", "description": "A value used to identify the claims that issued the JWT. Default: map[]" }, "enabled": { "type": "boolean", "description": "Whether to use JWT authentication in requests. Default: false" }, "headers": { "type": "object", "description": "Add optional key/value headers to the JWT. Default: map[]" }, "private_key_file": { "type": "string", "description": "A file with the PEM encoded via PKCS1 or PKCS8 as private key. Default: " }, "signing_method": { "type": "string", "description": "A method used to sign the token such as RS256, RS384, RS512 or EdDSA. Default: " } } }, "oauth": { "type": "object", "description": "Allows you to specify open authentication via OAuth version 1.", "properties": { "access_token": { "type": "string", "description": "A value used to gain access to the protected resources on behalf of the user. Default: " }, "access_token_secret": { "type": "string", "description": "A secret provided in order to establish ownership of a given access token. Default: " }, "consumer_key": { "type": "string", "description": "A value used to identify the client to the service provider. Default: " }, "consumer_secret": { "type": "string", "description": "A secret used to establish ownership of the consumer key. Default: " }, "enabled": { "type": "boolean", "description": "Whether to use OAuth version 1 in requests. Default: false" } } }, "refresh_period": { "type": "string", "description": "The period after which a schema is refreshed for each subject, this is done by polling the schema registry service. Default: 10m", "examples": [ "60s", "1h" ] }, "subject": { "type": "string", "description": "The schema subject to derive schemas from.", "examples": [ "foo", "${! metadata(\"kafka_topic\") }" ] }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "transport": { "type": "object", "description": "Custom transport options.", "properties": { "dial_context": { "type": "object", "description": "Settings for the dialer used to create new connections.", "properties": { "keep_alive": { "type": "string", "description": "Keep-alive period for active network connections used by the dialer. Default: 30s" }, "timeout": { "type": "string", "description": "Timeout for establishing new network connections. Default: 30s" } } }, "expect_continue_timeout": { "type": "string", "description": "Time to wait for a server's first response headers after sending request headers when 'Expect: 100-continue' is used. Zero means send body immediately. Default: 1s" }, "force_http2": { "type": "boolean", "description": "If true, the transport will attempt to use HTTP/2. Default: true" }, "idle_connection_timeout": { "type": "string", "description": "Maximum time an idle keep-alive connection remains open before closing itself. Default: 90s" }, "max_idle_connections": { "type": "integer", "description": "Maximum number of idle keep-alive connections. Zero = unlimited. Default: 100" }, "tls_handshake_timeout": { "type": "string", "description": "Maximum time allowed for TLS handshake to complete. Default: 10s" } } }, "url": { "type": "string", "description": "The base URL of the schema registry service." } } }, "select_parts": { "type": "object", "properties": { "parts": { "type": "integer", "description": "An array of message indexes of a batch. Indexes can be negative, and if so the part will be selected from the end counting backwards starting from -1. Default: []", "items": { "type": "integer" } } } }, "sentry_capture": { "type": "object", "properties": { "context": { "type": "string", "description": "A mapping that must evaluate to an object-of-objects or `deleted()`. If this mapping produces a value, then it is set on a sentry event as additional context.", "examples": [ "root = {\"order\": {\"product_id\": \"P93174\", \"quantity\": 5}}", "root = deleted()" ] }, "dsn": { "type": "string", "description": "The DSN address to send sentry events to. If left empty, then SENTRY_DSN is used. Default: " }, "environment": { "type": "string", "description": "The environment to be sent with events. If left empty, then SENTRY_ENVIRONMENT is used. Default: " }, "flush_timeout": { "type": "string", "description": "The duration to wait when closing the processor to flush any remaining enqueued events. Default: 5s" }, "level": { "type": "string", "description": "Sets the level on sentry events similar to logging levels. Default: INFO" }, "message": { "type": "string", "description": "A message to set on the sentry event", "examples": [ "webhook event received", "failed to find product in database: ${! error() }" ] }, "release": { "type": "string", "description": "The version of the code deployed to an environment. If left empty, then the Sentry client will attempt to detect the release from the environment. Default: " }, "sampling_rate": { "type": "number", "description": "The rate at which events are sent to the server. A value of 0 disables capturing sentry events entirely. A value of 1 results in sending all events to Sentry. Any value in between results sending some percentage of events. Default: 1" }, "tags": { "type": "object", "description": "Sets key/value string tags on an event. Unlike context, these are indexed and searchable on Sentry but have length limitations." }, "transport_mode": { "type": "string", "description": "Determines how events are sent. A sync transport will block when sending each event until a response is received from the Sentry server. The recommended async transport will enqueue events in a buffer and send them in the background. Default: async" } } }, "sleep": { "type": "object", "properties": { "duration": { "type": "string", "description": "The duration of time to sleep for each execution." } } }, "split": { "type": "object", "properties": { "byte_size": { "type": "integer", "description": "An optional target of total message bytes. Default: 0" }, "size": { "type": "integer", "description": "The target number of messages. Default: 1" } } }, "sql": { "type": "object", "properties": { "args_mapping": { "type": "string", "description": "An optional [Bloblang mapping](/docs/guides/bloblang/about) which should evaluate to an array of values matching in size to the number of placeholder arguments in the field `query`.", "examples": [ "root = [ this.cat.meow, this.doc.woofs[0] ]", "root = [ metadata(\"user.id\").string() ]" ] }, "data_source_name": { "type": "string", "description": "Data source name." }, "driver": { "type": "string", "description": "A database [driver](#drivers) to use." }, "query": { "type": "string", "description": "The query to execute. The style of placeholder to use depends on the driver, some drivers require question marks (`?`) whereas others expect incrementing dollar signs (`$1`, `$2`, and so on) or colons (`:1`, `:2` and so on). The style to use is outlined in this table:\n\n| Driver | Placeholder Style |\n|---|---|\n| `clickhouse` | Dollar sign |\n| `mysql` | Question mark |\n| `postgres` | Dollar sign |\n| `mssql` | Question mark |\n| `sqlite` | Question mark |\n| `oracle` | Colon |\n| `snowflake` | Question mark |\n| `spanner` | Question mark |\n| `trino` | Question mark |\n| `gocosmos` | Colon |\n", "examples": [ "INSERT INTO footable (foo, bar, baz) VALUES (?, ?, ?);" ] }, "result_codec": { "type": "string", "description": "Result codec. Default: none" }, "unsafe_dynamic_query": { "type": "boolean", "description": "Whether to enable [interpolation functions](/docs/configuration/interpolation/#bloblang-queries) in the query. Great care should be made to ensure your queries are defended against injection attacks. Default: false" } } }, "sql_insert": { "type": "object", "properties": { "args_mapping": { "type": "string", "description": "A [Bloblang mapping](/docs/guides/bloblang/about) which should evaluate to an array of values matching in size to the number of columns specified.", "examples": [ "root = [ this.cat.meow, this.doc.woofs[0] ]", "root = [ metadata(\"user.id\").string() ]" ] }, "azure": { "type": "object", "description": "Optional Fields that can be set to use Azure based authentication for Azure Postgres SQL", "properties": { "entra_enabled": { "type": "boolean", "description": "An optional field used to generate an entra token to connect to 'Azure Database for PostgreSQL flexible server', This will create a new connection string with the host, user and database from the DSN field - you may need to URL encode the dsn! The [Default Azure Credential Chain](https://learn.microsoft.com/en-gb/azure/developer/go/sdk/authentication/authentication-overview#defaultazurecredential) is used from the Azure SDK. Default: false" }, "token_request_options": { "type": "object", "properties": { "claims": { "type": "string", "description": "Set additional claims for the token. Default: " }, "enable_cae": { "type": "boolean", "description": "Indicates whether to enable Continuous Access Evaluation (CAE) for the requested token Default: false" }, "scopes": { "type": "array", "description": "Scopes contains the list of permission scopes required for the token. Default: [https://ossrdbms-aad.database.windows.net/.default]", "items": { "type": "string" } }, "tenant_id": { "type": "string", "description": "tenant_id identifies the tenant from which to request the token. azure credentials authenticate in their configured default tenants when this field isn't set. Default: " } } } } }, "columns": { "type": "array", "description": "A list of columns to insert.", "items": { "type": "string" }, "examples": [ [ "foo", "bar", "baz" ] ] }, "conn_max_idle": { "type": "integer", "description": "An optional maximum number of connections in the idle connection pool. If conn_max_open is greater than 0 but less than the new conn_max_idle, then the new conn_max_idle will be reduced to match the conn_max_open limit. If `value \u003c= 0`, no idle connections are retained. The default max idle connections is currently 2. This may change in a future release. Default: 2" }, "conn_max_idle_time": { "type": "string", "description": "An optional maximum amount of time a connection may be idle. Expired connections may be closed lazily before reuse. If `value \u003c= 0`, connections are not closed due to a connections idle time." }, "conn_max_life_time": { "type": "string", "description": "An optional maximum amount of time a connection may be reused. Expired connections may be closed lazily before reuse. If `value \u003c= 0`, connections are not closed due to a connections age." }, "conn_max_open": { "type": "integer", "description": "An optional maximum number of open connections to the database. If conn_max_idle is greater than 0 and the new conn_max_open is less than conn_max_idle, then conn_max_idle will be reduced to match the new conn_max_open limit. If `value \u003c= 0`, then there is no limit on the number of open connections. The default is 0 (unlimited)." }, "credentials": { "type": "object", "description": "Optional manual configuration of AWS credentials to use. More information can be found [in this document](/docs/guides/cloud/aws).", "properties": { "expiry_window": { "type": "string", "description": "Allow the credentials to trigger refreshing prior to the credentials actually expiring. This is beneficial so race conditions with expiring credentials do not cause requests to fail. For example '10s' would refresh credentials ten seconds before expiration. Setting to a duration of `0` disables the expiry window. Default: " }, "from_ec2_role": { "type": "boolean", "description": "Use the credentials of a host EC2 machine configured to assume [an IAM role associated with the instance](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html). Default: false" }, "id": { "type": "string", "description": "The ID of credentials to use. Default: " }, "profile": { "type": "string", "description": "A profile from `~/.aws/credentials` to use. Default: " }, "role": { "type": "string", "description": "A role ARN to assume. Default: " }, "role_external_id": { "type": "string", "description": "An external ID to provide when assuming a role. Default: " }, "secret": { "type": "string", "description": "The secret for the credentials being used. Default: " }, "token": { "type": "string", "description": "The token for the credentials being used, required when using short term credentials. Default: " } } }, "driver": { "type": "string", "description": "A database [driver](#drivers) to use." }, "dsn": { "type": "string", "description": "A Data Source Name to identify the target database.\n\n#### Drivers\n\nThe following is a list of supported drivers, their placeholder style, and their respective DSN formats:\n\n| Driver | Data Source Name Format |\n|---|---|\n| `clickhouse` | [`clickhouse://[username[:password]@][netloc][:port]/dbname[?param1=value1\u0026...\u0026paramN=valueN]`](https://github.com/ClickHouse/clickhouse-go#dsn) |\n| `mysql` | `[username[:password]@][protocol[(address)]]/dbname[?param1=value1\u0026...\u0026paramN=valueN]` |\n| `postgres` | `postgres://[user[:password]@][netloc][:port][/dbname][?param1=value1\u0026...]` |\n| `mssql` | `sqlserver://[user[:password]@][netloc][:port][?database=dbname\u0026param1=value1\u0026...]` |\n| `sqlite` | `file:/path/to/filename.db[?param\u0026=value1\u0026...]` |\n| `oracle` | `oracle://[username[:password]@][netloc][:port]/service_name?server=server2\u0026server=server3` |\n| `snowflake` | `username[:password]@account_identifier/dbname/schemaname[?param1=value\u0026...\u0026paramN=valueN]` |\n| `spanner` | `projects/[project]/instances/[instance]/databases/dbname` |\n| `trino` | [`http[s]://user[:pass]@host[:port][?parameters]`](https://github.com/trinodb/trino-go-client#dsn-data-source-name) |\n| `gocosmos` | [`AccountEndpoint=\u003ccosmosdb-endpoint\u003e;AccountKey=\u003ccosmosdb-account-key\u003e[;TimeoutMs=\u003ctimeout-in-ms\u003e][;Version=\u003ccosmosdb-api-version\u003e][;DefaultDb/Db=\u003cdb-name\u003e][;AutoId=\u003ctrue/false\u003e][;InsecureSkipVerify=\u003ctrue/false\u003e]`](https://pkg.go.dev/github.com/microsoft/gocosmos#readme-example-usage) |\n\nPlease note that the `postgres` driver enforces SSL by default, you can override this with the parameter `sslmode=disable` if required.\n\nThe `snowflake` driver supports multiple DSN formats. Please consult [the docs](https://pkg.go.dev/github.com/snowflakedb/gosnowflake#hdr-Connection_String) for more details. For [key pair authentication](https://docs.snowflake.com/en/user-guide/key-pair-auth.html#configuring-key-pair-authentication), the DSN has the following format: `\u003csnowflake_user\u003e@\u003csnowflake_account\u003e/\u003cdb_name\u003e/\u003cschema_name\u003e?warehouse=\u003cwarehouse\u003e\u0026role=\u003crole\u003e\u0026authenticator=snowflake_jwt\u0026privateKey=\u003cbase64_url_encoded_private_key\u003e`, where the value for the `privateKey` parameter can be constructed from an unencrypted RSA private key file `rsa_key.p8` using `openssl enc -d -base64 -in rsa_key.p8 | basenc --base64url -w0` (you can use `gbasenc` insted of `basenc` on OSX if you install `coreutils` via Homebrew). If you have a password-encrypted private key, you can decrypt it using `openssl pkcs8 -in rsa_key_encrypted.p8 -out rsa_key.p8`. Also, make sure fields such as the username are URL-encoded.\n\nThe [`gocosmos`](https://pkg.go.dev/github.com/microsoft/gocosmos) driver is still experimental, but it has support for [hierarchical partition keys](https://learn.microsoft.com/en-us/azure/cosmos-db/hierarchical-partition-keys) as well as [cross-partition queries](https://learn.microsoft.com/en-us/azure/cosmos-db/nosql/how-to-query-container#cross-partition-query). Please refer to the [SQL notes](https://github.com/microsoft/gocosmos/blob/main/SQL.md) for details.", "examples": [ "clickhouse://username:password@host1:9000,host2:9000/database?dial_timeout=200ms\u0026max_execution_time=60", "foouser:foopassword@tcp(localhost:3306)/foodb", "postgres://foouser:foopass@localhost:5432/foodb?sslmode=disable", "oracle://foouser:foopass@localhost:1521/service_name" ] }, "endpoint": { "type": "string", "description": "Allows you to specify a custom endpoint for the AWS API. Default: " }, "iam_enabled": { "type": "boolean", "description": "An optional field used to generate an IAM authentication token to connect to an Amazon Relational Database (RDS) DB instance. This will overwrite the Password in the DSN with the generated token only if the drivers are `mysql` or `postgres`. Default: false" }, "init_files": { "type": "array", "description": "\nAn optional list of file paths containing SQL statements to execute immediately upon the first connection to the target database. This is a useful way to initialise tables before processing data. Glob patterns are supported, including super globs (double star).\n\nCare should be taken to ensure that the statements are idempotent, and therefore would not cause issues when run multiple times after service restarts. If both `init_statement` and `init_files` are specified the `init_statement` is executed _after_ the `init_files`.\n\nIf a statement fails for any reason a warning log will be emitted but the operation of this component will not be stopped.\n", "items": { "type": "string" }, "examples": [ [ "./init/*.sql" ], [ "./foo.sql", "./bar.sql" ] ] }, "init_statement": { "type": "string", "description": "\nAn optional SQL statement to execute immediately upon the first connection to the target database. This is a useful way to initialise tables before processing data. Care should be taken to ensure that the statement is idempotent, and therefore would not cause issues when run multiple times after service restarts.\n\nIf both `init_statement` and `init_files` are specified the `init_statement` is executed _after_ the `init_files`.\n\nIf the statement fails for any reason a warning log will be emitted but the operation of this component will not be stopped.\n", "examples": [ "\nCREATE TABLE IF NOT EXISTS some_table (\n foo varchar(50) not null,\n bar integer,\n baz varchar(50),\n primary key (foo)\n) WITHOUT ROWID;\n" ] }, "init_verify_conn": { "type": "boolean", "description": "Whether to verify the database connection on startup by performing a simple ping, by default this is disabled. Default: false" }, "prefix": { "type": "string", "description": "An optional prefix to prepend to the insert query (before INSERT)." }, "region": { "type": "string", "description": "The AWS region to target. Default: " }, "secret_name": { "type": "string", "description": "An optional field that can be used to get the Username + Password from AWS Secrets Manager. This will overwrite the Username + Password in the DSN with the values from the Secret only if the driver is set to `postgres`." }, "suffix": { "type": "string", "description": "An optional suffix to append to the insert query.", "examples": [ "ON CONFLICT (name) DO NOTHING" ] }, "table": { "type": "string", "description": "The table to insert to.", "examples": [ "foo" ] } } }, "sql_raw": { "type": "object", "properties": { "args_mapping": { "type": "string", "description": "An optional [Bloblang mapping](/docs/guides/bloblang/about) which should evaluate to an array of values matching in size to the number of placeholder arguments in the field `query`.", "examples": [ "root = [ this.cat.meow, this.doc.woofs[0] ]", "root = [ metadata(\"user.id\").string() ]" ] }, "azure": { "type": "object", "description": "Optional Fields that can be set to use Azure based authentication for Azure Postgres SQL", "properties": { "entra_enabled": { "type": "boolean", "description": "An optional field used to generate an entra token to connect to 'Azure Database for PostgreSQL flexible server', This will create a new connection string with the host, user and database from the DSN field - you may need to URL encode the dsn! The [Default Azure Credential Chain](https://learn.microsoft.com/en-gb/azure/developer/go/sdk/authentication/authentication-overview#defaultazurecredential) is used from the Azure SDK. Default: false" }, "token_request_options": { "type": "object", "properties": { "claims": { "type": "string", "description": "Set additional claims for the token. Default: " }, "enable_cae": { "type": "boolean", "description": "Indicates whether to enable Continuous Access Evaluation (CAE) for the requested token Default: false" }, "scopes": { "type": "array", "description": "Scopes contains the list of permission scopes required for the token. Default: [https://ossrdbms-aad.database.windows.net/.default]", "items": { "type": "string" } }, "tenant_id": { "type": "string", "description": "tenant_id identifies the tenant from which to request the token. azure credentials authenticate in their configured default tenants when this field isn't set. Default: " } } } } }, "conn_max_idle": { "type": "integer", "description": "An optional maximum number of connections in the idle connection pool. If conn_max_open is greater than 0 but less than the new conn_max_idle, then the new conn_max_idle will be reduced to match the conn_max_open limit. If `value \u003c= 0`, no idle connections are retained. The default max idle connections is currently 2. This may change in a future release. Default: 2" }, "conn_max_idle_time": { "type": "string", "description": "An optional maximum amount of time a connection may be idle. Expired connections may be closed lazily before reuse. If `value \u003c= 0`, connections are not closed due to a connections idle time." }, "conn_max_life_time": { "type": "string", "description": "An optional maximum amount of time a connection may be reused. Expired connections may be closed lazily before reuse. If `value \u003c= 0`, connections are not closed due to a connections age." }, "conn_max_open": { "type": "integer", "description": "An optional maximum number of open connections to the database. If conn_max_idle is greater than 0 and the new conn_max_open is less than conn_max_idle, then conn_max_idle will be reduced to match the new conn_max_open limit. If `value \u003c= 0`, then there is no limit on the number of open connections. The default is 0 (unlimited)." }, "credentials": { "type": "object", "description": "Optional manual configuration of AWS credentials to use. More information can be found [in this document](/docs/guides/cloud/aws).", "properties": { "expiry_window": { "type": "string", "description": "Allow the credentials to trigger refreshing prior to the credentials actually expiring. This is beneficial so race conditions with expiring credentials do not cause requests to fail. For example '10s' would refresh credentials ten seconds before expiration. Setting to a duration of `0` disables the expiry window. Default: " }, "from_ec2_role": { "type": "boolean", "description": "Use the credentials of a host EC2 machine configured to assume [an IAM role associated with the instance](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html). Default: false" }, "id": { "type": "string", "description": "The ID of credentials to use. Default: " }, "profile": { "type": "string", "description": "A profile from `~/.aws/credentials` to use. Default: " }, "role": { "type": "string", "description": "A role ARN to assume. Default: " }, "role_external_id": { "type": "string", "description": "An external ID to provide when assuming a role. Default: " }, "secret": { "type": "string", "description": "The secret for the credentials being used. Default: " }, "token": { "type": "string", "description": "The token for the credentials being used, required when using short term credentials. Default: " } } }, "driver": { "type": "string", "description": "A database [driver](#drivers) to use." }, "dsn": { "type": "string", "description": "A Data Source Name to identify the target database.\n\n#### Drivers\n\nThe following is a list of supported drivers, their placeholder style, and their respective DSN formats:\n\n| Driver | Data Source Name Format |\n|---|---|\n| `clickhouse` | [`clickhouse://[username[:password]@][netloc][:port]/dbname[?param1=value1\u0026...\u0026paramN=valueN]`](https://github.com/ClickHouse/clickhouse-go#dsn) |\n| `mysql` | `[username[:password]@][protocol[(address)]]/dbname[?param1=value1\u0026...\u0026paramN=valueN]` |\n| `postgres` | `postgres://[user[:password]@][netloc][:port][/dbname][?param1=value1\u0026...]` |\n| `mssql` | `sqlserver://[user[:password]@][netloc][:port][?database=dbname\u0026param1=value1\u0026...]` |\n| `sqlite` | `file:/path/to/filename.db[?param\u0026=value1\u0026...]` |\n| `oracle` | `oracle://[username[:password]@][netloc][:port]/service_name?server=server2\u0026server=server3` |\n| `snowflake` | `username[:password]@account_identifier/dbname/schemaname[?param1=value\u0026...\u0026paramN=valueN]` |\n| `spanner` | `projects/[project]/instances/[instance]/databases/dbname` |\n| `trino` | [`http[s]://user[:pass]@host[:port][?parameters]`](https://github.com/trinodb/trino-go-client#dsn-data-source-name) |\n| `gocosmos` | [`AccountEndpoint=\u003ccosmosdb-endpoint\u003e;AccountKey=\u003ccosmosdb-account-key\u003e[;TimeoutMs=\u003ctimeout-in-ms\u003e][;Version=\u003ccosmosdb-api-version\u003e][;DefaultDb/Db=\u003cdb-name\u003e][;AutoId=\u003ctrue/false\u003e][;InsecureSkipVerify=\u003ctrue/false\u003e]`](https://pkg.go.dev/github.com/microsoft/gocosmos#readme-example-usage) |\n\nPlease note that the `postgres` driver enforces SSL by default, you can override this with the parameter `sslmode=disable` if required.\n\nThe `snowflake` driver supports multiple DSN formats. Please consult [the docs](https://pkg.go.dev/github.com/snowflakedb/gosnowflake#hdr-Connection_String) for more details. For [key pair authentication](https://docs.snowflake.com/en/user-guide/key-pair-auth.html#configuring-key-pair-authentication), the DSN has the following format: `\u003csnowflake_user\u003e@\u003csnowflake_account\u003e/\u003cdb_name\u003e/\u003cschema_name\u003e?warehouse=\u003cwarehouse\u003e\u0026role=\u003crole\u003e\u0026authenticator=snowflake_jwt\u0026privateKey=\u003cbase64_url_encoded_private_key\u003e`, where the value for the `privateKey` parameter can be constructed from an unencrypted RSA private key file `rsa_key.p8` using `openssl enc -d -base64 -in rsa_key.p8 | basenc --base64url -w0` (you can use `gbasenc` insted of `basenc` on OSX if you install `coreutils` via Homebrew). If you have a password-encrypted private key, you can decrypt it using `openssl pkcs8 -in rsa_key_encrypted.p8 -out rsa_key.p8`. Also, make sure fields such as the username are URL-encoded.\n\nThe [`gocosmos`](https://pkg.go.dev/github.com/microsoft/gocosmos) driver is still experimental, but it has support for [hierarchical partition keys](https://learn.microsoft.com/en-us/azure/cosmos-db/hierarchical-partition-keys) as well as [cross-partition queries](https://learn.microsoft.com/en-us/azure/cosmos-db/nosql/how-to-query-container#cross-partition-query). Please refer to the [SQL notes](https://github.com/microsoft/gocosmos/blob/main/SQL.md) for details.", "examples": [ "clickhouse://username:password@host1:9000,host2:9000/database?dial_timeout=200ms\u0026max_execution_time=60", "foouser:foopassword@tcp(localhost:3306)/foodb", "postgres://foouser:foopass@localhost:5432/foodb?sslmode=disable", "oracle://foouser:foopass@localhost:1521/service_name" ] }, "endpoint": { "type": "string", "description": "Allows you to specify a custom endpoint for the AWS API. Default: " }, "exec_only": { "type": "boolean", "description": "Whether the query result should be discarded. When set to `true` the message contents will remain unchanged, which is useful in cases where you are executing inserts, updates, etc. Default: false" }, "iam_enabled": { "type": "boolean", "description": "An optional field used to generate an IAM authentication token to connect to an Amazon Relational Database (RDS) DB instance. This will overwrite the Password in the DSN with the generated token only if the drivers are `mysql` or `postgres`. Default: false" }, "init_files": { "type": "array", "description": "\nAn optional list of file paths containing SQL statements to execute immediately upon the first connection to the target database. This is a useful way to initialise tables before processing data. Glob patterns are supported, including super globs (double star).\n\nCare should be taken to ensure that the statements are idempotent, and therefore would not cause issues when run multiple times after service restarts. If both `init_statement` and `init_files` are specified the `init_statement` is executed _after_ the `init_files`.\n\nIf a statement fails for any reason a warning log will be emitted but the operation of this component will not be stopped.\n", "items": { "type": "string" }, "examples": [ [ "./init/*.sql" ], [ "./foo.sql", "./bar.sql" ] ] }, "init_statement": { "type": "string", "description": "\nAn optional SQL statement to execute immediately upon the first connection to the target database. This is a useful way to initialise tables before processing data. Care should be taken to ensure that the statement is idempotent, and therefore would not cause issues when run multiple times after service restarts.\n\nIf both `init_statement` and `init_files` are specified the `init_statement` is executed _after_ the `init_files`.\n\nIf the statement fails for any reason a warning log will be emitted but the operation of this component will not be stopped.\n", "examples": [ "\nCREATE TABLE IF NOT EXISTS some_table (\n foo varchar(50) not null,\n bar integer,\n baz varchar(50),\n primary key (foo)\n) WITHOUT ROWID;\n" ] }, "init_verify_conn": { "type": "boolean", "description": "Whether to verify the database connection on startup by performing a simple ping, by default this is disabled. Default: false" }, "query": { "type": "string", "description": "The query to execute. The style of placeholder to use depends on the driver, some drivers require question marks (`?`) whereas others expect incrementing dollar signs (`$1`, `$2`, and so on) or colons (`:1`, `:2` and so on). The style to use is outlined in this table:\n\n| Driver | Placeholder Style |\n|---|---|\n| `clickhouse` | Dollar sign |\n| `mysql` | Question mark |\n| `postgres` | Dollar sign |\n| `mssql` | Question mark |\n| `sqlite` | Question mark |\n| `oracle` | Colon |\n| `snowflake` | Question mark |\n| `spanner` | Question mark |\n| `trino` | Question mark |\n| `gocosmos` | Colon |\n", "examples": [ "INSERT INTO footable (foo, bar, baz) VALUES (?, ?, ?);", "SELECT * FROM footable WHERE user_id = $1;" ] }, "region": { "type": "string", "description": "The AWS region to target. Default: " }, "secret_name": { "type": "string", "description": "An optional field that can be used to get the Username + Password from AWS Secrets Manager. This will overwrite the Username + Password in the DSN with the values from the Secret only if the driver is set to `postgres`." }, "unsafe_dynamic_query": { "type": "boolean", "description": "Whether to enable [interpolation functions](/docs/configuration/interpolation/#bloblang-queries) in the query. Great care should be made to ensure your queries are defended against injection attacks. Default: false" } } }, "sql_select": { "type": "object", "properties": { "args_mapping": { "type": "string", "description": "An optional [Bloblang mapping](/docs/guides/bloblang/about) which should evaluate to an array of values matching in size to the number of placeholder arguments in the field `where`.", "examples": [ "root = [ this.cat.meow, this.doc.woofs[0] ]", "root = [ metadata(\"user.id\").string() ]" ] }, "azure": { "type": "object", "description": "Optional Fields that can be set to use Azure based authentication for Azure Postgres SQL", "properties": { "entra_enabled": { "type": "boolean", "description": "An optional field used to generate an entra token to connect to 'Azure Database for PostgreSQL flexible server', This will create a new connection string with the host, user and database from the DSN field - you may need to URL encode the dsn! The [Default Azure Credential Chain](https://learn.microsoft.com/en-gb/azure/developer/go/sdk/authentication/authentication-overview#defaultazurecredential) is used from the Azure SDK. Default: false" }, "token_request_options": { "type": "object", "properties": { "claims": { "type": "string", "description": "Set additional claims for the token. Default: " }, "enable_cae": { "type": "boolean", "description": "Indicates whether to enable Continuous Access Evaluation (CAE) for the requested token Default: false" }, "scopes": { "type": "array", "description": "Scopes contains the list of permission scopes required for the token. Default: [https://ossrdbms-aad.database.windows.net/.default]", "items": { "type": "string" } }, "tenant_id": { "type": "string", "description": "tenant_id identifies the tenant from which to request the token. azure credentials authenticate in their configured default tenants when this field isn't set. Default: " } } } } }, "columns": { "type": "array", "description": "A list of columns to query.", "items": { "type": "string" }, "examples": [ [ "*" ], [ "foo", "bar", "baz" ] ] }, "conn_max_idle": { "type": "integer", "description": "An optional maximum number of connections in the idle connection pool. If conn_max_open is greater than 0 but less than the new conn_max_idle, then the new conn_max_idle will be reduced to match the conn_max_open limit. If `value \u003c= 0`, no idle connections are retained. The default max idle connections is currently 2. This may change in a future release. Default: 2" }, "conn_max_idle_time": { "type": "string", "description": "An optional maximum amount of time a connection may be idle. Expired connections may be closed lazily before reuse. If `value \u003c= 0`, connections are not closed due to a connections idle time." }, "conn_max_life_time": { "type": "string", "description": "An optional maximum amount of time a connection may be reused. Expired connections may be closed lazily before reuse. If `value \u003c= 0`, connections are not closed due to a connections age." }, "conn_max_open": { "type": "integer", "description": "An optional maximum number of open connections to the database. If conn_max_idle is greater than 0 and the new conn_max_open is less than conn_max_idle, then conn_max_idle will be reduced to match the new conn_max_open limit. If `value \u003c= 0`, then there is no limit on the number of open connections. The default is 0 (unlimited)." }, "credentials": { "type": "object", "description": "Optional manual configuration of AWS credentials to use. More information can be found [in this document](/docs/guides/cloud/aws).", "properties": { "expiry_window": { "type": "string", "description": "Allow the credentials to trigger refreshing prior to the credentials actually expiring. This is beneficial so race conditions with expiring credentials do not cause requests to fail. For example '10s' would refresh credentials ten seconds before expiration. Setting to a duration of `0` disables the expiry window. Default: " }, "from_ec2_role": { "type": "boolean", "description": "Use the credentials of a host EC2 machine configured to assume [an IAM role associated with the instance](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html). Default: false" }, "id": { "type": "string", "description": "The ID of credentials to use. Default: " }, "profile": { "type": "string", "description": "A profile from `~/.aws/credentials` to use. Default: " }, "role": { "type": "string", "description": "A role ARN to assume. Default: " }, "role_external_id": { "type": "string", "description": "An external ID to provide when assuming a role. Default: " }, "secret": { "type": "string", "description": "The secret for the credentials being used. Default: " }, "token": { "type": "string", "description": "The token for the credentials being used, required when using short term credentials. Default: " } } }, "driver": { "type": "string", "description": "A database [driver](#drivers) to use." }, "dsn": { "type": "string", "description": "A Data Source Name to identify the target database.\n\n#### Drivers\n\nThe following is a list of supported drivers, their placeholder style, and their respective DSN formats:\n\n| Driver | Data Source Name Format |\n|---|---|\n| `clickhouse` | [`clickhouse://[username[:password]@][netloc][:port]/dbname[?param1=value1\u0026...\u0026paramN=valueN]`](https://github.com/ClickHouse/clickhouse-go#dsn) |\n| `mysql` | `[username[:password]@][protocol[(address)]]/dbname[?param1=value1\u0026...\u0026paramN=valueN]` |\n| `postgres` | `postgres://[user[:password]@][netloc][:port][/dbname][?param1=value1\u0026...]` |\n| `mssql` | `sqlserver://[user[:password]@][netloc][:port][?database=dbname\u0026param1=value1\u0026...]` |\n| `sqlite` | `file:/path/to/filename.db[?param\u0026=value1\u0026...]` |\n| `oracle` | `oracle://[username[:password]@][netloc][:port]/service_name?server=server2\u0026server=server3` |\n| `snowflake` | `username[:password]@account_identifier/dbname/schemaname[?param1=value\u0026...\u0026paramN=valueN]` |\n| `spanner` | `projects/[project]/instances/[instance]/databases/dbname` |\n| `trino` | [`http[s]://user[:pass]@host[:port][?parameters]`](https://github.com/trinodb/trino-go-client#dsn-data-source-name) |\n| `gocosmos` | [`AccountEndpoint=\u003ccosmosdb-endpoint\u003e;AccountKey=\u003ccosmosdb-account-key\u003e[;TimeoutMs=\u003ctimeout-in-ms\u003e][;Version=\u003ccosmosdb-api-version\u003e][;DefaultDb/Db=\u003cdb-name\u003e][;AutoId=\u003ctrue/false\u003e][;InsecureSkipVerify=\u003ctrue/false\u003e]`](https://pkg.go.dev/github.com/microsoft/gocosmos#readme-example-usage) |\n\nPlease note that the `postgres` driver enforces SSL by default, you can override this with the parameter `sslmode=disable` if required.\n\nThe `snowflake` driver supports multiple DSN formats. Please consult [the docs](https://pkg.go.dev/github.com/snowflakedb/gosnowflake#hdr-Connection_String) for more details. For [key pair authentication](https://docs.snowflake.com/en/user-guide/key-pair-auth.html#configuring-key-pair-authentication), the DSN has the following format: `\u003csnowflake_user\u003e@\u003csnowflake_account\u003e/\u003cdb_name\u003e/\u003cschema_name\u003e?warehouse=\u003cwarehouse\u003e\u0026role=\u003crole\u003e\u0026authenticator=snowflake_jwt\u0026privateKey=\u003cbase64_url_encoded_private_key\u003e`, where the value for the `privateKey` parameter can be constructed from an unencrypted RSA private key file `rsa_key.p8` using `openssl enc -d -base64 -in rsa_key.p8 | basenc --base64url -w0` (you can use `gbasenc` insted of `basenc` on OSX if you install `coreutils` via Homebrew). If you have a password-encrypted private key, you can decrypt it using `openssl pkcs8 -in rsa_key_encrypted.p8 -out rsa_key.p8`. Also, make sure fields such as the username are URL-encoded.\n\nThe [`gocosmos`](https://pkg.go.dev/github.com/microsoft/gocosmos) driver is still experimental, but it has support for [hierarchical partition keys](https://learn.microsoft.com/en-us/azure/cosmos-db/hierarchical-partition-keys) as well as [cross-partition queries](https://learn.microsoft.com/en-us/azure/cosmos-db/nosql/how-to-query-container#cross-partition-query). Please refer to the [SQL notes](https://github.com/microsoft/gocosmos/blob/main/SQL.md) for details.", "examples": [ "clickhouse://username:password@host1:9000,host2:9000/database?dial_timeout=200ms\u0026max_execution_time=60", "foouser:foopassword@tcp(localhost:3306)/foodb", "postgres://foouser:foopass@localhost:5432/foodb?sslmode=disable", "oracle://foouser:foopass@localhost:1521/service_name" ] }, "endpoint": { "type": "string", "description": "Allows you to specify a custom endpoint for the AWS API. Default: " }, "iam_enabled": { "type": "boolean", "description": "An optional field used to generate an IAM authentication token to connect to an Amazon Relational Database (RDS) DB instance. This will overwrite the Password in the DSN with the generated token only if the drivers are `mysql` or `postgres`. Default: false" }, "init_files": { "type": "array", "description": "\nAn optional list of file paths containing SQL statements to execute immediately upon the first connection to the target database. This is a useful way to initialise tables before processing data. Glob patterns are supported, including super globs (double star).\n\nCare should be taken to ensure that the statements are idempotent, and therefore would not cause issues when run multiple times after service restarts. If both `init_statement` and `init_files` are specified the `init_statement` is executed _after_ the `init_files`.\n\nIf a statement fails for any reason a warning log will be emitted but the operation of this component will not be stopped.\n", "items": { "type": "string" }, "examples": [ [ "./init/*.sql" ], [ "./foo.sql", "./bar.sql" ] ] }, "init_statement": { "type": "string", "description": "\nAn optional SQL statement to execute immediately upon the first connection to the target database. This is a useful way to initialise tables before processing data. Care should be taken to ensure that the statement is idempotent, and therefore would not cause issues when run multiple times after service restarts.\n\nIf both `init_statement` and `init_files` are specified the `init_statement` is executed _after_ the `init_files`.\n\nIf the statement fails for any reason a warning log will be emitted but the operation of this component will not be stopped.\n", "examples": [ "\nCREATE TABLE IF NOT EXISTS some_table (\n foo varchar(50) not null,\n bar integer,\n baz varchar(50),\n primary key (foo)\n) WITHOUT ROWID;\n" ] }, "init_verify_conn": { "type": "boolean", "description": "Whether to verify the database connection on startup by performing a simple ping, by default this is disabled. Default: false" }, "prefix": { "type": "string", "description": "An optional prefix to prepend to the query (before SELECT)." }, "region": { "type": "string", "description": "The AWS region to target. Default: " }, "secret_name": { "type": "string", "description": "An optional field that can be used to get the Username + Password from AWS Secrets Manager. This will overwrite the Username + Password in the DSN with the values from the Secret only if the driver is set to `postgres`." }, "suffix": { "type": "string", "description": "An optional suffix to append to the select query." }, "table": { "type": "string", "description": "The table to query.", "examples": [ "foo" ] }, "where": { "type": "string", "description": "An optional where clause to add. Placeholder arguments are populated with the `args_mapping` field. Placeholders should always be question marks, and will automatically be converted to dollar syntax when the postgres or clickhouse drivers are used.", "examples": [ "meow = ? and woof = ?", "user_id = ?" ] } } }, "subprocess": { "type": "object", "properties": { "args": { "type": "array", "description": "A list of arguments to provide the command. Default: []", "items": { "type": "string" } }, "codec_recv": { "type": "string", "description": "Determines how messages read from the subprocess are decoded, which allows them to be logically separated. Default: lines" }, "codec_send": { "type": "string", "description": "Determines how messages written to the subprocess are encoded, which allows them to be logically separated. Default: lines" }, "max_buffer": { "type": "integer", "description": "The maximum expected response size. Default: 65536" }, "name": { "type": "string", "description": "The command to execute as a subprocess.", "examples": [ "cat", "sed", "awk" ] } } }, "switch": { "type": "array", "items": { "type": "object" } }, "sync_response": { "type": "object", "description": " Default: map[]" }, "try": { "type": "array", "description": " Default: []", "items": { "type": "object" } }, "unarchive": { "type": "object", "properties": { "format": { "type": "string", "description": "The unarchiving format to apply." } } }, "wasm": { "type": "object", "properties": { "function": { "type": "string", "description": "The name of the function exported by the target WASM module to run for each message. Default: process" }, "module_path": { "type": "string", "description": "The path of the target WASM module to execute." } } }, "while": { "type": "object", "properties": { "at_least_once": { "type": "boolean", "description": "Whether to always run the child processors at least one time. Default: false" }, "check": { "type": "string", "description": "A [Bloblang query](/docs/guides/bloblang/about/) that should return a boolean value indicating whether the while loop should execute again. Default: ", "examples": [ "errored()", "this.urls.unprocessed.length() \u003e 0" ] }, "max_loops": { "type": "integer", "description": "An optional maximum number of loops to execute. Helps protect against accidentally creating infinite loops. Default: 0" }, "processors": { "type": "array", "description": "A list of child processors to execute on each loop.", "items": { "type": "object" } } } }, "workflow": { "type": "object", "properties": { "branch_resources": { "type": "array", "description": "An optional list of [`branch` processor](/docs/components/processors/branch) names that are configured as [resources](#resources). These resources will be included in the workflow with any branches configured inline within the [`branches`](#branches) field. The order and parallelism in which branches are executed is automatically resolved based on the mappings of each branch. When using resources with an explicit order it is not necessary to list resources in this field. Default: []", "items": { "type": "string" } }, "branches": { "type": "object", "description": "An object of named [`branch` processors](/docs/components/processors/branch) that make up the workflow. The order and parallelism in which branches are executed can either be made explicit with the field `order`, or if omitted an attempt is made to automatically resolve an ordering based on the mappings of each branch. Default: map[]", "properties": { "processors": { "type": "array", "description": "A list of processors to apply to mapped requests. When processing message batches the resulting batch must match the size and ordering of the input batch, therefore filtering, grouping should not be performed within these processors.", "items": { "type": "object" } }, "request_map": { "type": "string", "description": "A [Bloblang mapping](/docs/guides/bloblang/about) that describes how to create a request payload suitable for the child processors of this branch. If left empty then the branch will begin with an exact copy of the origin message (including metadata). Default: ", "examples": [ "root = {\n\t\"id\": this.doc.id,\n\t\"content\": this.doc.body.text\n}", "root = if this.type == \"foo\" {\n\tthis.foo.request\n} else {\n\tdeleted()\n}" ] }, "result_map": { "type": "string", "description": "A [Bloblang mapping](/docs/guides/bloblang/about) that describes how the resulting messages from branched processing should be mapped back into the original payload. If left empty the origin message will remain unchanged (including metadata). Default: ", "examples": [ "meta foo_code = metadata(\"code\")\nroot.foo_result = this", "meta = metadata()\nroot.bar.body = this.body\nroot.bar.id = this.user.id", "root.raw_result = content().string()", "root.enrichments.foo = if metadata(\"request_failed\") != null {\n throw(metadata(\"request_failed\"))\n} else {\n this\n}", "# Retain only the updated metadata fields which were present in the origin message\nmeta = metadata().filter(v -\u003e @.get(v.key) != null)" ] } } }, "meta_path": { "type": "string", "description": "A [dot path](/docs/configuration/field_paths) indicating where to store and reference [structured metadata](#structured-metadata) about the workflow execution. Default: meta.workflow" }, "order": { "type": "array", "description": "An explicit declaration of branch ordered tiers, which describes the order in which parallel tiers of branches should be executed. Branches should be identified by the name as they are configured in the field `branches`. It's also possible to specify branch processors configured [as a resource](#resources). Default: []", "items": { "items": { "type": "array" } }, "examples": [ [ [ "foo", "bar" ], [ "baz" ] ], [ [ "foo" ], [ "bar" ], [ "baz" ] ] ] } } }, "xml": { "type": "object", "properties": { "cast": { "type": "boolean", "description": "Whether to try to cast values that are numbers and booleans to the right type. Default: all values are strings. Default: false" }, "operator": { "type": "string", "description": "An XML [operation](#operators) to apply to messages. Default: " } } } } } }, "rate_limit_resources": { "type": "array", "description": "A list of rate limit resources, each must have a unique label. Default: []", "items": { "type": "object", "properties": { "local": { "type": "object", "properties": { "byte_size": { "type": "integer", "description": "The maximum number of bytes to allow for a given period of time. If `0` disables byte_size based rate-limiting. Default: 0" }, "count": { "type": "integer", "description": "The maximum number of requests to allow for a given period of time. If `0` disables count based rate-limiting. Default: 1000" }, "interval": { "type": "string", "description": "The time window to limit requests by. Default: 1s" } } }, "redis": { "type": "object", "properties": { "count": { "type": "integer", "description": "The maximum number of messages to allow for a given period of time. Default: 1000" }, "interval": { "type": "string", "description": "The time window to limit requests by. Default: 1s" }, "key": { "type": "string", "description": "The key to use for the rate limit." }, "kind": { "type": "string", "description": "Specifies a simple, cluster-aware, or failover-aware redis client. Default: simple" }, "master": { "type": "string", "description": "Name of the redis master when `kind` is `failover` Default: ", "examples": [ "mymaster" ] }, "tls": { "type": "object", "description": "Custom TLS settings can be used to override system defaults.\n\n**Troubleshooting**\n\nSome cloud hosted instances of Redis (such as Azure Cache) might need some hand holding in order to establish stable connections. Unfortunately, it is often the case that TLS issues will manifest as generic error messages such as \"i/o timeout\". If you're using TLS and are seeing connectivity problems consider setting `enable_renegotiation` to `true`, and ensuring that the server supports at least TLS version 1.2.", "properties": { "client_certs": { "type": "array", "description": "A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. Default: []", "items": { "type": "object" }, "examples": [ [ { "cert": "foo", "key": "bar" } ], [ { "cert_file": "./example.pem", "key_file": "./example.key" } ] ] }, "enable_renegotiation": { "type": "boolean", "description": "Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. Default: false" }, "enabled": { "type": "boolean", "description": "Whether custom TLS settings are enabled. Default: false" }, "root_cas": { "type": "string", "description": "An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----" ] }, "root_cas_file": { "type": "string", "description": "An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. Default: ", "examples": [ "./root_cas.pem" ] }, "skip_cert_verify": { "type": "boolean", "description": "Whether to skip server side certificate verification. Default: false" } } }, "url": { "type": "string", "description": "The URL of the target Redis server. Database is optional and is supplied as the URL path.", "examples": [ "redis://:6397", "redis://localhost:6379", "redis://foousername:foopassword@redisplace:6379", "redis://:foopassword@redisplace:6379", "redis://localhost:6379/1", "redis://localhost:6379/1,redis://localhost:6380/1" ] } } } } } }, "shutdown_delay": { "type": "string", "description": "A period of time to wait for metrics and traces to be pulled or pushed from the process. Default: 0s" }, "shutdown_timeout": { "type": "string", "description": "The maximum period of time to wait for a clean shutdown. If this time is exceeded Bento will forcefully close. Default: 20s" }, "tests": { "type": "array", "description": "A list of one or more unit tests to execute.", "items": { "type": "object" } }, "tracer": { "type": "object", "description": "A mechanism for exporting traces. Default: map[none:map[]]", "properties": { "gcp_cloudtrace": { "type": "object", "properties": { "flush_interval": { "type": "string", "description": "The period of time between each flush of tracing spans." }, "project": { "type": "string", "description": "The google project with Cloud Trace API enabled. If this is omitted then the Google Cloud SDK will attempt auto-detect it from the environment." }, "sampling_ratio": { "type": "number", "description": "Sets the ratio of traces to sample. Tuning the sampling ratio is recommended for high-volume production workloads. Default: 1", "examples": [ 1 ] }, "tags": { "type": "object", "description": "A map of tags to add to tracing spans. Default: map[]" } } }, "jaeger": { "type": "object", "properties": { "agent_address": { "type": "string", "description": "The address of a Jaeger agent to send tracing events to. Default: ", "examples": [ "jaeger-agent:6831" ] }, "collector_url": { "type": "string", "description": "The URL of a Jaeger collector to send tracing events to. If set, this will override `agent_address`. Default: ", "examples": [ "https://jaeger-collector:14268/api/traces" ] }, "flush_interval": { "type": "string", "description": "The period of time between each flush of tracing spans." }, "sampler_param": { "type": "number", "description": "A parameter to use for sampling. This field is unused for some sampling types. Default: 1" }, "sampler_type": { "type": "string", "description": "The sampler type to use. Default: const" }, "tags": { "type": "object", "description": "A map of tags to add to tracing spans. Default: map[]" } } }, "none": { "type": "object", "description": " Default: map[]" }, "open_telemetry_collector": { "type": "object", "properties": { "grpc": { "type": "array", "description": "A list of grpc collectors.", "items": { "type": "object" } }, "http": { "type": "array", "description": "A list of http collectors.", "items": { "type": "object" } }, "sampling": { "type": "object", "description": "Settings for trace sampling. Sampling is recommended for high-volume production workloads.", "properties": { "enabled": { "type": "boolean", "description": "Whether to enable sampling. Default: false" }, "ratio": { "type": "number", "description": "Sets the ratio of traces to sample.", "examples": [ 0.85, 0.5 ] } } }, "tags": { "type": "object", "description": "A map of tags to add to all tracing spans. Default: map[]" } } } } } } }