observability_pipelines

Creates, updates, deletes, gets or lists an observability_pipelines resource.

Overview

Name	observability_pipelines
Type	Resource
Id	datadog.logs.observability_pipelines

Fields

The following fields are returned by SELECT queries:

get_pipeline

Name	Datatype	Description
id	string	Unique identifier for the pipeline. (example: 3fa85f64-5717-4562-b3fc-2c963f66afa6)
attributes	object	Defines the pipeline’s name and its components (sources, processors, and destinations).
type	string	The resource type identifier. For pipeline resources, this should always be set to pipelines. (default: pipelines, example: pipelines)

list_pipelines

Name	Datatype	Description
id	string	Unique identifier for the pipeline. (example: 3fa85f64-5717-4562-b3fc-2c963f66afa6)
attributes	object	Defines the pipeline’s name and its components (sources, processors, and destinations).
type	string	The resource type identifier. For pipeline resources, this should always be set to pipelines. (default: pipelines, example: pipelines)

Methods

The following methods are available for this resource:

Name	Accessible by	Required Params	Optional Params	Description
get_pipeline	select	pipeline_id		Get a specific pipeline by its ID.
list_pipelines	select		page[size], page[number]	Retrieve a list of pipelines.
create_pipeline	insert	data		Create a new pipeline.
update_pipeline	replace	pipeline_id, data		Update a pipeline.
delete_pipeline	delete	pipeline_id		Delete a pipeline.
validate_pipeline	exec	data		Validates a pipeline configuration without creating or updating any resources.<br />Returns a list of validation errors, if any.

Parameters

Parameters can be passed in the WHERE clause of a query. Check the Methods section to see which parameters are required or optional for each operation.

Name	Datatype	Description
pipeline_id	string	The ID of the pipeline to delete.
site	string	The Datadog site (region) for your organization, for example datadoghq.com, us3.datadoghq.com, us5.datadoghq.com, ap1.datadoghq.com, ap2.datadoghq.com, datadoghq.eu, ddog-gov.com. Resolved from the DD_SITE environment variable when set. Optional: defaults to datadoghq.com, or the value of the DD_SITE environment variable when set; a WHERE value overrides both.
page[number]	integer (int64)	Specific page number to return.
page[size]	integer (int64)	Number of items to return per page. The maximum allowed value is 100.

SELECT examples

get_pipeline

Get a specific pipeline by its ID.

SELECT
id,
attributes,
type
FROM datadog.logs.observability_pipelines
WHERE pipeline_id = '{{ pipeline_id }}' -- required
;

list_pipelines

Retrieve a list of pipelines.

SELECT
id,
attributes,
type
FROM datadog.logs.observability_pipelines
WHERE page[size] = '{{ page[size] }}'
AND page[number] = '{{ page[number] }}'
;

INSERT examples

create_pipeline

Create a new pipeline.

INSERT INTO datadog.logs.observability_pipelines (
data
)
SELECT 
'{{ data }}' /* required */
RETURNING
data
;

Manifest

# Description fields are for documentation purposes
- name: observability_pipelines
props:
  - name: data
    description: |
      Contains the the pipeline configuration.
    value:
      attributes:
        config:
          destinations:
            - api_version: "{{ api_version }}"
              auth:
                password_key: "{{ password_key }}"
                strategy: "{{ strategy }}"
                username_key: "{{ username_key }}"
              buffer:
                max_size: {{ max_size }}
                type: "{{ type }}"
                when_full: "{{ when_full }}"
                max_events: {{ max_events }}
              bulk_index: "{{ bulk_index }}"
              compression:
                algorithm: "{{ algorithm }}"
                level: {{ level }}
              data_stream:
                auto_routing: {{ auto_routing }}
                dataset: "{{ dataset }}"
                dtype: "{{ dtype }}"
                namespace: "{{ namespace }}"
                sync_fields: {{ sync_fields }}
              endpoint_url_key: "{{ endpoint_url_key }}"
              id: "{{ id }}"
              id_key: "{{ id_key }}"
              inputs: "{{ inputs }}"
              pipeline: "{{ pipeline }}"
              request_retry_partial: {{ request_retry_partial }}
              tls:
                ca_file: "{{ ca_file }}"
                crt_file: "{{ crt_file }}"
                key_file: "{{ key_file }}"
                key_pass_key: "{{ key_pass_key }}"
              type: "{{ type }}"
              auth_strategy: "{{ auth_strategy }}"
              custom_key: "{{ custom_key }}"
              encoding: "{{ encoding }}"
              password_key: "{{ password_key }}"
              token_key: "{{ token_key }}"
              uri_key: "{{ uri_key }}"
              username_key: "{{ username_key }}"
              bucket: "{{ bucket }}"
              key_prefix: "{{ key_prefix }}"
              region: "{{ region }}"
              server_side_encryption: "{{ server_side_encryption }}"
              ssekms_key_id: "{{ ssekms_key_id }}"
              storage_class: "{{ storage_class }}"
              batch_settings:
                batch_size: {{ batch_size }}
                timeout_secs: {{ timeout_secs }}
              custom_source_name: "{{ custom_source_name }}"
              blob_prefix: "{{ blob_prefix }}"
              connection_string_key: "{{ connection_string_key }}"
              container_name: "{{ container_name }}"
              batch:
                max_events: {{ max_events }}
                timeout_secs: {{ timeout_secs }}
              batch_encoding:
                allow_nullable_fields: {{ allow_nullable_fields }}
                codec: "{{ codec }}"
              database: "{{ database }}"
              date_time_best_effort: {{ date_time_best_effort }}
              format: "{{ format }}"
              skip_unknown_fields: {{ skip_unknown_fields }}
              table: "{{ table }}"
              routes: "{{ routes }}"
              customer_id: "{{ customer_id }}"
              log_type: "{{ log_type }}"
              acl: "{{ acl }}"
              metadata: "{{ metadata }}"
              project: "{{ project }}"
              topic: "{{ topic }}"
              bootstrap_servers_key: "{{ bootstrap_servers_key }}"
              headers_key: "{{ headers_key }}"
              key_field: "{{ key_field }}"
              librdkafka_options: "{{ librdkafka_options }}"
              message_timeout_ms: {{ message_timeout_ms }}
              rate_limit_duration_secs: {{ rate_limit_duration_secs }}
              rate_limit_num: {{ rate_limit_num }}
              sasl:
                mechanism: "{{ mechanism }}"
                password_key: "{{ password_key }}"
                username_key: "{{ username_key }}"
              socket_timeout_ms: {{ socket_timeout_ms }}
              client_id: "{{ client_id }}"
              client_secret_key: "{{ client_secret_key }}"
              dce_uri_key: "{{ dce_uri_key }}"
              dcr_immutable_id: "{{ dcr_immutable_id }}"
              tenant_id: "{{ tenant_id }}"
              account_id_key: "{{ account_id_key }}"
              license_key_key: "{{ license_key_key }}"
              keepalive: {{ keepalive }}
              address_key: "{{ address_key }}"
              framing:
                method: "{{ method }}"
                delimiter: "{{ delimiter }}"
              mode: "{{ mode }}"
              auto_extract_timestamp: {{ auto_extract_timestamp }}
              index: "{{ index }}"
              indexed_fields: "{{ indexed_fields }}"
              sourcetype: "{{ sourcetype }}"
              token_strategy: "{{ token_strategy }}"
              header_custom_fields: "{{ header_custom_fields }}"
              header_host_name: "{{ header_host_name }}"
              header_source_category: "{{ header_source_category }}"
              header_source_name: "{{ header_source_name }}"
              ingestion_endpoint_key: "{{ ingestion_endpoint_key }}"
              table_name: "{{ table_name }}"
              unity_catalog_endpoint_key: "{{ unity_catalog_endpoint_key }}"
              default_namespace: "{{ default_namespace }}"
              source: "{{ source }}"
          pipeline_type: "{{ pipeline_type }}"
          processor_groups:
            - display_name: "{{ display_name }}"
              enabled: {{ enabled }}
              id: "{{ id }}"
              include: "{{ include }}"
              inputs: "{{ inputs }}"
              processors: "{{ processors }}"
          processors:
            - display_name: "{{ display_name }}"
              enabled: {{ enabled }}
              id: "{{ id }}"
              include: "{{ include }}"
              inputs: "{{ inputs }}"
              processors: "{{ processors }}"
          sources:
            - address_key: "{{ address_key }}"
              id: "{{ id }}"
              tls:
                ca_file: "{{ ca_file }}"
                crt_file: "{{ crt_file }}"
                key_file: "{{ key_file }}"
                key_pass_key: "{{ key_pass_key }}"
              type: "{{ type }}"
              auth:
                assume_role: "{{ assume_role }}"
                external_id: "{{ external_id }}"
                session_name: "{{ session_name }}"
              compression: "{{ compression }}"
              region: "{{ region }}"
              url_key: "{{ url_key }}"
              decoding: "{{ decoding }}"
              project: "{{ project }}"
              subscription: "{{ subscription }}"
              auth_strategy: "{{ auth_strategy }}"
              custom_key: "{{ custom_key }}"
              endpoint_url_key: "{{ endpoint_url_key }}"
              password_key: "{{ password_key }}"
              scrape_interval_secs: {{ scrape_interval_secs }}
              scrape_timeout_secs: {{ scrape_timeout_secs }}
              token_key: "{{ token_key }}"
              username_key: "{{ username_key }}"
              valid_tokens: "{{ valid_tokens }}"
              bootstrap_servers_key: "{{ bootstrap_servers_key }}"
              group_id: "{{ group_id }}"
              librdkafka_options: "{{ librdkafka_options }}"
              sasl:
                mechanism: "{{ mechanism }}"
                password_key: "{{ password_key }}"
                username_key: "{{ username_key }}"
              topics: "{{ topics }}"
              mode: "{{ mode }}"
              framing:
                method: "{{ method }}"
                delimiter: "{{ delimiter }}"
              store_hec_token: {{ store_hec_token }}
              uri_key: "{{ uri_key }}"
              grpc_address_key: "{{ grpc_address_key }}"
              http_address_key: "{{ http_address_key }}"
          use_legacy_search_syntax: {{ use_legacy_search_syntax }}
        name: "{{ name }}"
      type: "{{ type }}"

REPLACE examples

update_pipeline

Update a pipeline.

REPLACE datadog.logs.observability_pipelines
SET 
data = '{{ data }}'
WHERE 
pipeline_id = '{{ pipeline_id }}' --required
AND data = '{{ data }}' --required
RETURNING
data;

DELETE examples

delete_pipeline

Delete a pipeline.

DELETE FROM datadog.logs.observability_pipelines
WHERE pipeline_id = '{{ pipeline_id }}' --required
;

Lifecycle Methods

EXEC variables use wire (API) names.

validate_pipeline

Validates a pipeline configuration without creating or updating any resources.<br />Returns a list of validation errors, if any.

EXEC datadog.logs.observability_pipelines.validate_pipeline 
@@json=
'{
"data": "{{ data }}"
}'
;