359 lines
15 KiB
YAML
359 lines
15 KiB
YAML
---
|
|
# yamllint disable rule:document-start rule:line-length rule:trailing-spaces rule:empty-lines
|
|
suite: Test Tempo Integration - Metrics
|
|
templates:
|
|
- configmap.yaml
|
|
tests:
|
|
- it: should create the tempo metrics config
|
|
set:
|
|
deployAsConfigMap: true
|
|
tempo:
|
|
instances:
|
|
- name: tempo
|
|
labelSelectors:
|
|
app.kubernetes.io/name: tempo
|
|
asserts:
|
|
- isKind:
|
|
of: ConfigMap
|
|
- equal:
|
|
path: data["metrics.alloy"]
|
|
value: |-
|
|
declare "tempo_integration" {
|
|
argument "metrics_destinations" {
|
|
comment = "Must be a list of metric destinations where collected metrics should be forwarded to"
|
|
}
|
|
|
|
declare "tempo_integration_discovery" {
|
|
argument "namespaces" {
|
|
comment = "The namespaces to look for targets in (default: [] is all namespaces)"
|
|
optional = true
|
|
}
|
|
|
|
argument "field_selectors" {
|
|
comment = "The field selectors to use to find matching targets (default: [])"
|
|
optional = true
|
|
}
|
|
|
|
argument "label_selectors" {
|
|
comment = "The label selectors to use to find matching targets (default: [\"app.kubernetes.io/name=tempo\"])"
|
|
optional = true
|
|
}
|
|
|
|
argument "port_name" {
|
|
comment = "The of the port to scrape metrics from (default: http-metrics)"
|
|
optional = true
|
|
}
|
|
|
|
// tempo service discovery for all of the pods
|
|
discovery.kubernetes "tempo_pods" {
|
|
role = "pod"
|
|
|
|
selectors {
|
|
role = "pod"
|
|
field = string.join(coalesce(argument.field_selectors.value, []), ",")
|
|
label = string.join(coalesce(argument.label_selectors.value, ["app.kubernetes.io/name=tempo"]), ",")
|
|
}
|
|
|
|
namespaces {
|
|
names = coalesce(argument.namespaces.value, [])
|
|
}
|
|
}
|
|
|
|
// tempo relabelings (pre-scrape)
|
|
discovery.relabel "tempo_pods" {
|
|
targets = discovery.kubernetes.tempo_pods.targets
|
|
|
|
// keep only the specified metrics port name, and pods that are Running and ready
|
|
rule {
|
|
source_labels = [
|
|
"__meta_kubernetes_pod_container_port_name",
|
|
"__meta_kubernetes_pod_phase",
|
|
"__meta_kubernetes_pod_ready",
|
|
"__meta_kubernetes_pod_container_init",
|
|
]
|
|
separator = "@"
|
|
regex = coalesce(argument.port_name.value, "http-metrics") + "@Running@true@false"
|
|
action = "keep"
|
|
}
|
|
|
|
// the tempo-mixin expects the job label to be namespace/component
|
|
rule {
|
|
source_labels = ["__meta_kubernetes_namespace","__meta_kubernetes_pod_label_app_kubernetes_io_component"]
|
|
separator = "/"
|
|
target_label = "job"
|
|
}
|
|
|
|
|
|
|
|
rule {
|
|
source_labels = ["__meta_kubernetes_namespace"]
|
|
target_label = "namespace"
|
|
}
|
|
|
|
rule {
|
|
source_labels = ["__meta_kubernetes_pod_name"]
|
|
target_label = "pod"
|
|
}
|
|
|
|
rule {
|
|
source_labels = ["__meta_kubernetes_pod_container_name"]
|
|
target_label = "container"
|
|
}
|
|
|
|
// set the workload to the controller kind and name
|
|
rule {
|
|
action = "lowercase"
|
|
source_labels = ["__meta_kubernetes_pod_controller_kind"]
|
|
target_label = "workload_type"
|
|
}
|
|
|
|
rule {
|
|
source_labels = ["__meta_kubernetes_pod_controller_name"]
|
|
target_label = "workload"
|
|
}
|
|
|
|
// remove the hash from the ReplicaSet
|
|
rule {
|
|
source_labels = [
|
|
"workload_type",
|
|
"workload",
|
|
]
|
|
separator = "/"
|
|
regex = "replicaset/(.+)-.+$"
|
|
target_label = "workload"
|
|
}
|
|
|
|
// set the app name if specified as metadata labels "app:" or "app.kubernetes.io/name:" or "k8s-app:"
|
|
rule {
|
|
action = "replace"
|
|
source_labels = [
|
|
"__meta_kubernetes_pod_label_app_kubernetes_io_name",
|
|
"__meta_kubernetes_pod_label_k8s_app",
|
|
"__meta_kubernetes_pod_label_app",
|
|
]
|
|
separator = ";"
|
|
regex = "^(?:;*)?([^;]+).*$"
|
|
replacement = "$1"
|
|
target_label = "app"
|
|
}
|
|
|
|
// set the component if specified as metadata labels "component:" or "app.kubernetes.io/component:" or "k8s-component:"
|
|
rule {
|
|
action = "replace"
|
|
source_labels = [
|
|
"__meta_kubernetes_pod_label_app_kubernetes_io_component",
|
|
"__meta_kubernetes_pod_label_k8s_component",
|
|
"__meta_kubernetes_pod_label_component",
|
|
]
|
|
regex = "^(?:;*)?([^;]+).*$"
|
|
replacement = "$1"
|
|
target_label = "component"
|
|
}
|
|
|
|
// set a source label
|
|
rule {
|
|
action = "replace"
|
|
replacement = "kubernetes"
|
|
target_label = "source"
|
|
}
|
|
}
|
|
|
|
export "output" {
|
|
value = discovery.relabel.tempo_pods.output
|
|
}
|
|
}
|
|
|
|
declare "tempo_integration_scrape" {
|
|
argument "targets" {
|
|
comment = "Must be a list() of targets"
|
|
}
|
|
|
|
argument "forward_to" {
|
|
comment = "Must be a list(MetricsReceiver) where collected metrics should be forwarded to"
|
|
}
|
|
|
|
argument "job_label" {
|
|
comment = "The job label to add for all Tempo metrics (default: integrations/tempo)"
|
|
optional = true
|
|
}
|
|
|
|
argument "keep_metrics" {
|
|
comment = "A regular expression of metrics to keep (default: see below)"
|
|
optional = true
|
|
}
|
|
|
|
argument "drop_metrics" {
|
|
comment = "A regular expression of metrics to drop (default: see below)"
|
|
optional = true
|
|
}
|
|
|
|
argument "scrape_interval" {
|
|
comment = "How often to scrape metrics from the targets (default: 60s)"
|
|
optional = true
|
|
}
|
|
|
|
argument "max_cache_size" {
|
|
comment = "The maximum number of elements to hold in the relabeling cache (default: 100000). This should be at least 2x-5x your largest scrape target or samples appended rate."
|
|
optional = true
|
|
}
|
|
|
|
argument "clustering" {
|
|
comment = "Whether or not clustering should be enabled (default: false)"
|
|
optional = true
|
|
}
|
|
|
|
prometheus.scrape "tempo" {
|
|
job_name = coalesce(argument.job_label.value, "integrations/tempo")
|
|
forward_to = [prometheus.relabel.tempo.receiver]
|
|
targets = argument.targets.value
|
|
scrape_interval = coalesce(argument.scrape_interval.value, "60s")
|
|
|
|
clustering {
|
|
enabled = coalesce(argument.clustering.value, false)
|
|
}
|
|
}
|
|
|
|
// tempo metric relabelings (post-scrape)
|
|
prometheus.relabel "tempo" {
|
|
forward_to = argument.forward_to.value
|
|
max_cache_size = coalesce(argument.max_cache_size.value, 100000)
|
|
|
|
// drop metrics that match the drop_metrics regex
|
|
rule {
|
|
source_labels = ["__name__"]
|
|
regex = coalesce(argument.drop_metrics.value, "")
|
|
action = "drop"
|
|
}
|
|
|
|
// keep only metrics that match the keep_metrics regex
|
|
rule {
|
|
source_labels = ["__name__"]
|
|
regex = coalesce(argument.keep_metrics.value, "(.+)")
|
|
action = "keep"
|
|
}
|
|
|
|
// the tempo-mixin expects the instance label to be the node name
|
|
rule {
|
|
source_labels = ["node"]
|
|
target_label = "instance"
|
|
replacement = "$1"
|
|
}
|
|
rule {
|
|
action = "labeldrop"
|
|
regex = "node"
|
|
}
|
|
|
|
// set the memcached exporter container name from container="exporter" to container="memcached"
|
|
rule {
|
|
source_labels = ["component", "container"]
|
|
separator = ";"
|
|
regex = "memcached-[^;]+;exporter"
|
|
target_label = "container"
|
|
replacement = "memcached"
|
|
}
|
|
}
|
|
}
|
|
|
|
tempo_integration_discovery "tempo" {
|
|
namespaces = []
|
|
label_selectors = ["app.kubernetes.io/name=tempo"]
|
|
port_name = "http-metrics"
|
|
}
|
|
|
|
tempo_integration_scrape "tempo" {
|
|
targets = tempo_integration_discovery.tempo.output
|
|
job_label = "integrations/tempo"
|
|
clustering = true
|
|
keep_metrics = "up|scrape_samples_scraped|container_cpu_usage_seconds_total|container_memory_working_set_bytes|container_network_receive_bytes_total|container_network_transmit_bytes_total|container_spec_cpu_period|container_spec_cpu_quota|container_spec_memory_limit_bytes|gauge_memberlist_health_score|go_gc_duration_seconds_count|go_goroutines|go_memstats_heap_inuse_bytes|kube_deployment_spec_replicas|kube_deployment_status_replicas_unavailable|kube_deployment_status_replicas_updated|kube_pod_container_info|kube_pod_container_resource_requests|kube_pod_container_status_restarts_total|kube_statefulset_replicas|kube_statefulset_status_replicas_current|kube_statefulset_status_replicas_ready|kube_statefulset_status_replicas_updated|kubelet_volume_stats_available_bytes|promtail_custom_bad_words_total|tempo_build_info|tempo_discarded_spans_total|tempo_distributor_bytes_received_total|tempo_distributor_metrics_generator_pushes_failures_total|tempo_distributor_push_duration_seconds_bucket|tempo_distributor_push_duration_seconds_count|tempo_distributor_push_duration_seconds_sum|tempo_distributor_queue_length|tempo_distributor_spans_received_total|tempo_ingester_blocks_cleared_total|tempo_ingester_blocks_flushed_total|tempo_ingester_failed_flushes_total|tempo_ingester_flush_duration_seconds_bucket|tempo_ingester_live_traces|tempo_ingester_traces_created_total|tempo_limits_defaults|tempo_limits_overrides|tempo_memberlist_client_cluster_members_count|tempo_memberlist_client_cluster_node_health_score|tempo_memberlist_client_kv_store_count|tempo_memcache_request_duration_seconds_bucket|tempo_memcache_request_duration_seconds_count|tempo_memcache_request_duration_seconds_sum|tempo_metrics_generator_bytes_received_total|tempo_metrics_generator_registry_active_series|tempo_metrics_generator_spans_discarded_total|tempo_metrics_generator_spans_received_total|tempo_query_frontend_queries_total|tempo_receiver_accepted_spans|tempo_receiver_refused_spans|tempo_request_duration_seconds_bucket|tempo_request_duration_seconds_count|tempo_request_duration_seconds_sum|tempo_vulture_trace_error_total|tempo_vulture_trace_total|tempodb_backend_request_duration_seconds_bucket|tempodb_backend_request_duration_seconds_count|tempodb_backend_request_duration_seconds_sum|tempodb_blocklist_length|tempodb_blocklist_poll_errors_total|tempodb_compaction_blocks_total|tempodb_compaction_bytes_written_total|tempodb_compaction_errors_total|tempodb_compaction_objects_combined_total|tempodb_compaction_objects_written_total|tempodb_compaction_outstanding_blocks|tempodb_retention_deleted_total|tempodb_retention_errors_total|tempodb_retention_marked_for_deletion_total|tempodb_work_queue_length|tempodb_work_queue_max"
|
|
scrape_interval = "60s"
|
|
max_cache_size = 100000
|
|
forward_to = argument.metrics_destinations.value
|
|
}
|
|
}
|
|
|
|
- it: should allow you to restrict the namespaces for metrics
|
|
set:
|
|
deployAsConfigMap: true
|
|
tempo:
|
|
instances:
|
|
- name: tempo
|
|
namespaces:
|
|
- k8smon
|
|
asserts:
|
|
- isKind:
|
|
of: ConfigMap
|
|
- matchRegex:
|
|
path: data["metrics.alloy"]
|
|
pattern: namespaces = \["k8smon"\]
|
|
|
|
- it: should allow you to disable the default allow list
|
|
set:
|
|
deployAsConfigMap: true
|
|
tempo:
|
|
instances:
|
|
- name: tempo
|
|
metrics:
|
|
tuning:
|
|
useDefaultAllowList: false
|
|
asserts:
|
|
- isKind:
|
|
of: ConfigMap
|
|
- matchRegex:
|
|
path: data["metrics.alloy"]
|
|
# The pattern should look like this, but since the regex is escaped, it will be a bit different
|
|
# tempo_integration_scrape "tempo" {
|
|
# targets = tempo_integration_discovery.tempo.output
|
|
# job_label = "integrations/tempo"
|
|
# clustering = true
|
|
# scrape_interval = "60s"
|
|
# max_cache_size = 100000
|
|
# forward_to = argument.metrics_destinations.value
|
|
# }
|
|
pattern: |-
|
|
\s*tempo_integration_scrape "tempo" {
|
|
\s* targets = tempo_integration_discovery.tempo.output
|
|
\s* job_label = "integrations/tempo"
|
|
\s* clustering = true
|
|
\s* scrape_interval = "60s"
|
|
\s* max_cache_size = 100000
|
|
\s* forward_to = argument.metrics_destinations.value
|
|
\s*}
|
|
|
|
- it: should allow you to specific which metrics to include
|
|
set:
|
|
deployAsConfigMap: true
|
|
tempo:
|
|
instances:
|
|
- name: tempo
|
|
metrics:
|
|
tuning:
|
|
useDefaultAllowList: false
|
|
includeMetrics:
|
|
- foo
|
|
- bar
|
|
asserts:
|
|
- isKind:
|
|
of: ConfigMap
|
|
- matchRegex:
|
|
path: data["metrics.alloy"]
|
|
pattern: keep_metrics = "up\|scrape_samples_scraped\|foo\|bar"
|
|
|
|
- it: should allow you to specific which metrics to exclude
|
|
set:
|
|
deployAsConfigMap: true
|
|
tempo:
|
|
instances:
|
|
- name: tempo
|
|
metrics:
|
|
tuning:
|
|
useDefaultAllowList: false
|
|
excludeMetrics:
|
|
- foo
|
|
- bar
|
|
asserts:
|
|
- isKind:
|
|
of: ConfigMap
|
|
- matchRegex:
|
|
path: data["metrics.alloy"]
|
|
pattern: drop_metrics = "foo\|bar"
|