--Task-- name: Enable_prometheus_admin_api enabled: True class_name: EnablePrometheusAdminAPI source_name: controller source_namespace: >default< target_name: controller target_namespace: >default< start: 0.0 stop: None timeout: no timeout loop: False interval: None dependencies: [] wait_for: [] preceding_task: None options: {} group_name: None Current dir: /mnt/disks/data/xslou/lodestar-fork/pyrock ________________________________________________________________________________ [2024-04-11 01:16:20] Enable_prometheus_admin_api pre : N/A ________________________________________________________________________________ ________________________________________________________________________________ [2024-04-11 01:16:20] Enable_prometheus_admin_api step1 : Discover the prometheus pod ________________________________________________________________________________ [2024-04-11 01:16:20] - INFO: Discovering prometheus pods 2024-04-11 01:16:20,255 INFO 2024-04-11 01:16:20,256 INFO [loop_until]: kubectl --namespace=monitoring --context=gke_engineeringpit_us-east1-d_xlou-cdm get pods --output json --field-selector metadata.name=prometheus-prometheus-operator-kube-p-prometheus-0 2024-04-11 01:16:20,256 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 2024-04-11 01:16:20,698 INFO [loop_until]: OK (rc = 0) 2024-04-11 01:16:20,698 DEBUG --- stdout --- 2024-04-11 01:16:20,698 DEBUG [print_head_tail]: Print head (100 lines) and tail (100 lines) of input string (521 lines): 2024-04-11 01:16:20,698 DEBUG ----- output ----- 2024-04-11 01:16:20,698 DEBUG { 2024-04-11 01:16:20,698 DEBUG "apiVersion": "v1", 2024-04-11 01:16:20,698 DEBUG "items": [ 2024-04-11 01:16:20,698 DEBUG { 2024-04-11 01:16:20,698 DEBUG "apiVersion": "v1", 2024-04-11 01:16:20,698 DEBUG "kind": "Pod", 2024-04-11 01:16:20,699 DEBUG "metadata": { 2024-04-11 01:16:20,699 DEBUG "annotations": { 2024-04-11 01:16:20,699 DEBUG "cloud.google.com/cluster_autoscaler_unhelpable_since": "2024-04-10T15:11:23+0000", 2024-04-11 01:16:20,699 DEBUG "cloud.google.com/cluster_autoscaler_unhelpable_until": "Inf", 2024-04-11 01:16:20,699 DEBUG "kubectl.kubernetes.io/default-container": "prometheus" 2024-04-11 01:16:20,699 DEBUG }, 2024-04-11 01:16:20,699 DEBUG "creationTimestamp": "2024-04-10T15:11:21Z", 2024-04-11 01:16:20,699 DEBUG "generateName": "prometheus-prometheus-operator-kube-p-prometheus-", 2024-04-11 01:16:20,699 DEBUG "labels": { 2024-04-11 01:16:20,699 DEBUG "app.kubernetes.io/instance": "prometheus-operator-kube-p-prometheus", 2024-04-11 01:16:20,699 DEBUG "app.kubernetes.io/managed-by": "prometheus-operator", 2024-04-11 01:16:20,699 DEBUG "app.kubernetes.io/name": "prometheus", 2024-04-11 01:16:20,699 DEBUG "app.kubernetes.io/version": "2.44.0", 2024-04-11 01:16:20,699 DEBUG "controller-revision-hash": "prometheus-prometheus-operator-kube-p-prometheus-7b495c798d", 2024-04-11 01:16:20,699 DEBUG "operator.prometheus.io/name": "prometheus-operator-kube-p-prometheus", 2024-04-11 01:16:20,699 DEBUG "operator.prometheus.io/shard": "0", 2024-04-11 01:16:20,699 DEBUG "prometheus": "prometheus-operator-kube-p-prometheus", 2024-04-11 01:16:20,699 DEBUG "statefulset.kubernetes.io/pod-name": "prometheus-prometheus-operator-kube-p-prometheus-0" 2024-04-11 01:16:20,699 DEBUG }, 2024-04-11 01:16:20,699 DEBUG "name": "prometheus-prometheus-operator-kube-p-prometheus-0", 2024-04-11 01:16:20,699 DEBUG "namespace": "monitoring", 2024-04-11 01:16:20,699 DEBUG "ownerReferences": [ 2024-04-11 01:16:20,699 DEBUG { 2024-04-11 01:16:20,699 DEBUG "apiVersion": "apps/v1", 2024-04-11 01:16:20,699 DEBUG "blockOwnerDeletion": true, 2024-04-11 01:16:20,699 DEBUG "controller": true, 2024-04-11 01:16:20,699 DEBUG "kind": "StatefulSet", 2024-04-11 01:16:20,699 DEBUG "name": "prometheus-prometheus-operator-kube-p-prometheus", 2024-04-11 01:16:20,699 DEBUG "uid": "af636164-e309-4214-abfb-69e4f4ed19ff" 2024-04-11 01:16:20,699 DEBUG } 2024-04-11 01:16:20,699 DEBUG ], 2024-04-11 01:16:20,699 DEBUG "resourceVersion": "231192234", 2024-04-11 01:16:20,699 DEBUG "uid": "7c1e5bc8-2a94-4964-af28-4fa3debb7985" 2024-04-11 01:16:20,699 DEBUG }, 2024-04-11 01:16:20,699 DEBUG "spec": { 2024-04-11 01:16:20,699 DEBUG "automountServiceAccountToken": true, 2024-04-11 01:16:20,699 DEBUG "containers": [ 2024-04-11 01:16:20,699 DEBUG { 2024-04-11 01:16:20,699 DEBUG "args": [ 2024-04-11 01:16:20,699 DEBUG "--web.console.templates=/etc/prometheus/consoles", 2024-04-11 01:16:20,699 DEBUG "--web.console.libraries=/etc/prometheus/console_libraries", 2024-04-11 01:16:20,699 DEBUG "--config.file=/etc/prometheus/config_out/prometheus.env.yaml", 2024-04-11 01:16:20,699 DEBUG "--web.enable-lifecycle", 2024-04-11 01:16:20,699 DEBUG "--web.external-url=http://prometheus-operator-kube-p-prometheus.monitoring:9090", 2024-04-11 01:16:20,699 DEBUG "--web.route-prefix=/", 2024-04-11 01:16:20,699 DEBUG "--storage.tsdb.wal-compression", 2024-04-11 01:16:20,700 DEBUG "--storage.tsdb.retention.time=10d", 2024-04-11 01:16:20,700 DEBUG "--storage.tsdb.path=/prometheus", 2024-04-11 01:16:20,700 DEBUG "--web.enable-admin-api", 2024-04-11 01:16:20,700 DEBUG "--web.config.file=/etc/prometheus/web_config/web-config.yaml" 2024-04-11 01:16:20,700 DEBUG ], 2024-04-11 01:16:20,700 DEBUG "image": "quay.io/prometheus/prometheus:v2.44.0", 2024-04-11 01:16:20,700 DEBUG "imagePullPolicy": "IfNotPresent", 2024-04-11 01:16:20,700 DEBUG "livenessProbe": { 2024-04-11 01:16:20,700 DEBUG "failureThreshold": 6, 2024-04-11 01:16:20,700 DEBUG "httpGet": { 2024-04-11 01:16:20,700 DEBUG "path": "/-/healthy", 2024-04-11 01:16:20,700 DEBUG "port": "http-web", 2024-04-11 01:16:20,700 DEBUG "scheme": "HTTP" 2024-04-11 01:16:20,700 DEBUG }, 2024-04-11 01:16:20,700 DEBUG "periodSeconds": 5, 2024-04-11 01:16:20,700 DEBUG "successThreshold": 1, 2024-04-11 01:16:20,700 DEBUG "timeoutSeconds": 3 2024-04-11 01:16:20,700 DEBUG }, 2024-04-11 01:16:20,700 DEBUG "name": "prometheus", 2024-04-11 01:16:20,700 DEBUG "ports": [ 2024-04-11 01:16:20,700 DEBUG { 2024-04-11 01:16:20,700 DEBUG "containerPort": 9090, 2024-04-11 01:16:20,700 DEBUG "name": "http-web", 2024-04-11 01:16:20,700 DEBUG "protocol": "TCP" 2024-04-11 01:16:20,700 DEBUG } 2024-04-11 01:16:20,700 DEBUG ], 2024-04-11 01:16:20,700 DEBUG "readinessProbe": { 2024-04-11 01:16:20,700 DEBUG "failureThreshold": 3, 2024-04-11 01:16:20,700 DEBUG "httpGet": { 2024-04-11 01:16:20,700 DEBUG "path": "/-/ready", 2024-04-11 01:16:20,700 DEBUG "port": "http-web", 2024-04-11 01:16:20,700 DEBUG "scheme": "HTTP" 2024-04-11 01:16:20,700 DEBUG }, 2024-04-11 01:16:20,701 DEBUG "periodSeconds": 5, 2024-04-11 01:16:20,701 DEBUG "successThreshold": 1, 2024-04-11 01:16:20,701 DEBUG "timeoutSeconds": 3 2024-04-11 01:16:20,701 DEBUG }, 2024-04-11 01:16:20,701 DEBUG "resources": {}, 2024-04-11 01:16:20,701 DEBUG "securityContext": { 2024-04-11 01:16:20,701 DEBUG "allowPrivilegeEscalation": false, 2024-04-11 01:16:20,701 DEBUG "capabilities": { 2024-04-11 01:16:20,701 DEBUG "drop": [ 2024-04-11 01:16:20,701 DEBUG "ALL" 2024-04-11 01:16:20,701 DEBUG ] 2024-04-11 01:16:20,701 DEBUG }, 2024-04-11 01:16:20,701 DEBUG "readOnlyRootFilesystem": true 2024-04-11 01:16:20,701 DEBUG }, 2024-04-11 01:16:20,701 DEBUG "startupProbe": { 2024-04-11 01:16:20,701 DEBUG 2024-04-11 01:16:20,701 DEBUG 2024-04-11 01:16:20,701 DEBUG 2024-04-11 01:16:20,701 DEBUG [...] 2024-04-11 01:16:20,701 DEBUG 2024-04-11 01:16:20,701 DEBUG 2024-04-11 01:16:20,701 DEBUG 2024-04-11 01:16:20,701 DEBUG } 2024-04-11 01:16:20,701 DEBUG ] 2024-04-11 01:16:20,701 DEBUG }, 2024-04-11 01:16:20,701 DEBUG "status": { 2024-04-11 01:16:20,701 DEBUG "conditions": [ 2024-04-11 01:16:20,701 DEBUG { 2024-04-11 01:16:20,701 DEBUG "lastProbeTime": null, 2024-04-11 01:16:20,701 DEBUG "lastTransitionTime": "2024-04-10T15:40:17Z", 2024-04-11 01:16:20,701 DEBUG "status": "True", 2024-04-11 01:16:20,701 DEBUG "type": "Initialized" 2024-04-11 01:16:20,701 DEBUG }, 2024-04-11 01:16:20,701 DEBUG { 2024-04-11 01:16:20,701 DEBUG "lastProbeTime": null, 2024-04-11 01:16:20,701 DEBUG "lastTransitionTime": "2024-04-10T15:40:28Z", 2024-04-11 01:16:20,701 DEBUG "status": "True", 2024-04-11 01:16:20,701 DEBUG "type": "Ready" 2024-04-11 01:16:20,701 DEBUG }, 2024-04-11 01:16:20,701 DEBUG { 2024-04-11 01:16:20,701 DEBUG "lastProbeTime": null, 2024-04-11 01:16:20,701 DEBUG "lastTransitionTime": "2024-04-10T15:40:28Z", 2024-04-11 01:16:20,701 DEBUG "status": "True", 2024-04-11 01:16:20,701 DEBUG "type": "ContainersReady" 2024-04-11 01:16:20,701 DEBUG }, 2024-04-11 01:16:20,702 DEBUG { 2024-04-11 01:16:20,702 DEBUG "lastProbeTime": null, 2024-04-11 01:16:20,702 DEBUG "lastTransitionTime": "2024-04-10T15:40:05Z", 2024-04-11 01:16:20,702 DEBUG "status": "True", 2024-04-11 01:16:20,702 DEBUG "type": "PodScheduled" 2024-04-11 01:16:20,702 DEBUG } 2024-04-11 01:16:20,702 DEBUG ], 2024-04-11 01:16:20,702 DEBUG "containerStatuses": [ 2024-04-11 01:16:20,702 DEBUG { 2024-04-11 01:16:20,702 DEBUG "containerID": "containerd://9d2a0598716cece4a6db598d01341c370f59e6efcbf8bfe90270fdbbdfee680b", 2024-04-11 01:16:20,702 DEBUG "image": "quay.io/prometheus-operator/prometheus-config-reloader:v0.65.1", 2024-04-11 01:16:20,702 DEBUG "imageID": "quay.io/prometheus-operator/prometheus-config-reloader@sha256:ca8684fd1e35ae73c9d497ea37cf5216a32e88f1d0e2eeab22247b532f1e508f", 2024-04-11 01:16:20,702 DEBUG "lastState": {}, 2024-04-11 01:16:20,702 DEBUG "name": "config-reloader", 2024-04-11 01:16:20,702 DEBUG "ready": true, 2024-04-11 01:16:20,702 DEBUG "restartCount": 0, 2024-04-11 01:16:20,702 DEBUG "started": true, 2024-04-11 01:16:20,702 DEBUG "state": { 2024-04-11 01:16:20,702 DEBUG "running": { 2024-04-11 01:16:20,702 DEBUG "startedAt": "2024-04-10T15:40:21Z" 2024-04-11 01:16:20,702 DEBUG } 2024-04-11 01:16:20,702 DEBUG } 2024-04-11 01:16:20,702 DEBUG }, 2024-04-11 01:16:20,702 DEBUG { 2024-04-11 01:16:20,702 DEBUG "containerID": "containerd://5d53025053c6d6cb8711cdb43dc0b3318755d75ac4adc5ec27bcd0a07db6b381", 2024-04-11 01:16:20,702 DEBUG "image": "quay.io/prometheus/prometheus:v2.44.0", 2024-04-11 01:16:20,702 DEBUG "imageID": "quay.io/prometheus/prometheus@sha256:0f0b7feb6f02620df7d493ad7437b6ee95b6d16d8d18799f3607124e501444b1", 2024-04-11 01:16:20,702 DEBUG "lastState": {}, 2024-04-11 01:16:20,702 DEBUG "name": "prometheus", 2024-04-11 01:16:20,702 DEBUG "ready": true, 2024-04-11 01:16:20,702 DEBUG "restartCount": 0, 2024-04-11 01:16:20,702 DEBUG "started": true, 2024-04-11 01:16:20,702 DEBUG "state": { 2024-04-11 01:16:20,702 DEBUG "running": { 2024-04-11 01:16:20,702 DEBUG "startedAt": "2024-04-10T15:40:20Z" 2024-04-11 01:16:20,702 DEBUG } 2024-04-11 01:16:20,702 DEBUG } 2024-04-11 01:16:20,702 DEBUG } 2024-04-11 01:16:20,702 DEBUG ], 2024-04-11 01:16:20,702 DEBUG "hostIP": "10.142.0.109", 2024-04-11 01:16:20,702 DEBUG "initContainerStatuses": [ 2024-04-11 01:16:20,702 DEBUG { 2024-04-11 01:16:20,703 DEBUG "containerID": "containerd://6fd7c7bc838479dfd1f4811f7530ff8612a97e1b3e7043c20b5559c7c7d6e9b0", 2024-04-11 01:16:20,703 DEBUG "image": "quay.io/prometheus-operator/prometheus-config-reloader:v0.65.1", 2024-04-11 01:16:20,703 DEBUG "imageID": "quay.io/prometheus-operator/prometheus-config-reloader@sha256:ca8684fd1e35ae73c9d497ea37cf5216a32e88f1d0e2eeab22247b532f1e508f", 2024-04-11 01:16:20,703 DEBUG "lastState": {}, 2024-04-11 01:16:20,703 DEBUG "name": "init-config-reloader", 2024-04-11 01:16:20,703 DEBUG "ready": true, 2024-04-11 01:16:20,703 DEBUG "restartCount": 0, 2024-04-11 01:16:20,703 DEBUG "state": { 2024-04-11 01:16:20,703 DEBUG "terminated": { 2024-04-11 01:16:20,703 DEBUG "containerID": "containerd://6fd7c7bc838479dfd1f4811f7530ff8612a97e1b3e7043c20b5559c7c7d6e9b0", 2024-04-11 01:16:20,703 DEBUG "exitCode": 0, 2024-04-11 01:16:20,703 DEBUG "finishedAt": "2024-04-10T15:40:16Z", 2024-04-11 01:16:20,703 DEBUG "reason": "Completed", 2024-04-11 01:16:20,703 DEBUG "startedAt": "2024-04-10T15:40:15Z" 2024-04-11 01:16:20,703 DEBUG } 2024-04-11 01:16:20,703 DEBUG } 2024-04-11 01:16:20,703 DEBUG } 2024-04-11 01:16:20,703 DEBUG ], 2024-04-11 01:16:20,703 DEBUG "phase": "Running", 2024-04-11 01:16:20,703 DEBUG "podIP": "10.106.41.131", 2024-04-11 01:16:20,703 DEBUG "podIPs": [ 2024-04-11 01:16:20,703 DEBUG { 2024-04-11 01:16:20,703 DEBUG "ip": "10.106.41.131" 2024-04-11 01:16:20,703 DEBUG } 2024-04-11 01:16:20,703 DEBUG ], 2024-04-11 01:16:20,703 DEBUG "qosClass": "Burstable", 2024-04-11 01:16:20,703 DEBUG "startTime": "2024-04-10T15:40:04Z" 2024-04-11 01:16:20,703 DEBUG } 2024-04-11 01:16:20,703 DEBUG } 2024-04-11 01:16:20,703 DEBUG ], 2024-04-11 01:16:20,703 DEBUG "kind": "List", 2024-04-11 01:16:20,703 DEBUG "metadata": { 2024-04-11 01:16:20,703 DEBUG "resourceVersion": "" 2024-04-11 01:16:20,703 DEBUG } 2024-04-11 01:16:20,703 DEBUG } 2024-04-11 01:16:20,703 DEBUG --- stderr --- 2024-04-11 01:16:20,703 DEBUG ________________________________________________________________________________ [2024-04-11 01:16:20] Enable_prometheus_admin_api step2 : Enable the API if disabled ________________________________________________________________________________ [2024-04-11 01:16:20] - INFO: REST API is enabled: True ________________________________________________________________________________ [2024-04-11 01:16:20] Enable_prometheus_admin_api post : Post method ________________________________________________________________________________ Setting result to PASS Task has been successfully stopped