--Task--
              name: Enable_prometheus_admin_api
           enabled: True
        class_name: EnablePrometheusAdminAPI
       source_name: controller
  source_namespace: >default<
       target_name: controller
  target_namespace: >default<
             start: 0.0
              stop: None
           timeout: no timeout
              loop: False
          interval: None
      dependencies: []
          wait_for: []
    preceding_task: None
           options: {}
        group_name: None
       Current dir: /mnt/disks/data/xslou/lodestar-fork/pyrock

________________________________________________________________________________

[2024-04-25 23:10:48] Enable_prometheus_admin_api pre : N/A
________________________________________________________________________________

________________________________________________________________________________

[2024-04-25 23:10:48] Enable_prometheus_admin_api step1 : Discover the prometheus pod
________________________________________________________________________________

[2024-04-25 23:10:48] - INFO: Discovering prometheus pods
2024-04-25 23:10:48,915 INFO    
2024-04-25 23:10:48,915 INFO    [loop_until]: kubectl --namespace=monitoring --context=gke_engineeringpit_us-east1-d_xlou-cdm get pods --output json --field-selector metadata.name=prometheus-prometheus-operator-kube-p-prometheus-0
2024-04-25 23:10:48,915 INFO    [loop_until]: (max_time=180, interval=5, expected_rc=[0]
2024-04-25 23:10:49,357 INFO    [loop_until]: OK (rc = 0)
2024-04-25 23:10:49,357 DEBUG   --- stdout ---
2024-04-25 23:10:49,358 DEBUG   [print_head_tail]: Print head (100 lines) and tail (100 lines) of input string (519 lines):
2024-04-25 23:10:49,358 DEBUG   ----- output -----
2024-04-25 23:10:49,358 DEBUG   {
2024-04-25 23:10:49,358 DEBUG       "apiVersion": "v1",
2024-04-25 23:10:49,358 DEBUG       "items": [
2024-04-25 23:10:49,358 DEBUG           {
2024-04-25 23:10:49,358 DEBUG               "apiVersion": "v1",
2024-04-25 23:10:49,358 DEBUG               "kind": "Pod",
2024-04-25 23:10:49,358 DEBUG               "metadata": {
2024-04-25 23:10:49,358 DEBUG                   "annotations": {
2024-04-25 23:10:49,358 DEBUG                       "kubectl.kubernetes.io/default-container": "prometheus"
2024-04-25 23:10:49,358 DEBUG                   },
2024-04-25 23:10:49,358 DEBUG                   "creationTimestamp": "2024-04-23T22:09:38Z",
2024-04-25 23:10:49,358 DEBUG                   "generateName": "prometheus-prometheus-operator-kube-p-prometheus-",
2024-04-25 23:10:49,358 DEBUG                   "labels": {
2024-04-25 23:10:49,358 DEBUG                       "app.kubernetes.io/instance": "prometheus-operator-kube-p-prometheus",
2024-04-25 23:10:49,358 DEBUG                       "app.kubernetes.io/managed-by": "prometheus-operator",
2024-04-25 23:10:49,358 DEBUG                       "app.kubernetes.io/name": "prometheus",
2024-04-25 23:10:49,358 DEBUG                       "app.kubernetes.io/version": "2.44.0",
2024-04-25 23:10:49,358 DEBUG                       "controller-revision-hash": "prometheus-prometheus-operator-kube-p-prometheus-7b495c798d",
2024-04-25 23:10:49,358 DEBUG                       "operator.prometheus.io/name": "prometheus-operator-kube-p-prometheus",
2024-04-25 23:10:49,358 DEBUG                       "operator.prometheus.io/shard": "0",
2024-04-25 23:10:49,358 DEBUG                       "prometheus": "prometheus-operator-kube-p-prometheus",
2024-04-25 23:10:49,358 DEBUG                       "statefulset.kubernetes.io/pod-name": "prometheus-prometheus-operator-kube-p-prometheus-0"
2024-04-25 23:10:49,358 DEBUG                   },
2024-04-25 23:10:49,358 DEBUG                   "name": "prometheus-prometheus-operator-kube-p-prometheus-0",
2024-04-25 23:10:49,358 DEBUG                   "namespace": "monitoring",
2024-04-25 23:10:49,358 DEBUG                   "ownerReferences": [
2024-04-25 23:10:49,358 DEBUG                       {
2024-04-25 23:10:49,358 DEBUG                           "apiVersion": "apps/v1",
2024-04-25 23:10:49,358 DEBUG                           "blockOwnerDeletion": true,
2024-04-25 23:10:49,358 DEBUG                           "controller": true,
2024-04-25 23:10:49,359 DEBUG                           "kind": "StatefulSet",
2024-04-25 23:10:49,359 DEBUG                           "name": "prometheus-prometheus-operator-kube-p-prometheus",
2024-04-25 23:10:49,359 DEBUG                           "uid": "af636164-e309-4214-abfb-69e4f4ed19ff"
2024-04-25 23:10:49,359 DEBUG                       }
2024-04-25 23:10:49,359 DEBUG                   ],
2024-04-25 23:10:49,359 DEBUG                   "resourceVersion": "241853578",
2024-04-25 23:10:49,359 DEBUG                   "uid": "e31f39b7-05b0-4e99-8b57-1be5f41754d3"
2024-04-25 23:10:49,359 DEBUG               },
2024-04-25 23:10:49,359 DEBUG               "spec": {
2024-04-25 23:10:49,359 DEBUG                   "automountServiceAccountToken": true,
2024-04-25 23:10:49,359 DEBUG                   "containers": [
2024-04-25 23:10:49,359 DEBUG                       {
2024-04-25 23:10:49,359 DEBUG                           "args": [
2024-04-25 23:10:49,359 DEBUG                               "--web.console.templates=/etc/prometheus/consoles",
2024-04-25 23:10:49,359 DEBUG                               "--web.console.libraries=/etc/prometheus/console_libraries",
2024-04-25 23:10:49,359 DEBUG                               "--config.file=/etc/prometheus/config_out/prometheus.env.yaml",
2024-04-25 23:10:49,359 DEBUG                               "--web.enable-lifecycle",
2024-04-25 23:10:49,359 DEBUG                               "--web.external-url=http://prometheus-operator-kube-p-prometheus.monitoring:9090",
2024-04-25 23:10:49,359 DEBUG                               "--web.route-prefix=/",
2024-04-25 23:10:49,359 DEBUG                               "--storage.tsdb.wal-compression",
2024-04-25 23:10:49,359 DEBUG                               "--storage.tsdb.retention.time=10d",
2024-04-25 23:10:49,359 DEBUG                               "--storage.tsdb.path=/prometheus",
2024-04-25 23:10:49,359 DEBUG                               "--web.enable-admin-api",
2024-04-25 23:10:49,359 DEBUG                               "--web.config.file=/etc/prometheus/web_config/web-config.yaml"
2024-04-25 23:10:49,359 DEBUG                           ],
2024-04-25 23:10:49,359 DEBUG                           "image": "quay.io/prometheus/prometheus:v2.44.0",
2024-04-25 23:10:49,359 DEBUG                           "imagePullPolicy": "IfNotPresent",
2024-04-25 23:10:49,359 DEBUG                           "livenessProbe": {
2024-04-25 23:10:49,359 DEBUG                               "failureThreshold": 6,
2024-04-25 23:10:49,359 DEBUG                               "httpGet": {
2024-04-25 23:10:49,359 DEBUG                                   "path": "/-/healthy",
2024-04-25 23:10:49,359 DEBUG                                   "port": "http-web",
2024-04-25 23:10:49,359 DEBUG                                   "scheme": "HTTP"
2024-04-25 23:10:49,359 DEBUG                               },
2024-04-25 23:10:49,359 DEBUG                               "periodSeconds": 5,
2024-04-25 23:10:49,359 DEBUG                               "successThreshold": 1,
2024-04-25 23:10:49,359 DEBUG                               "timeoutSeconds": 3
2024-04-25 23:10:49,359 DEBUG                           },
2024-04-25 23:10:49,359 DEBUG                           "name": "prometheus",
2024-04-25 23:10:49,359 DEBUG                           "ports": [
2024-04-25 23:10:49,359 DEBUG                               {
2024-04-25 23:10:49,359 DEBUG                                   "containerPort": 9090,
2024-04-25 23:10:49,359 DEBUG                                   "name": "http-web",
2024-04-25 23:10:49,359 DEBUG                                   "protocol": "TCP"
2024-04-25 23:10:49,359 DEBUG                               }
2024-04-25 23:10:49,359 DEBUG                           ],
2024-04-25 23:10:49,359 DEBUG                           "readinessProbe": {
2024-04-25 23:10:49,359 DEBUG                               "failureThreshold": 3,
2024-04-25 23:10:49,360 DEBUG                               "httpGet": {
2024-04-25 23:10:49,360 DEBUG                                   "path": "/-/ready",
2024-04-25 23:10:49,360 DEBUG                                   "port": "http-web",
2024-04-25 23:10:49,360 DEBUG                                   "scheme": "HTTP"
2024-04-25 23:10:49,360 DEBUG                               },
2024-04-25 23:10:49,360 DEBUG                               "periodSeconds": 5,
2024-04-25 23:10:49,360 DEBUG                               "successThreshold": 1,
2024-04-25 23:10:49,360 DEBUG                               "timeoutSeconds": 3
2024-04-25 23:10:49,360 DEBUG                           },
2024-04-25 23:10:49,360 DEBUG                           "resources": {},
2024-04-25 23:10:49,360 DEBUG                           "securityContext": {
2024-04-25 23:10:49,360 DEBUG                               "allowPrivilegeEscalation": false,
2024-04-25 23:10:49,360 DEBUG                               "capabilities": {
2024-04-25 23:10:49,360 DEBUG                                   "drop": [
2024-04-25 23:10:49,360 DEBUG                                       "ALL"
2024-04-25 23:10:49,360 DEBUG                                   ]
2024-04-25 23:10:49,360 DEBUG                               },
2024-04-25 23:10:49,360 DEBUG                               "readOnlyRootFilesystem": true
2024-04-25 23:10:49,360 DEBUG                           },
2024-04-25 23:10:49,360 DEBUG                           "startupProbe": {
2024-04-25 23:10:49,360 DEBUG                               "failureThreshold": 60,
2024-04-25 23:10:49,360 DEBUG                               "httpGet": {
2024-04-25 23:10:49,360 DEBUG   
2024-04-25 23:10:49,360 DEBUG   
2024-04-25 23:10:49,360 DEBUG   
2024-04-25 23:10:49,360 DEBUG   [...]
2024-04-25 23:10:49,360 DEBUG   
2024-04-25 23:10:49,360 DEBUG   
2024-04-25 23:10:49,360 DEBUG   
2024-04-25 23:10:49,360 DEBUG                       }
2024-04-25 23:10:49,360 DEBUG                   ]
2024-04-25 23:10:49,360 DEBUG               },
2024-04-25 23:10:49,360 DEBUG               "status": {
2024-04-25 23:10:49,360 DEBUG                   "conditions": [
2024-04-25 23:10:49,360 DEBUG                       {
2024-04-25 23:10:49,360 DEBUG                           "lastProbeTime": null,
2024-04-25 23:10:49,360 DEBUG                           "lastTransitionTime": "2024-04-23T22:09:58Z",
2024-04-25 23:10:49,360 DEBUG                           "status": "True",
2024-04-25 23:10:49,360 DEBUG                           "type": "Initialized"
2024-04-25 23:10:49,360 DEBUG                       },
2024-04-25 23:10:49,360 DEBUG                       {
2024-04-25 23:10:49,360 DEBUG                           "lastProbeTime": null,
2024-04-25 23:10:49,360 DEBUG                           "lastTransitionTime": "2024-04-23T22:10:23Z",
2024-04-25 23:10:49,360 DEBUG                           "status": "True",
2024-04-25 23:10:49,360 DEBUG                           "type": "Ready"
2024-04-25 23:10:49,360 DEBUG                       },
2024-04-25 23:10:49,360 DEBUG                       {
2024-04-25 23:10:49,360 DEBUG                           "lastProbeTime": null,
2024-04-25 23:10:49,361 DEBUG                           "lastTransitionTime": "2024-04-23T22:10:23Z",
2024-04-25 23:10:49,361 DEBUG                           "status": "True",
2024-04-25 23:10:49,361 DEBUG                           "type": "ContainersReady"
2024-04-25 23:10:49,361 DEBUG                       },
2024-04-25 23:10:49,361 DEBUG                       {
2024-04-25 23:10:49,361 DEBUG                           "lastProbeTime": null,
2024-04-25 23:10:49,361 DEBUG                           "lastTransitionTime": "2024-04-23T22:09:39Z",
2024-04-25 23:10:49,361 DEBUG                           "status": "True",
2024-04-25 23:10:49,361 DEBUG                           "type": "PodScheduled"
2024-04-25 23:10:49,361 DEBUG                       }
2024-04-25 23:10:49,361 DEBUG                   ],
2024-04-25 23:10:49,361 DEBUG                   "containerStatuses": [
2024-04-25 23:10:49,361 DEBUG                       {
2024-04-25 23:10:49,361 DEBUG                           "containerID": "containerd://016f21c417eb786d04cb8bc32574edcf7f3a424ce4ed7c314187375e9a1a857c",
2024-04-25 23:10:49,361 DEBUG                           "image": "quay.io/prometheus-operator/prometheus-config-reloader:v0.65.1",
2024-04-25 23:10:49,361 DEBUG                           "imageID": "quay.io/prometheus-operator/prometheus-config-reloader@sha256:ca8684fd1e35ae73c9d497ea37cf5216a32e88f1d0e2eeab22247b532f1e508f",
2024-04-25 23:10:49,361 DEBUG                           "lastState": {},
2024-04-25 23:10:49,361 DEBUG                           "name": "config-reloader",
2024-04-25 23:10:49,361 DEBUG                           "ready": true,
2024-04-25 23:10:49,361 DEBUG                           "restartCount": 0,
2024-04-25 23:10:49,361 DEBUG                           "started": true,
2024-04-25 23:10:49,361 DEBUG                           "state": {
2024-04-25 23:10:49,361 DEBUG                               "running": {
2024-04-25 23:10:49,361 DEBUG                                   "startedAt": "2024-04-23T22:10:04Z"
2024-04-25 23:10:49,361 DEBUG                               }
2024-04-25 23:10:49,361 DEBUG                           }
2024-04-25 23:10:49,361 DEBUG                       },
2024-04-25 23:10:49,361 DEBUG                       {
2024-04-25 23:10:49,361 DEBUG                           "containerID": "containerd://420872495a8beb3e2b22b4744f07af83f1a97b4c4e5a59ad45eb3fb3188127b6",
2024-04-25 23:10:49,361 DEBUG                           "image": "quay.io/prometheus/prometheus:v2.44.0",
2024-04-25 23:10:49,361 DEBUG                           "imageID": "quay.io/prometheus/prometheus@sha256:0f0b7feb6f02620df7d493ad7437b6ee95b6d16d8d18799f3607124e501444b1",
2024-04-25 23:10:49,361 DEBUG                           "lastState": {},
2024-04-25 23:10:49,361 DEBUG                           "name": "prometheus",
2024-04-25 23:10:49,361 DEBUG                           "ready": true,
2024-04-25 23:10:49,361 DEBUG                           "restartCount": 0,
2024-04-25 23:10:49,361 DEBUG                           "started": true,
2024-04-25 23:10:49,361 DEBUG                           "state": {
2024-04-25 23:10:49,361 DEBUG                               "running": {
2024-04-25 23:10:49,361 DEBUG                                   "startedAt": "2024-04-23T22:10:03Z"
2024-04-25 23:10:49,361 DEBUG                               }
2024-04-25 23:10:49,361 DEBUG                           }
2024-04-25 23:10:49,361 DEBUG                       }
2024-04-25 23:10:49,361 DEBUG                   ],
2024-04-25 23:10:49,361 DEBUG                   "hostIP": "10.142.0.65",
2024-04-25 23:10:49,361 DEBUG                   "initContainerStatuses": [
2024-04-25 23:10:49,361 DEBUG                       {
2024-04-25 23:10:49,361 DEBUG                           "containerID": "containerd://e7c488f28c76013a89955c6379f42d0386ea256cec12a628cda5bfab263dc274",
2024-04-25 23:10:49,361 DEBUG                           "image": "quay.io/prometheus-operator/prometheus-config-reloader:v0.65.1",
2024-04-25 23:10:49,362 DEBUG                           "imageID": "quay.io/prometheus-operator/prometheus-config-reloader@sha256:ca8684fd1e35ae73c9d497ea37cf5216a32e88f1d0e2eeab22247b532f1e508f",
2024-04-25 23:10:49,362 DEBUG                           "lastState": {},
2024-04-25 23:10:49,362 DEBUG                           "name": "init-config-reloader",
2024-04-25 23:10:49,362 DEBUG                           "ready": true,
2024-04-25 23:10:49,362 DEBUG                           "restartCount": 0,
2024-04-25 23:10:49,362 DEBUG                           "state": {
2024-04-25 23:10:49,362 DEBUG                               "terminated": {
2024-04-25 23:10:49,362 DEBUG                                   "containerID": "containerd://e7c488f28c76013a89955c6379f42d0386ea256cec12a628cda5bfab263dc274",
2024-04-25 23:10:49,362 DEBUG                                   "exitCode": 0,
2024-04-25 23:10:49,362 DEBUG                                   "finishedAt": "2024-04-23T22:09:55Z",
2024-04-25 23:10:49,362 DEBUG                                   "reason": "Completed",
2024-04-25 23:10:49,362 DEBUG                                   "startedAt": "2024-04-23T22:09:55Z"
2024-04-25 23:10:49,362 DEBUG                               }
2024-04-25 23:10:49,362 DEBUG                           }
2024-04-25 23:10:49,362 DEBUG                       }
2024-04-25 23:10:49,362 DEBUG                   ],
2024-04-25 23:10:49,362 DEBUG                   "phase": "Running",
2024-04-25 23:10:49,362 DEBUG                   "podIP": "10.106.44.133",
2024-04-25 23:10:49,362 DEBUG                   "podIPs": [
2024-04-25 23:10:49,362 DEBUG                       {
2024-04-25 23:10:49,362 DEBUG                           "ip": "10.106.44.133"
2024-04-25 23:10:49,362 DEBUG                       }
2024-04-25 23:10:49,362 DEBUG                   ],
2024-04-25 23:10:49,362 DEBUG                   "qosClass": "Burstable",
2024-04-25 23:10:49,362 DEBUG                   "startTime": "2024-04-23T22:09:39Z"
2024-04-25 23:10:49,362 DEBUG               }
2024-04-25 23:10:49,362 DEBUG           }
2024-04-25 23:10:49,362 DEBUG       ],
2024-04-25 23:10:49,362 DEBUG       "kind": "List",
2024-04-25 23:10:49,362 DEBUG       "metadata": {
2024-04-25 23:10:49,362 DEBUG           "resourceVersion": ""
2024-04-25 23:10:49,362 DEBUG       }
2024-04-25 23:10:49,362 DEBUG   }
2024-04-25 23:10:49,362 DEBUG   --- stderr ---
2024-04-25 23:10:49,362 DEBUG   
________________________________________________________________________________

[2024-04-25 23:10:49] Enable_prometheus_admin_api step2 : Enable the API if disabled
________________________________________________________________________________

[2024-04-25 23:10:49] - INFO: REST API is enabled: True
________________________________________________________________________________

[2024-04-25 23:10:49] Enable_prometheus_admin_api post : Post method
________________________________________________________________________________

Setting result to PASS
Task has been successfully stopped