==================================================================================================== ========================================= Pod describe ========================================= ==================================================================================================== Name: lodemon-7cb5b9456d-dskfn Namespace: xlou Priority: 0 Service Account: k8s-svc-acct-lodemon Node: gke-xlou-cdm-default-pool-1b0769c8-7vmj/10.142.0.111 Start Time: Sat, 20 Apr 2024 05:31:56 +0000 Labels: app=lodemon app.kubernetes.io/name=lodemon pod-template-hash=7cb5b9456d skaffold.dev/run-id=56ed7a74-8149-4668-ba47-160036e927cd Annotations: Status: Running IP: 10.106.42.189 IPs: IP: 10.106.42.189 Controlled By: ReplicaSet/lodemon-7cb5b9456d Containers: lodemon: Container ID: containerd://9af3ed5a51af3c9836865bf0940e4a4e28ece7b99e3c20f754756f72429be5d0 Image: gcr.io/engineeringpit/lodestar-images/lodestarbox:master-stable Image ID: gcr.io/engineeringpit/lodestar-images/lodestarbox@sha256:03f76e6326d713f4d73dc474e0e25fbbbaecc7c1d8c0590bfd94d68ac0d98c97 Port: 8080/TCP Host Port: 0/TCP Command: python3 Args: /lodestar/scripts/lodemon_run.py -W default State: Running Started: Sat, 20 Apr 2024 05:31:57 +0000 Ready: True Restart Count: 0 Limits: cpu: 2 memory: 2Gi Requests: cpu: 1 memory: 1Gi Liveness: exec [cat /tmp/lodemon_alive] delay=20s timeout=1s period=10s #success=1 #failure=3 Readiness: exec [cat /tmp/lodemon_alive] delay=20s timeout=1s period=10s #success=1 #failure=3 Environment: SKAFFOLD_PROFILE: medium Mounts: /lodestar/config/config.yaml from config (rw,path="config.yaml") /lodestar/config/deployments_to_monitor.yaml from deployments-config (rw,path="deployments_to_monitor.yaml") /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-jm78k (ro) Conditions: Type Status Initialized True Ready True ContainersReady True PodScheduled True Volumes: config: Type: ConfigMap (a volume populated by a ConfigMap) Name: lodemon-config Optional: false deployments-config: Type: ConfigMap (a volume populated by a ConfigMap) Name: lodemon-deployments Optional: false kube-api-access-jm78k: Type: Projected (a volume that contains injected data from multiple sources) TokenExpirationSeconds: 3607 ConfigMapName: kube-root-ca.crt ConfigMapOptional: DownwardAPI: true QoS Class: Burstable Node-Selectors: Tolerations: node.kubernetes.io/not-ready:NoExecute op=Exists for 300s node.kubernetes.io/unreachable:NoExecute op=Exists for 300s Events: Type Reason Age From Message ---- ------ ---- ---- ------- Normal Scheduled 41m default-scheduler Successfully assigned xlou/lodemon-7cb5b9456d-dskfn to gke-xlou-cdm-default-pool-1b0769c8-7vmj Normal Pulling 41m kubelet Pulling image "gcr.io/engineeringpit/lodestar-images/lodestarbox:master-stable" Normal Pulled 41m kubelet Successfully pulled image "gcr.io/engineeringpit/lodestar-images/lodestarbox:master-stable" in 352.914478ms (352.925716ms including waiting) Normal Created 41m kubelet Created container lodemon Normal Started 41m kubelet Started container lodemon ==================================================================================================== =========================================== Pod logs =========================================== ==================================================================================================== * Serving Flask app 'lodemon_run' * Debug mode: off WARNING: This is a development server. Do not use it in a production deployment. Use a production WSGI server instead. * Running on all addresses (0.0.0.0) * Running on http://127.0.0.1:8080 * Running on http://10.106.42.189:8080 Press CTRL+C to quit 06:33:03 INFO 06:33:03 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 06:33:03 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:33:03 INFO [loop_until]: OK (rc = 0) 06:33:03 DEBUG --- stdout --- 06:33:03 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 06:33:03 DEBUG --- stderr --- 06:33:03 DEBUG 06:33:03 INFO 06:33:03 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 06:33:03 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:33:03 INFO [loop_until]: OK (rc = 0) 06:33:03 DEBUG --- stdout --- 06:33:03 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 06:33:03 DEBUG --- stderr --- 06:33:03 DEBUG 06:33:03 INFO 06:33:03 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 06:33:03 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:33:04 INFO [loop_until]: OK (rc = 0) 06:33:04 DEBUG --- stdout --- 06:33:04 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 06:33:04 DEBUG --- stderr --- 06:33:04 DEBUG 06:33:04 INFO 06:33:04 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 06:33:04 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:33:04 INFO [loop_until]: OK (rc = 0) 06:33:04 DEBUG --- stdout --- 06:33:04 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 06:33:04 DEBUG --- stderr --- 06:33:04 DEBUG 06:33:04 INFO 06:33:04 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 06:33:04 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:33:04 INFO [loop_until]: OK (rc = 0) 06:33:04 DEBUG --- stdout --- 06:33:04 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 06:33:04 DEBUG --- stderr --- 06:33:04 DEBUG 06:33:04 INFO 06:33:04 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 06:33:04 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:33:04 INFO [loop_until]: OK (rc = 0) 06:33:04 DEBUG --- stdout --- 06:33:04 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 06:33:04 DEBUG --- stderr --- 06:33:04 DEBUG 06:33:04 INFO 06:33:04 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 06:33:04 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:33:04 INFO [loop_until]: OK (rc = 0) 06:33:04 DEBUG --- stdout --- 06:33:04 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 06:33:04 DEBUG --- stderr --- 06:33:04 DEBUG 06:33:04 INFO 06:33:04 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 06:33:04 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:33:04 INFO [loop_until]: OK (rc = 0) 06:33:04 DEBUG --- stdout --- 06:33:04 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 06:33:04 DEBUG --- stderr --- 06:33:04 DEBUG 06:33:04 INFO 06:33:04 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 06:33:04 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:33:04 INFO [loop_until]: OK (rc = 0) 06:33:04 DEBUG --- stdout --- 06:33:04 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 06:33:04 DEBUG --- stderr --- 06:33:04 DEBUG 06:33:04 INFO 06:33:04 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 06:33:04 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:33:05 INFO [loop_until]: OK (rc = 0) 06:33:05 DEBUG --- stdout --- 06:33:05 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 06:33:05 DEBUG --- stderr --- 06:33:05 DEBUG 06:33:05 INFO 06:33:05 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 06:33:05 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:33:05 INFO [loop_until]: OK (rc = 0) 06:33:05 DEBUG --- stdout --- 06:33:05 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 06:33:05 DEBUG --- stderr --- 06:33:05 DEBUG 06:33:05 INFO 06:33:05 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 06:33:05 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:33:05 INFO [loop_until]: OK (rc = 0) 06:33:05 DEBUG --- stdout --- 06:33:05 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 06:33:05 DEBUG --- stderr --- 06:33:05 DEBUG 06:33:05 INFO 06:33:05 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 06:33:05 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:33:05 INFO [loop_until]: OK (rc = 0) 06:33:05 DEBUG --- stdout --- 06:33:05 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 06:33:05 DEBUG --- stderr --- 06:33:05 DEBUG 06:33:05 INFO 06:33:05 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 06:33:05 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:33:05 INFO [loop_until]: OK (rc = 0) 06:33:05 DEBUG --- stdout --- 06:33:05 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 06:33:05 DEBUG --- stderr --- 06:33:05 DEBUG 06:33:05 INFO 06:33:05 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 06:33:05 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:33:05 INFO [loop_until]: OK (rc = 0) 06:33:05 DEBUG --- stdout --- 06:33:05 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 06:33:05 DEBUG --- stderr --- 06:33:05 DEBUG 06:33:05 INFO 06:33:05 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 06:33:05 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:33:05 INFO [loop_until]: OK (rc = 0) 06:33:05 DEBUG --- stdout --- 06:33:05 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 06:33:05 DEBUG --- stderr --- 06:33:05 DEBUG 06:33:05 INFO 06:33:05 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 06:33:05 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:33:05 INFO [loop_until]: OK (rc = 0) 06:33:05 DEBUG --- stdout --- 06:33:05 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 06:33:05 DEBUG --- stderr --- 06:33:05 DEBUG 06:33:06 INFO 06:33:06 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 06:33:06 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:33:06 INFO [loop_until]: OK (rc = 0) 06:33:06 DEBUG --- stdout --- 06:33:06 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 06:33:06 DEBUG --- stderr --- 06:33:06 DEBUG 06:33:06 INFO 06:33:06 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 06:33:06 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:33:06 INFO [loop_until]: OK (rc = 0) 06:33:06 DEBUG --- stdout --- 06:33:06 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 06:33:06 DEBUG --- stderr --- 06:33:06 DEBUG 06:33:06 INFO 06:33:06 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 06:33:06 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:33:06 INFO [loop_until]: OK (rc = 0) 06:33:06 DEBUG --- stdout --- 06:33:06 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 06:33:06 DEBUG --- stderr --- 06:33:06 DEBUG 06:33:06 INFO 06:33:06 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 06:33:06 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:33:06 INFO [loop_until]: OK (rc = 0) 06:33:06 DEBUG --- stdout --- 06:33:06 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 06:33:06 DEBUG --- stderr --- 06:33:06 DEBUG 06:33:06 INFO 06:33:06 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 06:33:06 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:33:06 INFO [loop_until]: OK (rc = 0) 06:33:06 DEBUG --- stdout --- 06:33:06 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 06:33:06 DEBUG --- stderr --- 06:33:06 DEBUG 06:33:06 INFO 06:33:06 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 06:33:06 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:33:06 INFO [loop_until]: OK (rc = 0) 06:33:06 DEBUG --- stdout --- 06:33:06 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 06:33:06 DEBUG --- stderr --- 06:33:06 DEBUG 06:33:06 INFO 06:33:06 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 06:33:06 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:33:06 INFO [loop_until]: OK (rc = 0) 06:33:06 DEBUG --- stdout --- 06:33:06 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 06:33:06 DEBUG --- stderr --- 06:33:06 DEBUG 06:33:06 INFO 06:33:06 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 06:33:06 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:33:07 INFO [loop_until]: OK (rc = 0) 06:33:07 DEBUG --- stdout --- 06:33:07 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 06:33:07 DEBUG --- stderr --- 06:33:07 DEBUG 06:33:07 INFO 06:33:07 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 06:33:07 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:33:07 INFO [loop_until]: OK (rc = 0) 06:33:07 DEBUG --- stdout --- 06:33:07 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 06:33:07 DEBUG --- stderr --- 06:33:07 DEBUG 06:33:07 INFO 06:33:07 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 06:33:07 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:33:07 INFO [loop_until]: OK (rc = 0) 06:33:07 DEBUG --- stdout --- 06:33:07 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 06:33:07 DEBUG --- stderr --- 06:33:07 DEBUG 06:33:07 INFO Initializing monitoring instance threads 06:33:07 DEBUG Monitoring instance thread list: [, , , , , , , , , , , , , , , , , , , , , , , , , , , , ] 06:33:07 INFO Starting instance threads 06:33:07 INFO 06:33:07 INFO Thread started 06:33:07 INFO [loop_until]: kubectl --namespace=xlou top node 06:33:07 INFO 06:33:07 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:33:07 INFO Thread started 06:33:07 INFO [loop_until]: kubectl --namespace=xlou top pods 06:33:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591187" 06:33:07 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:33:07 INFO Thread started 06:33:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591187" 06:33:07 INFO Thread started 06:33:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591187" 06:33:07 INFO Thread started 06:33:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591187" 06:33:07 INFO Thread started 06:33:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713591187" 06:33:07 INFO Thread started 06:33:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713591187" 06:33:07 INFO Thread started 06:33:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713591187" 06:33:07 INFO Thread started 06:33:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713591187" 06:33:07 INFO Thread started 06:33:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713591187" 06:33:07 INFO Thread started 06:33:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591187" 06:33:07 INFO Thread started 06:33:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591187" 06:33:07 INFO Thread started 06:33:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591187" 06:33:07 INFO Thread started 06:33:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591187" 06:33:07 INFO Thread started 06:33:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591187" 06:33:07 INFO Thread started 06:33:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591187" 06:33:07 INFO Thread started 06:33:07 INFO [http_cmd]: http status code OK 06:33:07 INFO [http_cmd]: http status code OK 06:33:07 INFO [http_cmd]: http status code OK 06:33:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591187" 06:33:07 INFO [http_cmd]: http status code OK 06:33:07 INFO [http_cmd]: http status code OK 06:33:07 INFO [http_cmd]: http status code OK 06:33:07 INFO [http_cmd]: http status code OK 06:33:07 DEBUG --- status code --- 06:33:07 INFO [http_cmd]: http status code OK 06:33:07 INFO Thread started 06:33:07 INFO [http_cmd]: http status code OK 06:33:07 DEBUG --- status code --- 06:33:07 INFO [http_cmd]: http status code OK 06:33:07 DEBUG --- status code --- 06:33:07 DEBUG --- status code --- 06:33:07 DEBUG --- status code --- 06:33:07 INFO [http_cmd]: http status code OK 06:33:07 INFO [http_cmd]: http status code OK 06:33:07 DEBUG --- status code --- 06:33:07 DEBUG --- status code --- 06:33:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:33:07 DEBUG --- status code --- 06:33:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591187" 06:33:07 DEBUG --- status code --- 06:33:07 INFO Thread started 06:33:07 INFO [http_cmd]: http status code OK 06:33:07 INFO [http_cmd]: http status code OK 06:33:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:33:07 DEBUG --- status code --- 06:33:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:33:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:33:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:33:07 DEBUG --- status code --- 06:33:07 DEBUG --- status code --- 06:33:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:33:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:33:07 DEBUG --- http response --- 06:33:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:33:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:33:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591187" 06:33:07 INFO Thread started 06:33:07 DEBUG --- status code --- 06:33:07 DEBUG --- status code --- 06:33:07 DEBUG --- http response --- 06:33:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:33:07 DEBUG --- http response --- 06:33:07 DEBUG --- http response --- 06:33:07 DEBUG --- http response --- 06:33:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:33:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:33:07 DEBUG --- http response --- 06:33:07 DEBUG --- http response --- 06:33:07 DEBUG --- http response --- 06:33:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:33:07 INFO [http_cmd]: http status code OK 06:33:07 DEBUG --- http response --- 06:33:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591187" 06:33:07 INFO [http_cmd]: http status code OK 06:33:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:33:07 INFO Thread started 06:33:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:33:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 06:33:07 DEBUG --- http response --- 06:33:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:33:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:33:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:33:07 DEBUG --- http response --- 06:33:07 INFO [loop_until]: OK (rc = 0) 06:33:07 DEBUG --- http response --- 06:33:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:33:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:33:07 INFO [http_cmd]: http status code OK 06:33:07 INFO [http_cmd]: http status code OK 06:33:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 06:33:07 DEBUG ----- output ----- 06:33:07 DEBUG --- status code --- 06:33:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:33:07 DEBUG --- status code --- 06:33:07 DEBUG --- http response --- 06:33:07 DEBUG --- http response --- 06:33:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591187" 06:33:07 INFO Thread started 06:33:07 DEBUG ----- output ----- 06:33:07 DEBUG ----- output ----- 06:33:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:33:07 DEBUG ----- output ----- 06:33:07 DEBUG ----- output ----- 06:33:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:33:07 INFO [loop_until]: OK (rc = 0) 06:33:07 DEBUG --- stdout --- 06:33:07 DEBUG ----- output ----- 06:33:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:33:07 DEBUG ----- output ----- 06:33:07 DEBUG --- status code --- 06:33:07 DEBUG --- status code --- 06:33:07 DEBUG ----- output ----- 06:33:07 DEBUG { 06:33:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:33:07 DEBUG ----- output ----- 06:33:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:33:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:33:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:33:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591187" 06:33:07 INFO Thread started 06:33:07 DEBUG { 06:33:07 DEBUG { 06:33:07 DEBUG ----- output ----- 06:33:07 DEBUG { 06:33:07 DEBUG { 06:33:07 DEBUG ----- output ----- 06:33:07 DEBUG --- stdout --- 06:33:07 DEBUG { 06:33:07 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 6m 8600Mi am-767f6b67b5-qcqz2 7m 8657Mi am-767f6b67b5-vbltz 8m 8603Mi ds-cts-0 13m 10613Mi ds-cts-1 12m 10702Mi ds-cts-2 11m 10503Mi ds-idrepo-0 15m 3720Mi ds-idrepo-1 12m 3730Mi ds-idrepo-2 13m 3698Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1177Mi idm-b85f9d978-wnhxb 3m 1237Mi lodemon-7cb5b9456d-dskfn 2m 55Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 1m 1886Mi 06:33:07 DEBUG ----- output ----- 06:33:07 DEBUG { 06:33:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:33:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:33:07 DEBUG { 06:33:07 DEBUG "status": "success", 06:33:07 DEBUG --- http response --- 06:33:07 DEBUG { 06:33:07 DEBUG --- http response --- 06:33:07 DEBUG ----- output ----- 06:33:07 DEBUG ----- output ----- 06:33:07 DEBUG "status": "success", 06:33:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713591187" 06:33:07 DEBUG "status": "success", 06:33:07 INFO Thread started 06:33:07 DEBUG { 06:33:07 DEBUG "status": "success", 06:33:07 DEBUG "status": "success", 06:33:07 DEBUG { 06:33:07 DEBUG "status": "success", 06:33:07 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 73m 0% 6275Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 70m 0% 7362Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 51m 0% 11287Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 82m 0% 3381Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 69m 0% 13737Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 163m 0% 5187Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 59m 0% 11501Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 78m 0% 3189Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 60m 0% 10799Mi 9% gke-xlou-cdm-ds-0f601751-j488 68m 0% 6977Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 72m 0% 15728Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 92m 0% 7602Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 67m 0% 13072Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 65m 0% 5766Mi 9% 06:33:07 DEBUG --- stderr --- 06:33:07 DEBUG { 06:33:07 DEBUG "status": "success", 06:33:07 DEBUG --- http response --- 06:33:07 DEBUG --- http response --- 06:33:07 DEBUG "status": "success", 06:33:07 DEBUG "data": { 06:33:07 DEBUG "status": "success", 06:33:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:33:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:33:07 DEBUG { 06:33:07 DEBUG { 06:33:07 DEBUG "data": { 06:33:07 INFO [http_cmd]: http status code OK 06:33:07 DEBUG --- status code --- 06:33:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713591187" 06:33:07 DEBUG "status": "success", 06:33:07 INFO Thread started 06:33:07 DEBUG "data": { 06:33:07 DEBUG "data": { 06:33:07 DEBUG "status": "success", 06:33:07 DEBUG "data": { 06:33:07 DEBUG --- stderr --- 06:33:07 DEBUG 06:33:07 DEBUG "status": "success", 06:33:07 DEBUG "data": { 06:33:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:33:07 DEBUG "data": { 06:33:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:33:07 DEBUG "resultType": "vector", 06:33:07 DEBUG "data": { 06:33:07 DEBUG ----- output ----- 06:33:07 DEBUG ----- output ----- 06:33:07 DEBUG "status": "success", 06:33:07 DEBUG "status": "success", 06:33:07 DEBUG "resultType": "vector", 06:33:07 INFO [http_cmd]: http status code OK 06:33:07 DEBUG "data": { 06:33:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:33:07 INFO [http_cmd]: http status code OK 06:33:07 DEBUG "data": { 06:33:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713591187" 06:33:07 DEBUG "resultType": "vector", 06:33:07 INFO Thread started 06:33:07 DEBUG "resultType": "vector", 06:33:07 DEBUG "data": { 06:33:07 DEBUG "resultType": "vector", 06:33:07 DEBUG 06:33:07 DEBUG "data": { 06:33:07 DEBUG "resultType": "vector", 06:33:07 DEBUG ----- output ----- 06:33:07 DEBUG "resultType": "vector", 06:33:07 DEBUG ----- output ----- 06:33:07 INFO [http_cmd]: http status code OK 06:33:07 DEBUG "result": [ 06:33:07 DEBUG "resultType": "vector", 06:33:07 DEBUG { 06:33:07 DEBUG { 06:33:07 DEBUG "data": { 06:33:07 DEBUG "data": { 06:33:07 DEBUG "result": [ 06:33:07 DEBUG --- status code --- 06:33:07 DEBUG "resultType": "vector", 06:33:07 DEBUG --- http response --- 06:33:07 DEBUG --- status code --- 06:33:07 DEBUG "resultType": "vector", 06:33:07 DEBUG "result": [ 06:33:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713591187" 06:33:07 DEBUG "result": [ 06:33:07 INFO Thread started 06:33:07 DEBUG "resultType": "vector", 06:33:07 DEBUG "result": [ 06:33:07 DEBUG "resultType": "vector", 06:33:07 DEBUG "result": [ 06:33:07 DEBUG { 06:33:07 DEBUG "result": [ 06:33:07 DEBUG { 06:33:07 DEBUG --- status code --- 06:33:07 DEBUG { 06:33:07 DEBUG "result": [ 06:33:07 DEBUG "status": "success", 06:33:07 DEBUG "status": "success", 06:33:07 DEBUG "resultType": "vector", 06:33:07 DEBUG "resultType": "vector", 06:33:07 DEBUG { 06:33:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:33:07 DEBUG "result": [ 06:33:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:33:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:33:07 DEBUG "result": [ 06:33:07 DEBUG { 06:33:07 DEBUG { 06:33:07 DEBUG "metric": { 06:33:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591187" 06:33:07 DEBUG { 06:33:07 INFO Thread started 06:33:07 DEBUG "result": [ 06:33:07 DEBUG { 06:33:07 DEBUG "status": "success", 06:33:07 DEBUG { 06:33:07 DEBUG "status": "success", 06:33:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:33:07 DEBUG "metric": { 06:33:07 DEBUG { 06:33:07 DEBUG "data": { 06:33:07 DEBUG "data": { 06:33:07 DEBUG "result": [ 06:33:07 DEBUG "result": [ 06:33:07 DEBUG "metric": { 06:33:07 DEBUG --- http response --- 06:33:07 DEBUG { 06:33:07 DEBUG --- http response --- 06:33:07 DEBUG ----- output ----- 06:33:07 INFO [http_cmd]: http status code OK 06:33:07 DEBUG { 06:33:07 DEBUG "metric": { 06:33:07 DEBUG "result": [ 06:33:07 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 06:33:07 DEBUG "metric": { 06:33:07 DEBUG { 06:33:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713591187" 06:33:07 DEBUG "metric": { 06:33:07 INFO Thread started 06:33:07 DEBUG "data": { 06:33:07 DEBUG "metric": { 06:33:07 DEBUG "data": { 06:33:07 DEBUG --- http response --- 06:33:07 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:33:07 DEBUG "metric": { 06:33:07 DEBUG "resultType": "vector", 06:33:07 DEBUG "resultType": "vector", 06:33:07 DEBUG { 06:33:07 DEBUG { 06:33:07 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:33:07 DEBUG "metric": { 06:33:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:33:07 ERROR [http_cmd]: ERROR 06:33:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:33:07 DEBUG { 06:33:07 DEBUG --- status code --- 06:33:07 DEBUG "metric": { 06:33:07 DEBUG "instance": "10.142.0.107:9100" 06:33:07 DEBUG { 06:33:07 DEBUG }, 06:33:07 DEBUG "pod": "am-767f6b67b5-9zw22" 06:33:07 DEBUG "metric": { 06:33:07 DEBUG "instance": "10.142.0.107:9100" 06:33:07 INFO All threads has been started 127.0.0.1 - - [20/Apr/2024 06:33:07] "GET /monitoring/start HTTP/1.1" 200 - 06:33:07 INFO [http_cmd]: http status code OK 06:33:07 DEBUG "resultType": "vector", 06:33:07 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:33:07 DEBUG "resultType": "vector", 06:33:07 DEBUG }, 06:33:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:33:07 DEBUG "pod": "am-767f6b67b5-9zw22" 06:33:07 DEBUG "result": [ 06:33:07 DEBUG "result": [ 06:33:07 DEBUG "metric": { 06:33:07 DEBUG "metric": { 06:33:07 DEBUG }, 06:33:07 DEBUG "instance": "10.142.0.107:9100" 06:33:07 DEBUG ----- output ----- 06:33:07 ERROR --- status code --- 06:33:07 DEBUG ----- output ----- 06:33:07 DEBUG "status": "success", 06:33:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:33:07 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 06:33:07 DEBUG }, 06:33:07 DEBUG "metric": { 06:33:07 DEBUG "value": [ 06:33:07 DEBUG }, 06:33:07 DEBUG "pod": "ds-idrepo-0" 06:33:07 DEBUG }, 06:33:07 DEBUG --- status code --- 06:33:07 DEBUG "result": [ 06:33:07 DEBUG }, 06:33:07 DEBUG "result": [ 06:33:07 DEBUG "value": [ 06:33:07 DEBUG ----- output ----- 06:33:07 DEBUG }, 06:33:07 DEBUG { 06:33:07 DEBUG { 06:33:07 DEBUG "pod": "ds-idrepo-0" 06:33:07 DEBUG "pod": "ds-cts-0" 06:33:07 DEBUG "value": [ 06:33:07 DEBUG }, 06:33:07 INFO [http_cmd]: http status code OK 06:33:07 DEBUG { 06:33:07 ERROR http status code is 400 (expected [200, 401, 403, 404]) 06:33:07 DEBUG { 06:33:07 DEBUG "data": { 06:33:07 DEBUG --- http response --- 06:33:07 DEBUG }, 06:33:07 DEBUG "value": [ 06:33:07 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:33:07 DEBUG 06:33:07 DEBUG "value": [ 06:33:07 DEBUG }, 06:33:07 DEBUG "value": [ 06:33:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:33:07 DEBUG { 06:33:07 DEBUG "value": [ 06:33:07 DEBUG { 06:33:07 DEBUG 06:33:07 DEBUG { 06:33:07 DEBUG "value": [ 06:33:07 DEBUG "metric": { 06:33:07 DEBUG "metric": { 06:33:07 DEBUG }, 06:33:07 DEBUG }, 06:33:07 DEBUG 06:33:07 DEBUG "value": [ 06:33:07 DEBUG --- status code --- 06:33:07 DEBUG "status": "success", 06:33:07 ERROR --- http response --- 06:33:07 DEBUG "status": "success", 06:33:07 DEBUG "resultType": "vector", 06:33:07 DEBUG "value": [ 06:33:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:33:07 DEBUG 06:33:07 DEBUG }, 06:33:07 DEBUG 06:33:07 DEBUG 06:33:07 DEBUG "value": [ 06:33:07 DEBUG 06:33:07 DEBUG --- http response --- 06:33:07 DEBUG "metric": { 06:33:07 DEBUG 06:33:07 DEBUG "metric": { 06:33:07 DEBUG 06:33:07 DEBUG "status": "success", 06:33:07 DEBUG 06:33:07 DEBUG "pod": "ds-idrepo-0" 06:33:07 DEBUG "pod": "am-767f6b67b5-9zw22" 06:33:07 DEBUG }, 06:33:07 DEBUG "value": [ 06:33:07 DEBUG 06:33:07 DEBUG 06:33:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:33:07 DEBUG "data": { 06:33:07 DEBUG "data": { 06:33:07 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 06:33:07 DEBUG "result": [ 06:33:07 DEBUG 06:33:07 DEBUG ----- output ----- 06:33:07 DEBUG 06:33:07 DEBUG "value": [ 06:33:07 DEBUG 06:33:07 DEBUG 06:33:07 DEBUG 06:33:07 DEBUG 06:33:07 DEBUG "pod": "am-767f6b67b5-9zw22" 06:33:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:33:07 DEBUG 06:33:07 DEBUG "pod": "am-767f6b67b5-9zw22" 06:33:07 DEBUG 06:33:07 DEBUG "data": { 06:33:07 DEBUG 06:33:07 DEBUG }, 06:33:07 DEBUG "value": [ 06:33:07 DEBUG "value": [ 06:33:07 INFO [http_cmd]: http status code OK 06:33:07 DEBUG 06:33:07 DEBUG 06:33:07 DEBUG 06:33:07 DEBUG --- http response --- 06:33:07 DEBUG "resultType": "vector", 06:33:07 DEBUG "resultType": "vector", 06:33:07 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 06:33:07 DEBUG { 06:33:07 DEBUG 06:33:07 DEBUG { 06:33:07 DEBUG 06:33:07 DEBUG 06:33:07 DEBUG [...] 06:33:07 DEBUG 06:33:07 DEBUG 06:33:07 DEBUG 06:33:07 DEBUG }, 06:33:07 DEBUG ----- output ----- 06:33:07 DEBUG 06:33:07 DEBUG }, 06:33:07 DEBUG [...] 06:33:07 DEBUG "resultType": "vector", 06:33:07 DEBUG 06:33:07 DEBUG "value": [ 06:33:07 DEBUG 06:33:07 DEBUG 06:33:07 DEBUG --- status code --- 06:33:07 DEBUG 06:33:07 DEBUG [...] 06:33:07 DEBUG 06:33:07 DEBUG "result": [ 06:33:07 DEBUG "result": [ 06:33:07 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 06:33:07 DEBUG "metric": { 06:33:07 DEBUG 06:33:07 DEBUG "status": "success", 06:33:07 DEBUG [...] 06:33:07 DEBUG 06:33:07 DEBUG 06:33:07 DEBUG [...] 06:33:07 DEBUG 06:33:07 DEBUG [...] 06:33:07 DEBUG "value": [ 06:33:07 DEBUG { 06:33:07 DEBUG [...] 06:33:07 DEBUG 06:33:07 DEBUG 06:33:07 DEBUG "result": [ 06:33:07 DEBUG [...] 06:33:07 DEBUG 06:33:07 DEBUG 06:33:07 DEBUG 06:33:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:33:07 DEBUG 06:33:07 DEBUG 06:33:07 DEBUG [...] 06:33:07 DEBUG { 06:33:07 DEBUG { 06:33:07 DEBUG "pod": "ds-cts-0" 06:33:07 DEBUG [...] 06:33:07 DEBUG "data": { 06:33:07 DEBUG 06:33:07 DEBUG 06:33:07 DEBUG 06:33:07 DEBUG 06:33:07 DEBUG [...] 06:33:07 DEBUG 06:33:07 DEBUG 06:33:07 DEBUG "status": "success", 06:33:07 DEBUG "value": [ 06:33:07 DEBUG 06:33:07 DEBUG 06:33:07 DEBUG { 06:33:07 DEBUG 06:33:07 DEBUG 06:33:07 DEBUG 06:33:07 DEBUG 06:33:07 DEBUG --- http response --- 06:33:07 DEBUG [...] 06:33:07 DEBUG 06:33:07 DEBUG 06:33:07 DEBUG "metric": { 06:33:07 DEBUG "metric": { 06:33:07 DEBUG }, 06:33:07 DEBUG "value": [ 06:33:07 DEBUG "resultType": "vector", 06:33:07 DEBUG 06:33:07 DEBUG 06:33:07 DEBUG 06:33:07 DEBUG 06:33:07 DEBUG 06:33:07 DEBUG 06:33:07 DEBUG 06:33:07 DEBUG "data": { 06:33:07 DEBUG 06:33:07 DEBUG 06:33:07 DEBUG 06:33:07 DEBUG "metric": { 06:33:07 DEBUG 06:33:07 DEBUG 06:33:07 DEBUG 06:33:07 DEBUG [...] 06:33:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:33:07 DEBUG 06:33:07 DEBUG 06:33:07 DEBUG "pod": "lodemon-799cdc5bdb-7t5x7" 06:33:07 DEBUG 06:33:07 DEBUG "pod": "ds-cts-0" 06:33:07 DEBUG 06:33:07 DEBUG 06:33:07 DEBUG "result": [ 06:33:07 DEBUG [...] 06:33:07 DEBUG "instance": "10.142.0.43:9100" 06:33:07 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 06:33:07 DEBUG 06:33:07 DEBUG 06:33:07 DEBUG 06:33:07 DEBUG 06:33:07 DEBUG "resultType": "vector", 06:33:07 DEBUG 06:33:07 DEBUG "pod": "lodemon-799cdc5bdb-7t5x7" 06:33:07 DEBUG }, 06:33:07 DEBUG "pod": "am-767f6b67b5-9zw22" 06:33:07 DEBUG [...] 06:33:07 DEBUG "pod": "am-767f6b67b5-vbltz" 06:33:07 DEBUG [...] 06:33:07 DEBUG 06:33:07 DEBUG ----- output ----- 06:33:07 DEBUG 06:33:07 DEBUG "pod": "am-767f6b67b5-9zw22" 06:33:07 DEBUG }, 06:33:07 DEBUG 06:33:07 DEBUG }, 06:33:07 DEBUG 06:33:07 DEBUG 06:33:07 DEBUG { 06:33:07 DEBUG 06:33:07 DEBUG }, 06:33:07 DEBUG }, 06:33:07 DEBUG "pod": "am-767f6b67b5-vbltz" 06:33:07 DEBUG 06:33:07 DEBUG "pod": "ds-idrepo-2" 06:33:07 DEBUG [...] 06:33:07 DEBUG "result": [ 06:33:07 DEBUG 06:33:07 DEBUG "pod": "idm-b85f9d978-wnhxb" 06:33:07 DEBUG "value": [ 06:33:07 DEBUG }, 06:33:07 DEBUG 06:33:07 DEBUG }, 06:33:07 DEBUG "value": [ 06:33:07 DEBUG 06:33:07 DEBUG { 06:33:07 DEBUG 06:33:07 DEBUG }, 06:33:07 DEBUG "value": [ 06:33:07 DEBUG "instance": "10.142.0.43:9100" 06:33:07 DEBUG "value": [ 06:33:07 DEBUG 06:33:07 DEBUG 06:33:07 DEBUG "metric": { 06:33:07 DEBUG 06:33:07 DEBUG "value": [ 06:33:07 DEBUG "value": [ 06:33:07 DEBUG }, 06:33:07 DEBUG "instance": "10.142.0.43:9100" 06:33:07 DEBUG }, 06:33:07 DEBUG 06:33:07 DEBUG { 06:33:07 DEBUG [...] 06:33:07 DEBUG }, 06:33:07 DEBUG 1713591187, 06:33:07 DEBUG "value": [ 06:33:07 DEBUG 06:33:07 DEBUG 06:33:07 DEBUG 1713591187, 06:33:07 DEBUG 06:33:07 DEBUG "status": "success", 06:33:07 DEBUG "pod": "ds-cts-2" 06:33:07 DEBUG "value": [ 06:33:07 DEBUG 1713591187, 06:33:07 DEBUG }, 06:33:07 DEBUG 06:33:07 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 06:33:07 DEBUG [...] 06:33:07 DEBUG "pod": "ds-idrepo-0" 06:33:07 DEBUG 06:33:07 DEBUG 1713591187, 06:33:07 DEBUG "33985.73333333334" 06:33:07 DEBUG ] 06:33:07 DEBUG } 06:33:07 DEBUG "value": [ 06:33:07 DEBUG 06:33:07 DEBUG "metric": { 06:33:07 DEBUG 06:33:07 DEBUG "value": [ 06:33:07 DEBUG "0" 06:33:07 DEBUG 06:33:07 DEBUG 06:33:07 DEBUG 06:33:07 DEBUG "0" 06:33:07 DEBUG "pod": "am-767f6b67b5-vbltz" 06:33:07 DEBUG "data": { 06:33:07 DEBUG }, 06:33:07 DEBUG 06:33:07 DEBUG "0" 06:33:07 DEBUG "value": [ 06:33:07 DEBUG 06:33:07 DEBUG 06:33:07 DEBUG }, 06:33:07 DEBUG }, 06:33:07 DEBUG "pod": "idm-b85f9d978-wnhxb" 06:33:07 DEBUG "value": [ 06:33:07 DEBUG 1713591187, 06:33:07 DEBUG "0.02891310029635641" 06:33:07 DEBUG }, 06:33:07 DEBUG 1713591187, 06:33:07 DEBUG "0" 06:33:07 DEBUG "instance": "10.142.0.107:9100" 06:33:07 DEBUG 06:33:07 DEBUG 1713591187, 06:33:07 DEBUG ] 06:33:07 DEBUG 06:33:07 DEBUG "pod": "ds-idrepo-2" 06:33:07 DEBUG 06:33:07 DEBUG ] 06:33:07 DEBUG } 06:33:07 DEBUG "resultType": "vector", 06:33:07 DEBUG "value": [ 06:33:07 DEBUG 06:33:07 DEBUG ] 06:33:07 DEBUG 1713591187, 06:33:07 DEBUG 06:33:07 DEBUG [...] 06:33:07 DEBUG "value": [ 06:33:07 DEBUG "value": [ 06:33:07 DEBUG }, 06:33:07 DEBUG 1713591187, 06:33:07 DEBUG ] 06:33:07 DEBUG ] 06:33:07 DEBUG "value": [ 06:33:07 DEBUG 06:33:07 DEBUG ] 06:33:07 DEBUG }, 06:33:07 DEBUG 06:33:07 DEBUG "0" 06:33:07 DEBUG } 06:33:07 DEBUG 06:33:07 DEBUG }, 06:33:07 DEBUG "pod": "ds-idrepo-2" 06:33:07 DEBUG }, 06:33:07 DEBUG ] 06:33:07 DEBUG } 06:33:07 DEBUG 1713591187, 06:33:07 DEBUG 06:33:07 DEBUG } 06:33:07 DEBUG "0.0003333333333330302" 06:33:07 DEBUG 06:33:07 DEBUG 06:33:07 DEBUG 1713591187, 06:33:07 DEBUG 06:33:07 DEBUG "value": [ 06:33:07 DEBUG "0" 06:33:07 DEBUG ] 06:33:07 DEBUG } 06:33:07 DEBUG 1713591187, 06:33:07 DEBUG "pod": "am-767f6b67b5-vbltz" 06:33:07 DEBUG } 06:33:07 DEBUG "value": [ 06:33:07 DEBUG "pod": "am-767f6b67b5-vbltz" 06:33:07 DEBUG ] 06:33:07 DEBUG ] 06:33:07 DEBUG [...] 06:33:07 DEBUG "value": [ 06:33:07 DEBUG }, 06:33:07 DEBUG "value": [ 06:33:07 DEBUG "result": [ 06:33:07 DEBUG } 06:33:07 DEBUG "0" 06:33:07 DEBUG [...] 06:33:07 DEBUG ] 06:33:07 DEBUG ] 06:33:07 DEBUG 06:33:07 DEBUG 06:33:07 DEBUG "2074005504" 06:33:07 DEBUG 06:33:07 DEBUG 1713591187, 06:33:07 DEBUG } 06:33:07 DEBUG } 06:33:07 DEBUG ] 06:33:07 DEBUG "26426.5" 06:33:07 DEBUG }, 06:33:07 DEBUG ] 06:33:07 DEBUG 06:33:07 DEBUG }, 06:33:07 DEBUG } 06:33:07 DEBUG } 06:33:07 DEBUG 06:33:07 DEBUG 1713591187, 06:33:07 DEBUG "value": [ 06:33:07 DEBUG 1713591187, 06:33:07 DEBUG { 06:33:07 DEBUG ] 06:33:07 DEBUG 06:33:07 DEBUG } 06:33:07 DEBUG } 06:33:07 DEBUG "pod": "ds-cts-2" 06:33:07 DEBUG 06:33:07 DEBUG ] 06:33:07 DEBUG 06:33:07 DEBUG "587.7668161434977" 06:33:07 DEBUG } 06:33:07 DEBUG ] 06:33:07 DEBUG } 06:33:07 DEBUG } 06:33:07 DEBUG "value": [ 06:33:07 DEBUG } 06:33:07 DEBUG 06:33:07 DEBUG "value": [ 06:33:07 DEBUG ] 06:33:07 DEBUG } 06:33:07 DEBUG 06:33:07 DEBUG "0" 06:33:07 DEBUG 1713591187, 06:33:07 DEBUG "0" 06:33:07 DEBUG "metric": { 06:33:07 DEBUG } 06:33:07 DEBUG 06:33:07 DEBUG } 06:33:07 DEBUG ] 06:33:07 DEBUG }, 06:33:07 DEBUG "pod": "ds-cts-2" 06:33:07 DEBUG } 06:33:07 DEBUG [...] 06:33:07 DEBUG ] 06:33:07 DEBUG } 06:33:07 DEBUG ] 06:33:07 DEBUG 1713591187, 06:33:07 DEBUG } 06:33:07 DEBUG 06:33:07 DEBUG 1713591187, 06:33:07 DEBUG } 06:33:07 DEBUG 06:33:07 DEBUG ] 06:33:07 DEBUG "0" 06:33:07 DEBUG ] 06:33:07 DEBUG "instance": "10.142.0.107:9100" 06:33:07 DEBUG ] 06:33:07 DEBUG 06:33:07 DEBUG } 06:33:07 DEBUG "value": [ 06:33:07 DEBUG }, 06:33:07 DEBUG ] 06:33:07 DEBUG 06:33:07 DEBUG } 06:33:07 DEBUG } 06:33:07 DEBUG } 06:33:07 DEBUG "0" 06:33:07 DEBUG [...] 06:33:07 DEBUG "0" 06:33:07 DEBUG } 06:33:07 DEBUG "pod": "am-767f6b67b5-vbltz" 06:33:07 DEBUG } 06:33:07 DEBUG ] 06:33:07 DEBUG } 06:33:07 DEBUG }, 06:33:07 DEBUG } 06:33:07 DEBUG "pod": "am-767f6b67b5-vbltz" 06:33:07 DEBUG } 06:33:07 DEBUG 1713591187, 06:33:07 DEBUG "value": [ 06:33:07 DEBUG } 06:33:07 DEBUG 06:33:07 DEBUG ] 06:33:07 DEBUG ] 06:33:07 DEBUG ] 06:33:07 DEBUG 06:33:07 DEBUG ] 06:33:07 DEBUG }, 06:33:07 DEBUG ] 06:33:07 DEBUG } 06:33:07 DEBUG ] 06:33:07 DEBUG "value": [ 06:33:07 DEBUG } 06:33:07 DEBUG }, 06:33:07 DEBUG "0" 06:33:07 DEBUG 1713591187, 06:33:07 DEBUG } 06:33:07 DEBUG 06:33:07 DEBUG } 06:33:07 DEBUG } 06:33:07 DEBUG } 06:33:07 DEBUG 06:33:07 DEBUG } 06:33:07 DEBUG "value": [ 06:33:07 DEBUG } 06:33:07 DEBUG } 06:33:07 DEBUG ] 06:33:07 DEBUG 06:33:07 DEBUG "value": [ 06:33:07 DEBUG ] 06:33:07 DEBUG "0" 06:33:07 DEBUG "pod": "ds-cts-2" 06:33:07 DEBUG } 06:33:07 DEBUG } 06:33:07 DEBUG ] 06:33:07 DEBUG 06:33:07 DEBUG ] 06:33:07 DEBUG } 06:33:07 DEBUG } 06:33:07 DEBUG } 06:33:07 DEBUG 06:33:07 DEBUG 1713591187, 06:33:07 DEBUG } 06:33:07 DEBUG ] 06:33:07 DEBUG }, 06:33:07 DEBUG } 06:33:07 DEBUG "instance": "10.142.0.43:9100" 06:33:07 DEBUG 1713591187, 06:33:07 DEBUG } 06:33:07 DEBUG } 06:33:07 DEBUG } 06:33:07 DEBUG 06:33:07 DEBUG [...] 06:33:07 DEBUG "2" 06:33:07 DEBUG } 06:33:07 DEBUG "value": [ 06:33:07 DEBUG } 06:33:07 DEBUG }, 06:33:07 DEBUG "0.094" 06:33:07 DEBUG ] 06:33:07 DEBUG 06:33:07 DEBUG ] 06:33:07 DEBUG ] 06:33:07 DEBUG 1713591187, 06:33:07 DEBUG "value": [ 06:33:07 DEBUG ] 06:33:07 DEBUG } 06:33:07 DEBUG 06:33:07 DEBUG } 06:33:07 DEBUG } 06:33:07 DEBUG "0" 06:33:07 DEBUG 1713591187, 06:33:07 DEBUG } 06:33:07 DEBUG } 06:33:07 DEBUG 06:33:07 DEBUG ] 06:33:07 DEBUG } 06:33:07 DEBUG ] 06:33:07 DEBUG "3161792099328" 06:33:07 DEBUG ] 06:33:07 DEBUG "instance": "10.142.0.43:9100" 06:33:07 DEBUG } 06:33:07 DEBUG } 06:33:07 DEBUG ] 06:33:07 DEBUG } 06:33:07 DEBUG }, 06:33:07 DEBUG } 06:33:07 DEBUG ] 06:33:07 DEBUG } 06:33:07 DEBUG } 06:33:07 DEBUG "value": [ 06:33:07 DEBUG } 06:33:07 DEBUG ] 06:33:07 DEBUG 1713591187, 06:33:07 DEBUG } 06:33:07 DEBUG } 06:33:07 DEBUG "0.0008333333333808922" 06:33:07 DEBUG } 06:33:07 DEBUG ] 06:33:07 DEBUG } 06:33:07 DEBUG ] 06:33:07 DEBUG } 06:33:07 DEBUG } 06:34:07 INFO 06:34:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591247" 06:34:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591247" 06:34:07 INFO [loop_until]: kubectl --namespace=xlou top pods 06:34:07 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:34:07 INFO 06:34:07 INFO [loop_until]: kubectl --namespace=xlou top node 06:34:07 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:34:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591247" 06:34:07 INFO [http_cmd]: http status code OK 06:34:07 DEBUG --- status code --- 06:34:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:34:07 DEBUG --- http response --- 06:34:07 INFO [http_cmd]: http status code OK 06:34:07 DEBUG --- status code --- 06:34:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:34:07 DEBUG --- http response --- 06:34:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713591247" 06:34:07 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 06:34:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 06:34:07 DEBUG ----- output ----- 06:34:07 DEBUG { 06:34:07 DEBUG "status": "success", 06:34:07 DEBUG "data": { 06:34:07 DEBUG "resultType": "vector", 06:34:07 DEBUG "result": [ 06:34:07 DEBUG { 06:34:07 DEBUG "metric": { 06:34:07 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:34:07 DEBUG }, 06:34:07 DEBUG "value": [ 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG [...] 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG "pod": "ds-idrepo-2" 06:34:07 DEBUG }, 06:34:07 DEBUG "value": [ 06:34:07 DEBUG 1713591247, 06:34:07 DEBUG "4665.63114838977" 06:34:07 DEBUG ] 06:34:07 DEBUG } 06:34:07 DEBUG ] 06:34:07 DEBUG } 06:34:07 DEBUG } 06:34:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591247" 06:34:07 INFO [http_cmd]: http status code OK 06:34:07 DEBUG --- status code --- 06:34:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:34:07 DEBUG --- http response --- 06:34:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591247" 06:34:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:34:07 DEBUG ----- output ----- 06:34:07 DEBUG { 06:34:07 DEBUG "status": "success", 06:34:07 DEBUG "data": { 06:34:07 DEBUG "resultType": "vector", 06:34:07 DEBUG "result": [ 06:34:07 DEBUG { 06:34:07 DEBUG "metric": { 06:34:07 DEBUG "pod": "am-767f6b67b5-9zw22" 06:34:07 DEBUG }, 06:34:07 DEBUG "value": [ 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG [...] 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG "pod": "am-767f6b67b5-vbltz" 06:34:07 DEBUG }, 06:34:07 DEBUG "value": [ 06:34:07 DEBUG 1713591247, 06:34:07 DEBUG "10463.5" 06:34:07 DEBUG ] 06:34:07 DEBUG } 06:34:07 DEBUG ] 06:34:07 DEBUG } 06:34:07 DEBUG } 06:34:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713591247" 06:34:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591247" 06:34:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591247" 06:34:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713591247" 06:34:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591247" 06:34:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591247" 06:34:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591247" 06:34:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713591247" 06:34:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713591247" 06:34:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591247" 06:34:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591247" 06:34:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591247" 06:34:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591247" 06:34:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591247" 06:34:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591247" 06:34:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591247" 06:34:07 ERROR [http_cmd]: ERROR 06:34:07 INFO [http_cmd]: http status code OK 06:34:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713591247" 06:34:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713591247" 06:34:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713591247" 06:34:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713591247" 06:34:07 INFO [http_cmd]: http status code OK 06:34:07 DEBUG --- status code --- 06:34:07 INFO [http_cmd]: http status code OK 06:34:07 INFO [http_cmd]: http status code OK 06:34:07 INFO [http_cmd]: http status code OK 06:34:07 INFO [loop_until]: OK (rc = 0) 06:34:07 INFO [http_cmd]: http status code OK 06:34:07 INFO [http_cmd]: http status code OK 06:34:07 INFO [http_cmd]: http status code OK 06:34:07 ERROR --- status code --- 06:34:07 DEBUG --- status code --- 06:34:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713591247" 06:34:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:34:07 DEBUG --- status code --- 06:34:07 DEBUG --- status code --- 06:34:07 DEBUG --- status code --- 06:34:07 DEBUG --- stdout --- 06:34:07 DEBUG --- status code --- 06:34:07 DEBUG --- status code --- 06:34:07 INFO [http_cmd]: http status code OK 06:34:07 DEBUG --- status code --- 06:34:07 ERROR http status code is 400 (expected [200, 401, 403, 404]) 06:34:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:34:07 DEBUG --- http response --- 06:34:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:34:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:34:07 INFO [http_cmd]: http status code OK 06:34:07 INFO [loop_until]: OK (rc = 0) 06:34:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:34:07 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 1490m 8603Mi am-767f6b67b5-qcqz2 3937m 8666Mi am-767f6b67b5-vbltz 3775m 8609Mi ds-cts-0 1008m 10618Mi ds-cts-1 825m 10710Mi ds-cts-2 896m 10511Mi ds-idrepo-0 957m 3720Mi ds-idrepo-1 981m 3731Mi ds-idrepo-2 1361m 3697Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1177Mi idm-b85f9d978-wnhxb 4m 1236Mi lodemon-7cb5b9456d-dskfn 1m 61Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 3373m 2304Mi 06:34:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:34:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:34:07 DEBUG --- status code --- 06:34:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:34:07 ERROR --- http response --- 06:34:07 DEBUG --- http response --- 06:34:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:34:07 DEBUG --- http response --- 06:34:07 DEBUG --- http response --- 06:34:07 DEBUG --- status code --- 06:34:07 DEBUG --- stdout --- 06:34:07 DEBUG --- http response --- 06:34:07 INFO [http_cmd]: http status code OK 06:34:07 INFO [http_cmd]: http status code OK 06:34:07 INFO [http_cmd]: http status code OK 06:34:07 INFO [http_cmd]: http status code OK 06:34:07 INFO [http_cmd]: http status code OK 06:34:07 DEBUG --- stderr --- 06:34:07 INFO [http_cmd]: http status code OK 06:34:07 INFO [http_cmd]: http status code OK 06:34:07 INFO [http_cmd]: http status code OK 06:34:07 INFO [http_cmd]: http status code OK 06:34:07 DEBUG --- http response --- 06:34:07 INFO [http_cmd]: http status code OK 06:34:07 DEBUG --- http response --- 06:34:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:34:07 DEBUG --- http response --- 06:34:07 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 06:34:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:34:07 DEBUG ----- output ----- 06:34:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:34:07 INFO [http_cmd]: http status code OK 06:34:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 06:34:07 INFO [http_cmd]: http status code OK 06:34:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:34:07 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 73m 0% 6260Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 1673m 5% 7403Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 4395m 14% 11302Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 85m 0% 3384Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 1188m 3% 13738Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 151m 0% 5188Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 4528m 15% 11542Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 2481m 8% 3011Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 4140m 13% 10825Mi 9% gke-xlou-cdm-ds-0f601751-j488 2150m 7% 7027Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 2191m 7% 15703Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 1874m 6% 7626Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 2504m 8% 12610Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 3715m 23% 6183Mi 10% 06:34:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:34:07 DEBUG --- status code --- 06:34:07 INFO [http_cmd]: http status code OK 06:34:07 DEBUG --- status code --- 06:34:07 DEBUG --- status code --- 06:34:07 DEBUG --- status code --- 06:34:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:34:07 DEBUG --- status code --- 06:34:07 DEBUG --- status code --- 06:34:07 DEBUG --- status code --- 06:34:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:34:07 DEBUG --- status code --- 06:34:07 DEBUG --- status code --- 06:34:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:34:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:34:07 DEBUG --- http response --- 06:34:07 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 06:34:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 06:34:07 DEBUG ----- output ----- 06:34:07 DEBUG { 06:34:07 DEBUG ----- output ----- 06:34:07 DEBUG --- status code --- 06:34:07 DEBUG ----- output ----- 06:34:07 DEBUG --- status code --- 06:34:07 DEBUG --- http response --- 06:34:07 DEBUG --- stderr --- 06:34:07 DEBUG ----- output ----- 06:34:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:34:07 DEBUG --- status code --- 06:34:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:34:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:34:07 DEBUG 06:34:07 DEBUG --- http response --- 06:34:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:34:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:34:07 DEBUG --- status code --- 06:34:07 DEBUG --- http response --- 06:34:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:34:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:34:07 DEBUG ----- output ----- 06:34:07 DEBUG ----- output ----- 06:34:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:34:07 DEBUG ----- output ----- 06:34:07 DEBUG { 06:34:07 DEBUG "status": "success", 06:34:07 DEBUG { 06:34:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:34:07 DEBUG { 06:34:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:34:07 DEBUG 06:34:07 DEBUG { 06:34:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 06:34:07 DEBUG --- http response --- 06:34:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:34:07 DEBUG --- http response --- 06:34:07 DEBUG --- http response --- 06:34:07 DEBUG --- http response --- 06:34:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:34:07 DEBUG --- http response --- 06:34:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:34:07 DEBUG --- http response --- 06:34:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:34:07 DEBUG --- http response --- 06:34:07 DEBUG { 06:34:07 DEBUG { 06:34:07 DEBUG ----- output ----- 06:34:07 DEBUG { 06:34:07 DEBUG "status": "success", 06:34:07 DEBUG "data": { 06:34:07 DEBUG "status": "success", 06:34:07 DEBUG --- http response --- 06:34:07 DEBUG "status": "success", 06:34:07 DEBUG --- http response --- 06:34:07 DEBUG "status": "success", 06:34:07 DEBUG ----- output ----- 06:34:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:34:07 DEBUG --- http response --- 06:34:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:34:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:34:07 DEBUG ----- output ----- 06:34:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:34:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:34:07 DEBUG --- http response --- 06:34:07 DEBUG ----- output ----- 06:34:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:34:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:34:07 DEBUG "status": "success", 06:34:07 DEBUG "status": "success", 06:34:07 DEBUG { 06:34:07 DEBUG "status": "success", 06:34:07 DEBUG "data": { 06:34:07 DEBUG "resultType": "vector", 06:34:07 DEBUG "data": { 06:34:07 DEBUG "data": { 06:34:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:34:07 DEBUG "data": { 06:34:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:34:07 DEBUG { 06:34:07 DEBUG ----- output ----- 06:34:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:34:07 DEBUG ----- output ----- 06:34:07 DEBUG ----- output ----- 06:34:07 DEBUG { 06:34:07 DEBUG ----- output ----- 06:34:07 DEBUG ----- output ----- 06:34:07 DEBUG { 06:34:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:34:07 DEBUG ----- output ----- 06:34:07 DEBUG ----- output ----- 06:34:07 DEBUG "data": { 06:34:07 DEBUG "data": { 06:34:07 DEBUG "status": "success", 06:34:07 DEBUG "data": { 06:34:07 DEBUG "resultType": "vector", 06:34:07 DEBUG "result": [ 06:34:07 DEBUG "resultType": "vector", 06:34:07 DEBUG "resultType": "vector", 06:34:07 DEBUG ----- output ----- 06:34:07 DEBUG "resultType": "vector", 06:34:07 DEBUG ----- output ----- 06:34:07 DEBUG "status": "success", 06:34:07 DEBUG { 06:34:07 DEBUG ----- output ----- 06:34:07 DEBUG { 06:34:07 DEBUG { 06:34:07 DEBUG "status": "success", 06:34:07 DEBUG { 06:34:07 DEBUG { 06:34:07 DEBUG "status": "success", 06:34:07 DEBUG ----- output ----- 06:34:07 DEBUG { 06:34:07 DEBUG { 06:34:07 DEBUG "resultType": "vector", 06:34:07 DEBUG "resultType": "vector", 06:34:07 DEBUG "data": { 06:34:07 DEBUG "resultType": "vector", 06:34:07 DEBUG "result": [ 06:34:07 DEBUG { 06:34:07 DEBUG "result": [ 06:34:07 DEBUG "result": [ 06:34:07 DEBUG { 06:34:07 DEBUG "result": [ 06:34:07 DEBUG { 06:34:07 DEBUG "data": { 06:34:07 DEBUG "status": "success", 06:34:07 DEBUG { 06:34:07 DEBUG "status": "success", 06:34:07 DEBUG "status": "success", 06:34:07 DEBUG "data": { 06:34:07 DEBUG "status": "success", 06:34:07 DEBUG "status": "success", 06:34:07 DEBUG "data": { 06:34:07 DEBUG { 06:34:07 DEBUG "status": "success", 06:34:07 DEBUG "status": "success", 06:34:07 DEBUG "data": { 06:34:07 DEBUG "result": [ 06:34:07 DEBUG "resultType": "vector", 06:34:07 DEBUG "result": [ 06:34:07 DEBUG { 06:34:07 DEBUG "metric": { 06:34:07 DEBUG { 06:34:07 DEBUG { 06:34:07 DEBUG "status": "success", 06:34:07 DEBUG { 06:34:07 DEBUG "status": "success", 06:34:07 DEBUG "resultType": "vector", 06:34:07 DEBUG "data": { 06:34:07 DEBUG "status": "success", 06:34:07 DEBUG "data": { 06:34:07 DEBUG "data": { 06:34:07 DEBUG "resultType": "vector", 06:34:07 DEBUG "data": { 06:34:07 DEBUG "data": { 06:34:07 DEBUG "resultType": "vector", 06:34:07 DEBUG "status": "success", 06:34:07 DEBUG "data": { 06:34:07 DEBUG "resultType": "vector", 06:34:07 DEBUG "resultType": "vector", 06:34:07 DEBUG { 06:34:07 DEBUG "result": [ 06:34:07 DEBUG { 06:34:07 DEBUG "metric": { 06:34:07 DEBUG "pod": "am-767f6b67b5-9zw22" 06:34:07 DEBUG "metric": { 06:34:07 DEBUG "metric": { 06:34:07 DEBUG "data": { 06:34:07 DEBUG "metric": { 06:34:07 DEBUG "data": { 06:34:07 DEBUG "result": [ 06:34:07 DEBUG "resultType": "vector", 06:34:07 DEBUG "data": { 06:34:07 DEBUG "resultType": "vector", 06:34:07 DEBUG "resultType": "vector", 06:34:07 DEBUG "result": [ 06:34:07 DEBUG "resultType": "vector", 06:34:07 DEBUG "resultType": "vector", 06:34:07 DEBUG "result": [ 06:34:07 DEBUG "data": { 06:34:07 DEBUG "result": [ 06:34:07 DEBUG "result": [ 06:34:07 DEBUG "result": [ 06:34:07 DEBUG "metric": { 06:34:07 DEBUG "pod": "ds-idrepo-0" 06:34:07 DEBUG { 06:34:07 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:34:07 DEBUG }, 06:34:07 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 06:34:07 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:34:07 DEBUG "resultType": "vector", 06:34:07 DEBUG "instance": "10.142.0.107:9100" 06:34:07 DEBUG "resultType": "vector", 06:34:07 DEBUG { 06:34:07 DEBUG "result": [ 06:34:07 DEBUG "resultType": "vector", 06:34:07 DEBUG "result": [ 06:34:07 DEBUG "result": [ 06:34:07 DEBUG { 06:34:07 DEBUG "result": [ 06:34:07 DEBUG "result": [ 06:34:07 DEBUG { 06:34:07 DEBUG "resultType": "vector", 06:34:07 DEBUG { 06:34:07 DEBUG { 06:34:07 DEBUG { 06:34:07 DEBUG "metric": { 06:34:07 DEBUG }, 06:34:07 DEBUG "metric": { 06:34:07 DEBUG }, 06:34:07 DEBUG "value": [ 06:34:07 DEBUG }, 06:34:07 DEBUG }, 06:34:07 DEBUG "result": [ 06:34:07 DEBUG }, 06:34:07 DEBUG "result": [ 06:34:07 DEBUG "metric": { 06:34:07 DEBUG { 06:34:07 DEBUG "result": [ 06:34:07 DEBUG { 06:34:07 DEBUG { 06:34:07 DEBUG "metric": { 06:34:07 DEBUG { 06:34:07 DEBUG { 06:34:07 DEBUG "metric": { 06:34:07 DEBUG "result": [ 06:34:07 DEBUG "metric": { 06:34:07 DEBUG "metric": { 06:34:07 DEBUG "metric": { 06:34:07 DEBUG "pod": "am-767f6b67b5-9zw22" 06:34:07 DEBUG "value": [ 06:34:07 DEBUG "pod": "ds-idrepo-0" 06:34:07 DEBUG "value": [ 06:34:07 DEBUG 06:34:07 DEBUG "value": [ 06:34:07 DEBUG "value": [ 06:34:07 DEBUG { 06:34:07 DEBUG "value": [ 06:34:07 DEBUG { 06:34:07 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:34:07 DEBUG "metric": { 06:34:07 DEBUG { 06:34:07 DEBUG "metric": { 06:34:07 DEBUG "metric": { 06:34:07 DEBUG "pod": "ds-cts-0" 06:34:07 DEBUG }, 06:34:07 DEBUG "metric": { 06:34:07 DEBUG "pod": "ds-cts-0" 06:34:07 DEBUG { 06:34:07 DEBUG "pod": "am-767f6b67b5-9zw22" 06:34:07 DEBUG "instance": "10.142.0.107:9100" 06:34:07 DEBUG "instance": "10.142.0.107:9100" 06:34:07 DEBUG }, 06:34:07 DEBUG 06:34:07 DEBUG }, 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG "metric": { 06:34:07 DEBUG 06:34:07 DEBUG "metric": { 06:34:07 DEBUG }, 06:34:07 DEBUG "pod": "ds-idrepo-0" 06:34:07 DEBUG "metric": { 06:34:07 DEBUG "pod": "am-767f6b67b5-9zw22" 06:34:07 DEBUG "pod": "ds-cts-0" 06:34:07 DEBUG "metric": { 06:34:07 DEBUG "value": [ 06:34:07 DEBUG 06:34:07 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 06:34:07 DEBUG "metric": { 06:34:07 DEBUG "pod": "am-767f6b67b5-9zw22" 06:34:07 DEBUG }, 06:34:07 DEBUG }, 06:34:07 DEBUG "value": [ 06:34:07 DEBUG 06:34:07 DEBUG "value": [ 06:34:07 DEBUG 06:34:07 DEBUG [...] 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG "pod": "ds-idrepo-0" 06:34:07 DEBUG 06:34:07 DEBUG "instance": "10.142.0.107:9100" 06:34:07 DEBUG "value": [ 06:34:07 DEBUG }, 06:34:07 DEBUG "pod": "am-767f6b67b5-9zw22" 06:34:07 DEBUG }, 06:34:07 DEBUG }, 06:34:07 DEBUG "instance": "10.142.0.107:9100" 06:34:07 DEBUG }, 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG [...] 06:34:07 DEBUG }, 06:34:07 DEBUG "value": [ 06:34:07 DEBUG "value": [ 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG }, 06:34:07 DEBUG 06:34:07 DEBUG }, 06:34:07 DEBUG "value": [ 06:34:07 DEBUG 06:34:07 DEBUG }, 06:34:07 DEBUG "value": [ 06:34:07 DEBUG "value": [ 06:34:07 DEBUG }, 06:34:07 DEBUG "value": [ 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG "value": [ 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG [...] 06:34:07 DEBUG 06:34:07 DEBUG [...] 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG [...] 06:34:07 DEBUG "value": [ 06:34:07 DEBUG [...] 06:34:07 DEBUG "value": [ 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG "value": [ 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG "value": [ 06:34:07 DEBUG 06:34:07 DEBUG }, 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG [...] 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG }, 06:34:07 DEBUG "value": [ 06:34:07 DEBUG [...] 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG [...] 06:34:07 DEBUG 06:34:07 DEBUG [...] 06:34:07 DEBUG 06:34:07 DEBUG "pod": "am-767f6b67b5-vbltz" 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG [...] 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG "value": [ 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG "pod": "ds-cts-2" 06:34:07 DEBUG 06:34:07 DEBUG [...] 06:34:07 DEBUG [...] 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG }, 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG "instance": "10.142.0.43:9100" 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG [...] 06:34:07 DEBUG [...] 06:34:07 DEBUG [...] 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG }, 06:34:07 DEBUG "value": [ 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG "pod": "ds-idrepo-2" 06:34:07 DEBUG 06:34:07 DEBUG "pod": "ds-idrepo-2" 06:34:07 DEBUG "value": [ 06:34:07 DEBUG 06:34:07 DEBUG "pod": "ds-idrepo-2" 06:34:07 DEBUG [...] 06:34:07 DEBUG [...] 06:34:07 DEBUG }, 06:34:07 DEBUG "value": [ 06:34:07 DEBUG 06:34:07 DEBUG [...] 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG [...] 06:34:07 DEBUG 1713591247, 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG }, 06:34:07 DEBUG 06:34:07 DEBUG }, 06:34:07 DEBUG 1713591247, 06:34:07 DEBUG "0" 06:34:07 DEBUG }, 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG 1713591247, 06:34:07 DEBUG [...] 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG [...] 06:34:07 DEBUG 06:34:07 DEBUG "pod": "ds-cts-2" 06:34:07 DEBUG 06:34:07 DEBUG "0" 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG "pod": "am-767f6b67b5-vbltz" 06:34:07 DEBUG "value": [ 06:34:07 DEBUG "pod": "ds-idrepo-2" 06:34:07 DEBUG "value": [ 06:34:07 DEBUG ] 06:34:07 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 06:34:07 DEBUG "value": [ 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG "3055133.0999999996" 06:34:07 DEBUG 06:34:07 DEBUG "pod": "ds-idrepo-2" 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG "pod": "ds-cts-2" 06:34:07 DEBUG 06:34:07 DEBUG [...] 06:34:07 DEBUG }, 06:34:07 DEBUG 06:34:07 DEBUG ] 06:34:07 DEBUG "instance": "10.142.0.43:9100" 06:34:07 DEBUG "instance": "10.142.0.43:9100" 06:34:07 DEBUG }, 06:34:07 DEBUG 1713591247, 06:34:07 DEBUG }, 06:34:07 DEBUG 1713591247, 06:34:07 DEBUG } 06:34:07 DEBUG }, 06:34:07 DEBUG 1713591247, 06:34:07 DEBUG 06:34:07 DEBUG "pod": "ds-cts-2" 06:34:07 DEBUG ] 06:34:07 DEBUG 06:34:07 DEBUG }, 06:34:07 DEBUG 06:34:07 DEBUG "pod": "am-767f6b67b5-vbltz" 06:34:07 DEBUG 06:34:07 DEBUG "instance": "10.142.0.43:9100" 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG "value": [ 06:34:07 DEBUG 06:34:07 DEBUG } 06:34:07 DEBUG ] 06:34:07 DEBUG }, 06:34:07 DEBUG "value": [ 06:34:07 DEBUG "0" 06:34:07 DEBUG "value": [ 06:34:07 DEBUG "0" 06:34:07 DEBUG ] 06:34:07 DEBUG "value": [ 06:34:07 DEBUG "0.27697721859739494" 06:34:07 DEBUG 06:34:07 DEBUG }, 06:34:07 DEBUG } 06:34:07 DEBUG 06:34:07 DEBUG "value": [ 06:34:07 DEBUG "pod": "ds-idrepo-2" 06:34:07 DEBUG }, 06:34:07 DEBUG }, 06:34:07 DEBUG }, 06:34:07 DEBUG 06:34:07 DEBUG 06:34:07 DEBUG 1713591247, 06:34:07 DEBUG "pod": "am-767f6b67b5-vbltz" 06:34:07 DEBUG } 06:34:07 DEBUG }, 06:34:07 DEBUG "value": [ 06:34:07 DEBUG "value": [ 06:34:07 DEBUG ] 06:34:07 DEBUG 1713591247, 06:34:07 DEBUG ] 06:34:07 DEBUG } 06:34:07 DEBUG 1713591247, 06:34:07 DEBUG ] 06:34:07 DEBUG "instance": "10.142.0.43:9100" 06:34:07 DEBUG "value": [ 06:34:07 DEBUG ] 06:34:07 DEBUG "pod": "am-767f6b67b5-vbltz" 06:34:07 DEBUG 1713591247, 06:34:07 DEBUG }, 06:34:07 DEBUG "value": [ 06:34:07 DEBUG "value": [ 06:34:07 DEBUG 1713591247, 06:34:07 DEBUG "value": [ 06:34:07 DEBUG 06:34:07 DEBUG "0" 06:34:07 DEBUG ] 06:34:07 DEBUG } 06:34:07 DEBUG 1713591247, 06:34:07 DEBUG 1713591247, 06:34:07 DEBUG 1713591247, 06:34:07 DEBUG } 06:34:07 DEBUG "0" 06:34:07 DEBUG } 06:34:07 DEBUG } 06:34:07 DEBUG "0.690086949147377" 06:34:07 DEBUG } 06:34:07 DEBUG }, 06:34:07 DEBUG 1713591247, 06:34:07 DEBUG } 06:34:07 DEBUG }, 06:34:07 DEBUG "20924.648786717753" 06:34:07 DEBUG "value": [ 06:34:07 DEBUG 1713591247, 06:34:07 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 06:34:07 DEBUG "0.00004865333333333333" 06:34:07 DEBUG 1713591247, 06:34:07 DEBUG "pod": "am-767f6b67b5-vbltz" 06:34:07 DEBUG }, 06:34:07 DEBUG } 06:34:07 DEBUG "698" 06:34:07 DEBUG "0.0036666666666671215" 06:34:07 DEBUG "0.005399999999926308" 06:34:07 DEBUG ] 06:34:07 DEBUG ] 06:34:07 DEBUG ] 06:34:07 DEBUG } 06:34:07 DEBUG ] 06:34:07 DEBUG } 06:34:07 DEBUG "0" 06:34:07 DEBUG } 06:34:07 DEBUG "value": [ 06:34:07 DEBUG ] 06:34:07 DEBUG } 06:34:07 DEBUG 1713591247, 06:34:07 DEBUG }, 06:34:07 DEBUG "value": [ 06:34:07 DEBUG "3047765.7" 06:34:07 DEBUG }, 06:34:07 DEBUG "value": [ 06:34:07 DEBUG ] 06:34:07 DEBUG ] 06:34:07 DEBUG ] 06:34:07 DEBUG ] 06:34:07 DEBUG } 06:34:07 DEBUG } 06:34:07 DEBUG ] 06:34:07 DEBUG } 06:34:07 DEBUG "value": [ 06:34:07 DEBUG 1713591247, 06:34:07 DEBUG ] 06:34:07 DEBUG 1713591247, 06:34:07 DEBUG "0" 06:34:07 DEBUG ] 06:34:07 DEBUG "0" 06:34:07 DEBUG ] 06:34:07 DEBUG 1713591247, 06:34:07 DEBUG ] 06:34:07 DEBUG "value": [ 06:34:07 DEBUG 1713591247, 06:34:07 DEBUG } 06:34:07 DEBUG } 06:34:07 DEBUG } 06:34:07 DEBUG } 06:34:07 DEBUG } 06:34:07 DEBUG ] 06:34:07 DEBUG } 06:34:07 DEBUG ] 06:34:07 DEBUG "3161815530496" 06:34:07 DEBUG } 06:34:07 DEBUG "0.094" 06:34:07 DEBUG ] 06:34:07 DEBUG } 06:34:07 DEBUG ] 06:34:07 DEBUG } 06:34:07 DEBUG "2446655488" 06:34:07 DEBUG } 06:34:07 DEBUG 1713591247, 06:34:07 DEBUG "698.5666666666666" 06:34:07 DEBUG } 06:34:07 DEBUG ] 06:34:07 DEBUG ] 06:34:07 DEBUG ] 06:34:07 DEBUG } 06:34:07 DEBUG } 06:34:07 DEBUG } 06:34:07 DEBUG ] 06:34:07 DEBUG ] 06:34:07 DEBUG ] 06:34:07 DEBUG } 06:34:07 DEBUG } 06:34:07 DEBUG } 06:34:07 DEBUG ] 06:34:07 DEBUG ] 06:34:07 DEBUG ] 06:34:07 DEBUG "698.8" 06:34:07 DEBUG ] 06:34:07 DEBUG } 06:34:07 DEBUG } 06:34:07 DEBUG } 06:34:07 DEBUG } 06:34:07 DEBUG } 06:34:07 DEBUG } 06:34:07 DEBUG } 06:34:07 DEBUG } 06:34:07 DEBUG ] 06:34:07 DEBUG ] 06:34:07 DEBUG } 06:34:07 DEBUG } 06:34:07 DEBUG } 06:34:07 DEBUG ] 06:34:07 DEBUG } 06:34:07 DEBUG } 06:34:07 DEBUG } 06:34:07 DEBUG } 06:34:07 DEBUG ] 06:34:07 DEBUG } 06:34:07 DEBUG ] 06:34:07 DEBUG } 06:34:07 DEBUG } 06:34:07 DEBUG } 06:34:07 DEBUG ] 06:34:07 DEBUG } 06:34:07 DEBUG } 06:34:07 DEBUG ] 06:34:07 DEBUG } 06:34:07 DEBUG } 06:34:07 DEBUG } 06:34:07 DEBUG } 06:34:07 DEBUG } 06:34:07 DEBUG ] 06:34:07 DEBUG } 06:34:07 DEBUG } 06:34:07 DEBUG } 06:34:07 DEBUG } 06:34:07 DEBUG } 06:34:07 DEBUG } 06:34:07 DEBUG } 06:35:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591307" 06:35:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591307" 06:35:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591307" 06:35:07 INFO [http_cmd]: http status code OK 06:35:07 DEBUG --- status code --- 06:35:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:35:07 DEBUG --- http response --- 06:35:07 INFO [http_cmd]: http status code OK 06:35:07 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 06:35:07 DEBUG --- status code --- 06:35:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:35:07 DEBUG --- http response --- 06:35:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:35:07 DEBUG ----- output ----- 06:35:07 DEBUG { 06:35:07 DEBUG "status": "success", 06:35:07 DEBUG "data": { 06:35:07 DEBUG "resultType": "vector", 06:35:07 DEBUG "result": [ 06:35:07 DEBUG { 06:35:07 DEBUG "metric": { 06:35:07 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:35:07 DEBUG }, 06:35:07 DEBUG "value": [ 06:35:07 DEBUG 06:35:07 DEBUG 06:35:07 DEBUG 06:35:07 DEBUG [...] 06:35:07 DEBUG 06:35:07 DEBUG 06:35:07 DEBUG 06:35:07 DEBUG "pod": "ds-idrepo-2" 06:35:07 DEBUG }, 06:35:07 DEBUG "value": [ 06:35:07 DEBUG 1713591307, 06:35:07 DEBUG "2053.2325481349353" 06:35:07 DEBUG ] 06:35:07 DEBUG } 06:35:07 DEBUG ] 06:35:07 DEBUG } 06:35:07 DEBUG } 06:35:07 INFO [http_cmd]: http status code OK 06:35:07 DEBUG --- status code --- 06:35:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:35:07 DEBUG --- http response --- 06:35:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:35:07 DEBUG ----- output ----- 06:35:07 DEBUG { 06:35:07 DEBUG "status": "success", 06:35:07 DEBUG "data": { 06:35:07 DEBUG "resultType": "vector", 06:35:07 DEBUG "result": [ 06:35:07 DEBUG { 06:35:07 DEBUG "metric": { 06:35:07 DEBUG "pod": "am-767f6b67b5-9zw22" 06:35:07 DEBUG }, 06:35:07 DEBUG "value": [ 06:35:07 DEBUG 06:35:07 DEBUG 06:35:07 DEBUG 06:35:07 DEBUG [...] 06:35:07 DEBUG 06:35:07 DEBUG 06:35:07 DEBUG 06:35:07 DEBUG "pod": "am-767f6b67b5-vbltz" 06:35:07 DEBUG }, 06:35:07 DEBUG "value": [ 06:35:07 DEBUG 1713591307, 06:35:07 DEBUG "136909" 06:35:07 DEBUG ] 06:35:07 DEBUG } 06:35:07 DEBUG ] 06:35:07 DEBUG } 06:35:07 DEBUG } 06:35:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713591307" 06:35:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591307" 06:35:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591307" 06:35:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713591307" 06:35:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713591307" 06:35:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713591307" 06:35:07 INFO 06:35:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713591307" 06:35:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591307" 06:35:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591307" 06:35:07 INFO 06:35:07 INFO [loop_until]: kubectl --namespace=xlou top pods 06:35:07 INFO [loop_until]: kubectl --namespace=xlou top node 06:35:07 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:35:07 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:35:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591307" 06:35:07 ERROR [http_cmd]: ERROR 06:35:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713591307" 06:35:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591307" 06:35:07 INFO [http_cmd]: http status code OK 06:35:07 ERROR --- status code --- 06:35:07 INFO [http_cmd]: http status code OK 06:35:07 DEBUG --- status code --- 06:35:07 INFO [http_cmd]: http status code OK 06:35:07 DEBUG --- status code --- 06:35:07 INFO [http_cmd]: http status code OK 06:35:07 INFO [http_cmd]: http status code OK 06:35:07 INFO [http_cmd]: http status code OK 06:35:07 DEBUG --- status code --- 06:35:07 INFO [http_cmd]: http status code OK 06:35:07 INFO [http_cmd]: http status code OK 06:35:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:35:07 ERROR http status code is 400 (expected [200, 401, 403, 404]) 06:35:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:35:07 DEBUG --- status code --- 06:35:07 DEBUG --- status code --- 06:35:07 DEBUG --- status code --- 06:35:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:35:07 DEBUG --- status code --- 06:35:07 DEBUG --- status code --- 06:35:07 DEBUG --- http response --- 06:35:07 ERROR --- http response --- 06:35:07 DEBUG --- http response --- 06:35:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:35:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:35:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:35:07 DEBUG --- http response --- 06:35:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:35:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:35:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:35:07 INFO [http_cmd]: http status code OK 06:35:07 INFO [http_cmd]: http status code OK 06:35:07 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 06:35:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:35:07 DEBUG --- http response --- 06:35:07 DEBUG --- http response --- 06:35:07 DEBUG --- http response --- 06:35:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:35:07 DEBUG --- http response --- 06:35:07 DEBUG --- http response --- 06:35:07 DEBUG ----- output ----- 06:35:07 DEBUG --- status code --- 06:35:07 DEBUG --- status code --- 06:35:07 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 06:35:07 DEBUG ----- output ----- 06:35:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:35:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:35:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:35:07 DEBUG ----- output ----- 06:35:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:35:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713591307" 06:35:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591307" 06:35:07 INFO [http_cmd]: http status code OK 06:35:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591307" 06:35:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:35:07 DEBUG { 06:35:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:35:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:35:07 DEBUG { 06:35:07 DEBUG ----- output ----- 06:35:07 DEBUG ----- output ----- 06:35:07 DEBUG ----- output ----- 06:35:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591307" 06:35:07 DEBUG { 06:35:07 DEBUG ----- output ----- 06:35:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591307" 06:35:07 DEBUG --- status code --- 06:35:07 DEBUG ----- output ----- 06:35:07 DEBUG "status": "success", 06:35:07 DEBUG --- http response --- 06:35:07 DEBUG --- http response --- 06:35:07 DEBUG "status": "success", 06:35:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591307" 06:35:07 DEBUG { 06:35:07 DEBUG { 06:35:07 DEBUG { 06:35:07 DEBUG "status": "success", 06:35:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713591307" 06:35:07 DEBUG { 06:35:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591307" 06:35:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713591307" 06:35:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713591307" 06:35:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591307" 06:35:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591307" 06:35:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:35:07 DEBUG { 06:35:07 DEBUG "data": { 06:35:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:35:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:35:07 DEBUG "data": { 06:35:07 DEBUG "status": "success", 06:35:07 INFO [http_cmd]: http status code OK 06:35:07 INFO [http_cmd]: http status code OK 06:35:07 INFO [http_cmd]: http status code OK 06:35:07 DEBUG --- status code --- 06:35:07 DEBUG "status": "success", 06:35:07 DEBUG "data": { 06:35:07 INFO [http_cmd]: http status code OK 06:35:07 DEBUG "status": "success", 06:35:07 INFO [http_cmd]: http status code OK 06:35:07 DEBUG --- http response --- 06:35:07 INFO [http_cmd]: http status code OK 06:35:07 DEBUG "status": "success", 06:35:07 DEBUG "resultType": "vector", 06:35:07 DEBUG ----- output ----- 06:35:07 DEBUG ----- output ----- 06:35:07 DEBUG "resultType": "vector", 06:35:07 DEBUG "data": { 06:35:07 DEBUG --- status code --- 06:35:07 DEBUG --- status code --- 06:35:07 DEBUG "status": "success", 06:35:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:35:07 DEBUG "data": { 06:35:07 INFO [http_cmd]: http status code OK 06:35:07 DEBUG "resultType": "vector", 06:35:07 DEBUG --- status code --- 06:35:07 DEBUG "data": { 06:35:07 DEBUG --- status code --- 06:35:07 DEBUG --- status code --- 06:35:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:35:07 DEBUG "data": { 06:35:07 INFO [http_cmd]: http status code OK 06:35:07 INFO [http_cmd]: http status code OK 06:35:07 DEBUG "result": [ 06:35:07 DEBUG { 06:35:07 DEBUG { 06:35:07 DEBUG "result": [ 06:35:07 DEBUG "resultType": "vector", 06:35:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:35:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:35:07 DEBUG "data": { 06:35:07 DEBUG --- http response --- 06:35:07 DEBUG "resultType": "vector", 06:35:07 DEBUG --- status code --- 06:35:07 DEBUG "result": [ 06:35:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:35:07 DEBUG "resultType": "vector", 06:35:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:35:07 DEBUG --- http response --- 06:35:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:35:07 DEBUG ----- output ----- 06:35:07 INFO [http_cmd]: http status code OK 06:35:07 DEBUG "resultType": "vector", 06:35:07 DEBUG --- status code --- 06:35:07 INFO [loop_until]: OK (rc = 0) 06:35:07 DEBUG --- status code --- 06:35:07 DEBUG { 06:35:07 DEBUG "status": "success", 06:35:07 DEBUG "status": "success", 06:35:07 DEBUG { 06:35:07 DEBUG "result": [ 06:35:07 INFO [http_cmd]: http status code OK 06:35:07 DEBUG --- http response --- 06:35:07 DEBUG --- http response --- 06:35:07 DEBUG "resultType": "vector", 06:35:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:35:07 DEBUG "result": [ 06:35:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:35:07 DEBUG { 06:35:07 DEBUG --- http response --- 06:35:07 DEBUG "result": [ 06:35:07 INFO [http_cmd]: http status code OK 06:35:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:35:07 DEBUG --- http response --- 06:35:07 DEBUG { 06:35:07 DEBUG --- status code --- 06:35:07 DEBUG "result": [ 06:35:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:35:07 DEBUG --- stdout --- 06:35:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:35:07 DEBUG "metric": { 06:35:07 DEBUG "data": { 06:35:07 DEBUG "data": { 06:35:07 DEBUG "metric": { 06:35:07 DEBUG { 06:35:07 DEBUG --- status code --- 06:35:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:35:07 DEBUG "result": [ 06:35:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:35:07 DEBUG ----- output ----- 06:35:07 DEBUG { 06:35:07 DEBUG --- http response --- 06:35:07 DEBUG "metric": { 06:35:07 DEBUG { 06:35:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:35:07 DEBUG --- status code --- 06:35:07 DEBUG ----- output ----- 06:35:07 DEBUG "status": "success", 06:35:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:35:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:35:07 DEBUG { 06:35:07 DEBUG --- http response --- 06:35:07 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 14059m 8619Mi am-767f6b67b5-qcqz2 13972m 8676Mi am-767f6b67b5-vbltz 15924m 8620Mi ds-cts-0 7989m 10400Mi ds-cts-1 7685m 10669Mi ds-cts-2 8019m 10021Mi ds-idrepo-0 6793m 3717Mi ds-idrepo-1 6685m 3705Mi ds-idrepo-2 6955m 3675Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1177Mi idm-b85f9d978-wnhxb 3m 1236Mi lodemon-7cb5b9456d-dskfn 2m 62Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 8070m 2875Mi 06:35:07 DEBUG --- http response --- 06:35:07 DEBUG "pod": "ds-cts-0" 06:35:07 DEBUG "resultType": "vector", 06:35:07 DEBUG "resultType": "vector", 06:35:07 DEBUG "pod": "ds-idrepo-0" 06:35:07 DEBUG "metric": { 06:35:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:35:07 DEBUG ----- output ----- 06:35:07 DEBUG { 06:35:07 DEBUG ----- output ----- 06:35:07 DEBUG { 06:35:07 DEBUG "metric": { 06:35:07 DEBUG "pod": "am-767f6b67b5-9zw22" 06:35:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:35:07 DEBUG "metric": { 06:35:07 DEBUG ----- output ----- 06:35:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:35:07 DEBUG { 06:35:07 DEBUG "data": { 06:35:07 DEBUG ----- output ----- 06:35:07 DEBUG --- http response --- 06:35:07 DEBUG "metric": { 06:35:07 DEBUG --- stderr --- 06:35:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:35:07 DEBUG }, 06:35:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:35:07 DEBUG "result": [ 06:35:07 DEBUG "result": [ 06:35:07 DEBUG }, 06:35:07 DEBUG "pod": "am-767f6b67b5-9zw22" 06:35:07 DEBUG --- http response --- 06:35:07 DEBUG { 06:35:07 DEBUG "metric": { 06:35:07 DEBUG { 06:35:07 DEBUG "status": "success", 06:35:07 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 06:35:07 DEBUG }, 06:35:07 DEBUG ----- output ----- 06:35:07 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 06:35:07 DEBUG { 06:35:07 DEBUG --- http response --- 06:35:07 DEBUG "status": "success", 06:35:07 DEBUG "resultType": "vector", 06:35:07 DEBUG { 06:35:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:35:07 DEBUG "instance": "10.142.0.107:9100" 06:35:07 DEBUG 06:35:07 DEBUG ----- output ----- 06:35:07 DEBUG "value": [ 06:35:07 DEBUG ----- output ----- 06:35:07 DEBUG { 06:35:07 DEBUG { 06:35:07 DEBUG "value": [ 06:35:07 DEBUG }, 06:35:07 DEBUG "status": "success", 06:35:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:35:07 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:35:07 DEBUG "status": "success", 06:35:07 DEBUG "data": { 06:35:07 DEBUG }, 06:35:07 DEBUG "value": [ 06:35:07 DEBUG { 06:35:07 DEBUG }, 06:35:07 DEBUG "status": "success", 06:35:07 DEBUG "data": { 06:35:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:35:07 DEBUG "result": [ 06:35:07 DEBUG "status": "success", 06:35:07 DEBUG ----- output ----- 06:35:07 DEBUG }, 06:35:07 DEBUG { 06:35:07 DEBUG 06:35:07 DEBUG { 06:35:07 DEBUG "metric": { 06:35:07 DEBUG "metric": { 06:35:07 DEBUG 06:35:07 DEBUG "value": [ 06:35:07 DEBUG "data": { 06:35:07 DEBUG ----- output ----- 06:35:07 DEBUG }, 06:35:07 DEBUG "data": { 06:35:07 DEBUG "resultType": "vector", 06:35:07 DEBUG "value": [ 06:35:07 DEBUG 06:35:07 DEBUG "status": "success", 06:35:07 DEBUG "value": [ 06:35:07 DEBUG "data": { 06:35:07 DEBUG "resultType": "vector", 06:35:07 DEBUG ----- output ----- 06:35:07 DEBUG { 06:35:07 DEBUG "data": { 06:35:07 DEBUG { 06:35:07 DEBUG "value": [ 06:35:07 DEBUG "status": "success", 06:35:07 DEBUG 06:35:07 DEBUG "status": "success", 06:35:07 DEBUG "pod": "am-767f6b67b5-9zw22" 06:35:07 DEBUG "instance": "10.142.0.107:9100" 06:35:07 DEBUG 06:35:07 DEBUG 06:35:07 DEBUG "resultType": "vector", 06:35:07 DEBUG { 06:35:07 DEBUG "value": [ 06:35:07 DEBUG "resultType": "vector", 06:35:07 DEBUG "result": [ 06:35:07 DEBUG 06:35:07 DEBUG 06:35:07 DEBUG "data": { 06:35:07 DEBUG 06:35:07 DEBUG "resultType": "vector", 06:35:07 DEBUG "result": [ 06:35:07 DEBUG { 06:35:07 DEBUG "metric": { 06:35:07 DEBUG "resultType": "vector", 06:35:07 DEBUG "status": "success", 06:35:07 DEBUG 06:35:07 DEBUG "data": { 06:35:07 DEBUG 06:35:07 DEBUG "data": { 06:35:07 DEBUG }, 06:35:07 DEBUG }, 06:35:07 DEBUG 06:35:07 DEBUG 06:35:07 DEBUG "result": [ 06:35:07 DEBUG "status": "success", 06:35:07 DEBUG 06:35:07 DEBUG "result": [ 06:35:07 DEBUG { 06:35:07 DEBUG 06:35:07 DEBUG 06:35:07 DEBUG "resultType": "vector", 06:35:07 DEBUG 06:35:07 DEBUG "result": [ 06:35:07 DEBUG { 06:35:07 DEBUG "status": "success", 06:35:07 DEBUG "pod": "ds-idrepo-0" 06:35:07 DEBUG "result": [ 06:35:07 DEBUG "data": { 06:35:07 DEBUG 06:35:07 DEBUG "resultType": "vector", 06:35:07 DEBUG [...] 06:35:07 DEBUG "resultType": "vector", 06:35:07 DEBUG "result": [ 06:35:07 DEBUG "value": [ 06:35:07 DEBUG [...] 06:35:07 DEBUG 06:35:07 DEBUG { 06:35:07 DEBUG "data": { 06:35:07 DEBUG 06:35:07 DEBUG { 06:35:07 DEBUG "metric": { 06:35:07 DEBUG 06:35:07 DEBUG [...] 06:35:07 DEBUG "result": [ 06:35:07 DEBUG 06:35:07 DEBUG { 06:35:07 DEBUG "metric": { 06:35:07 DEBUG "data": { 06:35:07 DEBUG }, 06:35:07 DEBUG { 06:35:07 DEBUG "resultType": "vector", 06:35:07 DEBUG 06:35:07 DEBUG "result": [ 06:35:07 DEBUG 06:35:07 DEBUG "value": [ 06:35:07 DEBUG { 06:35:07 DEBUG 06:35:07 DEBUG 06:35:07 DEBUG [...] 06:35:07 DEBUG "metric": { 06:35:07 DEBUG "resultType": "vector", 06:35:07 DEBUG 06:35:07 DEBUG "metric": { 06:35:07 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:35:07 DEBUG [...] 06:35:07 DEBUG 06:35:07 DEBUG { 06:35:07 DEBUG [...] 06:35:07 DEBUG "metric": { 06:35:07 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:35:07 INFO [loop_until]: OK (rc = 0) 06:35:07 DEBUG "resultType": "vector", 06:35:07 DEBUG "value": [ 06:35:07 DEBUG "metric": { 06:35:07 DEBUG "result": [ 06:35:07 DEBUG [...] 06:35:07 DEBUG { 06:35:07 DEBUG 06:35:07 DEBUG 06:35:07 DEBUG 06:35:07 DEBUG 06:35:07 DEBUG 06:35:07 DEBUG 06:35:07 DEBUG "pod": "ds-idrepo-0" 06:35:07 DEBUG "result": [ 06:35:07 DEBUG [...] 06:35:07 DEBUG "instance": "10.142.0.107:9100" 06:35:07 DEBUG }, 06:35:07 DEBUG 06:35:07 DEBUG 06:35:07 DEBUG "metric": { 06:35:07 DEBUG 06:35:07 DEBUG "pod": "ds-cts-0" 06:35:07 DEBUG }, 06:35:07 DEBUG --- stdout --- 06:35:07 DEBUG "result": [ 06:35:07 DEBUG 06:35:07 DEBUG "pod": "ds-cts-0" 06:35:07 DEBUG { 06:35:07 DEBUG 06:35:07 DEBUG "metric": { 06:35:07 DEBUG 06:35:07 DEBUG "metric": { 06:35:07 DEBUG 06:35:07 DEBUG 06:35:07 DEBUG 06:35:07 DEBUG 06:35:07 DEBUG }, 06:35:07 DEBUG { 06:35:07 DEBUG 06:35:07 DEBUG }, 06:35:07 DEBUG "value": [ 06:35:07 DEBUG 06:35:07 DEBUG 06:35:07 DEBUG "instance": "10.142.0.107:9100" 06:35:07 DEBUG 06:35:07 DEBUG 06:35:07 DEBUG "value": [ 06:35:07 DEBUG { 06:35:07 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 73m 0% 6263Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7153m 23% 7388Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 14052m 47% 11320Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 77m 0% 3394Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 7926m 26% 13575Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 152m 0% 5215Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 14299m 47% 11550Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 17656m 59% 3086Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 15871m 53% 10822Mi 9% gke-xlou-cdm-ds-0f601751-j488 7332m 24% 7004Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 7576m 25% 15636Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 7264m 24% 7612Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 8061m 27% 12681Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 8178m 51% 6735Mi 11% 06:35:07 DEBUG 06:35:07 DEBUG }, 06:35:07 DEBUG "metric": { 06:35:07 DEBUG 06:35:07 DEBUG "pod": "ds-idrepo-0" 06:35:07 DEBUG "pod": "ds-cts-2" 06:35:07 DEBUG "pod": "am-767f6b67b5-9zw22" 06:35:07 DEBUG [...] 06:35:07 DEBUG [...] 06:35:07 DEBUG "pod": "ds-cts-2" 06:35:07 DEBUG 06:35:07 DEBUG "value": [ 06:35:07 DEBUG "metric": { 06:35:07 DEBUG 06:35:07 DEBUG "value": [ 06:35:07 DEBUG 06:35:07 DEBUG 06:35:07 DEBUG "pod": "am-767f6b67b5-vbltz" 06:35:07 DEBUG }, 06:35:07 DEBUG }, 06:35:07 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 06:35:07 DEBUG 06:35:07 DEBUG "metric": { 06:35:07 DEBUG --- stderr --- 06:35:07 DEBUG 06:35:07 DEBUG "value": [ 06:35:07 DEBUG "instance": "10.142.0.107:9100" 06:35:07 DEBUG 06:35:07 DEBUG }, 06:35:07 DEBUG }, 06:35:07 DEBUG }, 06:35:07 DEBUG 06:35:07 DEBUG 06:35:07 DEBUG }, 06:35:07 DEBUG "pod": "am-767f6b67b5-vbltz" 06:35:07 DEBUG 06:35:07 DEBUG "pod": "am-767f6b67b5-9zw22" 06:35:07 DEBUG 06:35:07 DEBUG 06:35:07 DEBUG 06:35:07 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 06:35:07 DEBUG }, 06:35:07 DEBUG "value": [ 06:35:07 DEBUG "value": [ 06:35:07 DEBUG }, 06:35:07 DEBUG 06:35:07 DEBUG "pod": "am-767f6b67b5-9zw22" 06:35:07 DEBUG [...] 06:35:07 DEBUG 06:35:07 DEBUG 06:35:07 DEBUG }, 06:35:07 DEBUG "instance": "10.142.0.43:9100" 06:35:07 DEBUG "value": [ 06:35:07 DEBUG "value": [ 06:35:07 DEBUG "value": [ 06:35:07 DEBUG 06:35:07 DEBUG 06:35:07 DEBUG "value": [ 06:35:07 DEBUG }, 06:35:07 DEBUG 06:35:07 DEBUG }, 06:35:07 DEBUG "pod": "ds-idrepo-2" 06:35:07 DEBUG 06:35:07 DEBUG 06:35:07 DEBUG }, 06:35:07 DEBUG "value": [ 06:35:07 DEBUG 06:35:07 DEBUG 06:35:07 DEBUG "value": [ 06:35:07 DEBUG 06:35:07 DEBUG }, 06:35:07 DEBUG 06:35:07 DEBUG 06:35:07 DEBUG "value": [ 06:35:07 DEBUG }, 06:35:07 DEBUG 06:35:07 DEBUG 1713591307, 06:35:07 DEBUG 06:35:07 DEBUG 06:35:07 DEBUG 06:35:07 DEBUG 1713591307, 06:35:07 DEBUG "0" 06:35:07 DEBUG 06:35:07 DEBUG "value": [ 06:35:07 DEBUG }, 06:35:07 DEBUG 06:35:07 DEBUG [...] 06:35:07 DEBUG "value": [ 06:35:07 DEBUG 1713591307, 06:35:07 DEBUG 06:35:07 DEBUG 06:35:07 DEBUG 1713591307, 06:35:07 DEBUG [...] 06:35:07 DEBUG "value": [ 06:35:07 DEBUG 06:35:07 DEBUG 06:35:07 DEBUG 06:35:07 DEBUG "value": [ 06:35:07 DEBUG 06:35:07 DEBUG "0" 06:35:07 DEBUG 06:35:07 DEBUG "pod": "am-767f6b67b5-vbltz" 06:35:07 DEBUG "instance": "10.142.0.43:9100" 06:35:07 DEBUG "value": [ 06:35:07 DEBUG 1713591307, 06:35:07 DEBUG [...] 06:35:07 DEBUG 06:35:07 DEBUG "value": [ 06:35:07 DEBUG [...] 06:35:07 DEBUG 06:35:07 DEBUG 1713591307, 06:35:07 DEBUG "0" 06:35:07 DEBUG 06:35:07 DEBUG 06:35:07 DEBUG "3109691392" 06:35:07 DEBUG 06:35:07 DEBUG 06:35:07 DEBUG 06:35:07 DEBUG [...] 06:35:07 DEBUG 06:35:07 DEBUG 1713591307, 06:35:07 DEBUG 06:35:07 DEBUG ] 06:35:07 DEBUG 06:35:07 DEBUG }, 06:35:07 DEBUG }, 06:35:07 DEBUG ] 06:35:07 DEBUG } 06:35:07 DEBUG 06:35:07 DEBUG 06:35:07 DEBUG 1713591307, 06:35:07 DEBUG 06:35:07 DEBUG 06:35:07 DEBUG "8.045982859971677" 06:35:07 DEBUG ] 06:35:07 DEBUG [...] 06:35:07 DEBUG [...] 06:35:07 DEBUG ] 06:35:07 DEBUG 06:35:07 DEBUG 06:35:07 DEBUG "pod": "ds-idrepo-2" 06:35:07 DEBUG 06:35:07 DEBUG 06:35:07 DEBUG "0.03700000000000045" 06:35:07 DEBUG [...] 06:35:07 DEBUG } 06:35:07 DEBUG [...] 06:35:07 DEBUG "value": [ 06:35:07 DEBUG "value": [ 06:35:07 DEBUG "0" 06:35:07 DEBUG ] 06:35:07 DEBUG 06:35:07 DEBUG 06:35:07 DEBUG "129062547.78456534" 06:35:07 DEBUG 06:35:07 DEBUG 06:35:07 DEBUG ] 06:35:07 DEBUG } 06:35:07 DEBUG 06:35:07 DEBUG 06:35:07 DEBUG } 06:35:07 DEBUG 06:35:07 DEBUG 06:35:07 DEBUG }, 06:35:07 DEBUG 06:35:07 DEBUG [...] 06:35:07 DEBUG ] 06:35:07 DEBUG 06:35:07 DEBUG ] 06:35:07 DEBUG 06:35:07 DEBUG 1713591307, 06:35:07 DEBUG 1713591307, 06:35:07 DEBUG ] 06:35:07 DEBUG } 06:35:07 DEBUG 06:35:07 DEBUG [...] 06:35:07 DEBUG ] 06:35:07 DEBUG 06:35:07 DEBUG "pod": "ds-idrepo-2" 06:35:07 DEBUG } 06:35:07 DEBUG ] 06:35:07 DEBUG 06:35:07 DEBUG 06:35:07 DEBUG ] 06:35:07 DEBUG "pod": "ds-cts-2" 06:35:07 DEBUG [...] 06:35:07 DEBUG "value": [ 06:35:07 DEBUG 06:35:07 DEBUG 06:35:07 DEBUG } 06:35:07 DEBUG 06:35:07 DEBUG } 06:35:07 DEBUG 06:35:07 DEBUG "2670.6666666666665" 06:35:07 DEBUG "0.05296666666662399" 06:35:07 DEBUG ] 06:35:07 DEBUG } 06:35:07 DEBUG "pod": "ds-idrepo-2" 06:35:07 DEBUG 06:35:07 DEBUG } 06:35:07 DEBUG "instance": "10.142.0.43:9100" 06:35:07 DEBUG }, 06:35:07 DEBUG ] 06:35:07 DEBUG } 06:35:07 DEBUG 06:35:07 DEBUG 06:35:07 DEBUG } 06:35:07 DEBUG }, 06:35:07 DEBUG 06:35:07 DEBUG 1713591307, 06:35:07 DEBUG "pod": "ds-cts-2" 06:35:07 DEBUG 06:35:07 DEBUG ] 06:35:07 DEBUG 06:35:07 DEBUG } 06:35:07 DEBUG 06:35:07 DEBUG ] 06:35:07 DEBUG } 06:35:07 DEBUG } 06:35:07 DEBUG }, 06:35:07 DEBUG 06:35:07 DEBUG ] 06:35:07 DEBUG }, 06:35:07 DEBUG "value": [ 06:35:07 DEBUG } 06:35:07 DEBUG } 06:35:07 DEBUG "instance": "10.142.0.43:9100" 06:35:07 DEBUG "pod": "ds-idrepo-2" 06:35:07 DEBUG }, 06:35:07 DEBUG "value": [ 06:35:07 DEBUG 06:35:07 DEBUG "0" 06:35:07 DEBUG }, 06:35:07 DEBUG 06:35:07 DEBUG } 06:35:07 DEBUG "pod": "ds-idrepo-2" 06:35:07 DEBUG "pod": "am-767f6b67b5-vbltz" 06:35:07 DEBUG } 06:35:07 DEBUG ] 06:35:07 DEBUG ] 06:35:07 DEBUG "value": [ 06:35:07 DEBUG 06:35:07 DEBUG } 06:35:07 DEBUG "value": [ 06:35:07 DEBUG 1713591307, 06:35:07 DEBUG } 06:35:07 DEBUG }, 06:35:07 DEBUG } 06:35:07 DEBUG "value": [ 06:35:07 DEBUG 1713591307, 06:35:07 DEBUG 06:35:07 DEBUG ] 06:35:07 DEBUG "value": [ 06:35:07 DEBUG "instance": "10.142.0.43:9100" 06:35:07 DEBUG } 06:35:07 DEBUG }, 06:35:07 DEBUG }, 06:35:07 DEBUG ] 06:35:07 DEBUG } 06:35:07 DEBUG } 06:35:07 DEBUG 1713591307, 06:35:07 DEBUG "pod": "am-767f6b67b5-vbltz" 06:35:07 DEBUG } 06:35:07 DEBUG 1713591307, 06:35:07 DEBUG "0" 06:35:07 DEBUG "value": [ 06:35:07 DEBUG 1713591307, 06:35:07 DEBUG "0" 06:35:07 DEBUG "pod": "am-767f6b67b5-vbltz" 06:35:07 DEBUG } 06:35:07 DEBUG 1713591307, 06:35:07 DEBUG }, 06:35:07 DEBUG "value": [ 06:35:07 DEBUG "value": [ 06:35:07 DEBUG } 06:35:07 DEBUG } 06:35:07 DEBUG } 06:35:07 DEBUG "0" 06:35:07 DEBUG }, 06:35:07 DEBUG "63766802.400000006" 06:35:07 DEBUG ] 06:35:07 DEBUG 1713591307, 06:35:07 DEBUG "3.6028599367309253" 06:35:07 DEBUG ] 06:35:07 DEBUG }, 06:35:07 DEBUG ] 06:35:07 DEBUG "0" 06:35:07 DEBUG "value": [ 06:35:07 DEBUG 1713591307, 06:35:07 DEBUG 1713591307, 06:35:07 DEBUG } 06:35:07 DEBUG ] 06:35:07 DEBUG "value": [ 06:35:07 DEBUG ] 06:35:07 DEBUG } 06:35:07 DEBUG "63763419.16666667" 06:35:07 DEBUG ] 06:35:07 DEBUG } 06:35:07 DEBUG "value": [ 06:35:07 DEBUG } 06:35:07 DEBUG ] 06:35:07 DEBUG 1713591307, 06:35:07 DEBUG "0" 06:35:07 DEBUG "0.094" 06:35:07 DEBUG } 06:35:07 DEBUG 1713591307, 06:35:07 DEBUG } 06:35:07 DEBUG ] 06:35:07 DEBUG ] 06:35:07 DEBUG } 06:35:07 DEBUG ] 06:35:07 DEBUG 1713591307, 06:35:07 DEBUG } 06:35:07 DEBUG } 06:35:07 DEBUG "3163602344960" 06:35:07 DEBUG ] 06:35:07 DEBUG ] 06:35:07 DEBUG ] 06:35:07 DEBUG "2671.2" 06:35:07 DEBUG ] 06:35:07 DEBUG } 06:35:07 DEBUG } 06:35:07 DEBUG ] 06:35:07 DEBUG } 06:35:07 DEBUG "2671.0333333333333" 06:35:07 DEBUG ] 06:35:07 DEBUG ] 06:35:07 DEBUG } 06:35:07 DEBUG } 06:35:07 DEBUG } 06:35:07 DEBUG ] 06:35:07 DEBUG } 06:35:07 DEBUG } 06:35:07 DEBUG ] 06:35:07 DEBUG } 06:35:07 DEBUG } 06:35:07 DEBUG ] 06:35:07 DEBUG } 06:35:07 DEBUG } 06:35:07 DEBUG ] 06:35:07 DEBUG ] 06:35:07 DEBUG } 06:35:07 DEBUG } 06:35:07 DEBUG ] 06:35:07 DEBUG } 06:35:07 DEBUG } 06:35:07 DEBUG } 06:35:07 DEBUG } 06:35:07 DEBUG ] 06:35:07 DEBUG } 06:35:07 DEBUG } 06:35:07 DEBUG } 06:35:07 DEBUG } 06:35:07 DEBUG ] 06:35:07 DEBUG } 06:35:07 DEBUG } 06:35:07 DEBUG } 06:35:07 DEBUG } 06:35:07 DEBUG } 06:35:07 DEBUG } 06:35:07 DEBUG } 06:35:07 DEBUG } 06:36:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591367" 06:36:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591367" 06:36:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591367" 06:36:07 INFO [http_cmd]: http status code OK 06:36:07 DEBUG --- status code --- 06:36:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:36:07 DEBUG --- http response --- 06:36:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 06:36:07 DEBUG ----- output ----- 06:36:07 DEBUG { 06:36:07 DEBUG "status": "success", 06:36:07 DEBUG "data": { 06:36:07 DEBUG "resultType": "vector", 06:36:07 DEBUG "result": [ 06:36:07 DEBUG { 06:36:07 DEBUG "metric": { 06:36:07 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:36:07 DEBUG }, 06:36:07 DEBUG "value": [ 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG [...] 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 06:36:07 DEBUG }, 06:36:07 DEBUG "value": [ 06:36:07 DEBUG 1713591367, 06:36:07 DEBUG "0" 06:36:07 DEBUG ] 06:36:07 DEBUG } 06:36:07 DEBUG ] 06:36:07 DEBUG } 06:36:07 DEBUG } 06:36:07 INFO [http_cmd]: http status code OK 06:36:07 DEBUG --- status code --- 06:36:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:36:07 DEBUG --- http response --- 06:36:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:36:07 DEBUG ----- output ----- 06:36:07 DEBUG { 06:36:07 DEBUG "status": "success", 06:36:07 DEBUG "data": { 06:36:07 DEBUG "resultType": "vector", 06:36:07 DEBUG "result": [ 06:36:07 DEBUG { 06:36:07 DEBUG "metric": { 06:36:07 DEBUG "pod": "am-767f6b67b5-9zw22" 06:36:07 DEBUG }, 06:36:07 DEBUG "value": [ 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG [...] 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG "pod": "am-767f6b67b5-vbltz" 06:36:07 DEBUG }, 06:36:07 DEBUG "value": [ 06:36:07 DEBUG 1713591367, 06:36:07 DEBUG "296802" 06:36:07 DEBUG ] 06:36:07 DEBUG } 06:36:07 DEBUG ] 06:36:07 DEBUG } 06:36:07 DEBUG } 06:36:07 INFO [http_cmd]: http status code OK 06:36:07 DEBUG --- status code --- 06:36:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:36:07 DEBUG --- http response --- 06:36:07 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 06:36:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713591367" 06:36:07 ERROR [http_cmd]: ERROR 06:36:07 ERROR --- status code --- 06:36:07 ERROR http status code is 400 (expected [200, 401, 403, 404]) 06:36:07 ERROR --- http response --- 06:36:07 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 06:36:07 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 06:36:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591367" 06:36:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713591367" 06:36:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591367" 06:36:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591367" 06:36:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591367" 06:36:07 INFO [http_cmd]: http status code OK 06:36:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591367" 06:36:07 DEBUG --- status code --- 06:36:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:36:07 DEBUG --- http response --- 06:36:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:36:07 DEBUG ----- output ----- 06:36:07 DEBUG { 06:36:07 DEBUG "status": "success", 06:36:07 DEBUG "data": { 06:36:07 DEBUG "resultType": "vector", 06:36:07 DEBUG "result": [ 06:36:07 DEBUG { 06:36:07 DEBUG "metric": { 06:36:07 DEBUG "pod": "ds-idrepo-0" 06:36:07 DEBUG }, 06:36:07 DEBUG "value": [ 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG [...] 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG "pod": "ds-idrepo-2" 06:36:07 DEBUG }, 06:36:07 DEBUG "value": [ 06:36:07 DEBUG 1713591367, 06:36:07 DEBUG "0" 06:36:07 DEBUG ] 06:36:07 DEBUG } 06:36:07 DEBUG ] 06:36:07 DEBUG } 06:36:07 DEBUG } 06:36:07 INFO [http_cmd]: http status code OK 06:36:07 DEBUG --- status code --- 06:36:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:36:07 DEBUG --- http response --- 06:36:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:36:07 DEBUG ----- output ----- 06:36:07 DEBUG { 06:36:07 DEBUG "status": "success", 06:36:07 DEBUG "data": { 06:36:07 DEBUG "resultType": "vector", 06:36:07 DEBUG "result": [ 06:36:07 DEBUG { 06:36:07 DEBUG "metric": { 06:36:07 DEBUG "instance": "10.142.0.107:9100" 06:36:07 DEBUG }, 06:36:07 DEBUG "value": [ 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG [...] 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG "instance": "10.142.0.43:9100" 06:36:07 DEBUG }, 06:36:07 DEBUG "value": [ 06:36:07 DEBUG 1713591367, 06:36:07 DEBUG "3166494231552" 06:36:07 DEBUG ] 06:36:07 DEBUG } 06:36:07 DEBUG ] 06:36:07 DEBUG } 06:36:07 DEBUG } 06:36:07 INFO [http_cmd]: http status code OK 06:36:07 DEBUG --- status code --- 06:36:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:36:07 DEBUG --- http response --- 06:36:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:36:07 DEBUG ----- output ----- 06:36:07 DEBUG { 06:36:07 DEBUG "status": "success", 06:36:07 DEBUG "data": { 06:36:07 DEBUG "resultType": "vector", 06:36:07 DEBUG "result": [ 06:36:07 DEBUG { 06:36:07 DEBUG "metric": { 06:36:07 DEBUG "pod": "am-767f6b67b5-9zw22" 06:36:07 DEBUG }, 06:36:07 DEBUG "value": [ 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG [...] 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG "pod": "am-767f6b67b5-vbltz" 06:36:07 DEBUG }, 06:36:07 DEBUG "value": [ 06:36:07 DEBUG 1713591367, 06:36:07 DEBUG "2664.133333333333" 06:36:07 DEBUG ] 06:36:07 DEBUG } 06:36:07 DEBUG ] 06:36:07 DEBUG } 06:36:07 DEBUG } 06:36:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713591367" 06:36:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713591367" 06:36:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591367" 06:36:07 INFO [http_cmd]: http status code OK 06:36:07 INFO 06:36:07 INFO [http_cmd]: http status code OK 06:36:07 DEBUG --- status code --- 06:36:07 INFO [http_cmd]: http status code OK 06:36:07 INFO [loop_until]: kubectl --namespace=xlou top pods 06:36:07 DEBUG --- status code --- 06:36:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:36:07 DEBUG --- status code --- 06:36:07 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:36:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:36:07 DEBUG --- http response --- 06:36:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:36:07 DEBUG --- http response --- 06:36:07 DEBUG --- http response --- 06:36:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:36:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:36:07 DEBUG ----- output ----- 06:36:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:36:07 DEBUG ----- output ----- 06:36:07 DEBUG { 06:36:07 DEBUG ----- output ----- 06:36:07 DEBUG { 06:36:07 DEBUG "status": "success", 06:36:07 DEBUG { 06:36:07 DEBUG "status": "success", 06:36:07 DEBUG "data": { 06:36:07 INFO [http_cmd]: http status code OK 06:36:07 DEBUG "status": "success", 06:36:07 INFO [http_cmd]: http status code OK 06:36:07 DEBUG "data": { 06:36:07 DEBUG "resultType": "vector", 06:36:07 DEBUG --- status code --- 06:36:07 DEBUG "data": { 06:36:07 DEBUG --- status code --- 06:36:07 DEBUG "resultType": "vector", 06:36:07 DEBUG "result": [ 06:36:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:36:07 DEBUG "resultType": "vector", 06:36:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:36:07 DEBUG "result": [ 06:36:07 DEBUG { 06:36:07 DEBUG --- http response --- 06:36:07 DEBUG "result": [ 06:36:07 DEBUG --- http response --- 06:36:07 DEBUG { 06:36:07 DEBUG "metric": { 06:36:07 DEBUG { 06:36:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:36:07 DEBUG "metric": { 06:36:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:36:07 INFO 06:36:07 DEBUG "pod": "ds-cts-0" 06:36:07 DEBUG "metric": { 06:36:07 INFO [http_cmd]: http status code OK 06:36:07 DEBUG ----- output ----- 06:36:07 DEBUG "pod": "ds-cts-0" 06:36:07 DEBUG ----- output ----- 06:36:07 INFO [loop_until]: kubectl --namespace=xlou top node 06:36:07 DEBUG }, 06:36:07 DEBUG "pod": "ds-idrepo-0" 06:36:07 DEBUG --- status code --- 06:36:07 DEBUG { 06:36:07 DEBUG }, 06:36:07 DEBUG { 06:36:07 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:36:07 DEBUG "value": [ 06:36:07 DEBUG }, 06:36:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:36:07 DEBUG "status": "success", 06:36:07 DEBUG "value": [ 06:36:07 DEBUG "status": "success", 06:36:07 DEBUG 06:36:07 DEBUG "value": [ 06:36:07 DEBUG --- http response --- 06:36:07 DEBUG "data": { 06:36:07 DEBUG 06:36:07 DEBUG "data": { 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 06:36:07 DEBUG "resultType": "vector", 06:36:07 DEBUG 06:36:07 DEBUG "resultType": "vector", 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG ----- output ----- 06:36:07 DEBUG "result": [ 06:36:07 DEBUG 06:36:07 DEBUG "result": [ 06:36:07 DEBUG [...] 06:36:07 DEBUG 06:36:07 DEBUG { 06:36:07 DEBUG { 06:36:07 DEBUG [...] 06:36:07 DEBUG { 06:36:07 DEBUG 06:36:07 DEBUG [...] 06:36:07 DEBUG "status": "success", 06:36:07 DEBUG "metric": { 06:36:07 DEBUG 06:36:07 DEBUG "metric": { 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG "data": { 06:36:07 DEBUG "pod": "ds-idrepo-0" 06:36:07 DEBUG 06:36:07 DEBUG "instance": "10.142.0.107:9100" 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG "resultType": "vector", 06:36:07 DEBUG }, 06:36:07 DEBUG 06:36:07 DEBUG }, 06:36:07 DEBUG "pod": "ds-cts-2" 06:36:07 DEBUG 06:36:07 DEBUG "result": [ 06:36:07 DEBUG "value": [ 06:36:07 DEBUG "pod": "ds-cts-2" 06:36:07 DEBUG "value": [ 06:36:07 DEBUG }, 06:36:07 DEBUG "pod": "ds-idrepo-2" 06:36:07 DEBUG { 06:36:07 DEBUG 06:36:07 DEBUG }, 06:36:07 DEBUG 06:36:07 DEBUG "value": [ 06:36:07 DEBUG }, 06:36:07 DEBUG "metric": { 06:36:07 DEBUG 06:36:07 DEBUG "value": [ 06:36:07 DEBUG 06:36:07 DEBUG 1713591367, 06:36:07 DEBUG "value": [ 06:36:07 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:36:07 DEBUG 06:36:07 DEBUG 1713591367, 06:36:07 DEBUG 06:36:07 DEBUG "0" 06:36:07 DEBUG 1713591367, 06:36:07 DEBUG }, 06:36:07 DEBUG [...] 06:36:07 DEBUG "0" 06:36:07 DEBUG [...] 06:36:07 DEBUG ] 06:36:07 DEBUG "0" 06:36:07 DEBUG "value": [ 06:36:07 DEBUG 06:36:07 DEBUG ] 06:36:07 DEBUG 06:36:07 DEBUG } 06:36:07 DEBUG ] 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG } 06:36:07 DEBUG 06:36:07 DEBUG ] 06:36:07 DEBUG } 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG ] 06:36:07 DEBUG 06:36:07 DEBUG } 06:36:07 DEBUG ] 06:36:07 DEBUG 06:36:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591367" 06:36:07 DEBUG "pod": "ds-cts-2" 06:36:07 DEBUG } 06:36:07 DEBUG "instance": "10.142.0.43:9100" 06:36:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713591367" 06:36:07 DEBUG } 06:36:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713591367" 06:36:07 DEBUG } 06:36:07 DEBUG [...] 06:36:07 DEBUG }, 06:36:07 DEBUG } 06:36:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591367" 06:36:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713591367" 06:36:07 DEBUG }, 06:36:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591367" 06:36:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713591367" 06:36:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591367" 06:36:07 DEBUG } 06:36:07 DEBUG 06:36:07 DEBUG "value": [ 06:36:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591367" 06:36:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591367" 06:36:07 DEBUG "value": [ 06:36:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591367" 06:36:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713591367" 06:36:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591367" 06:36:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713591367" 06:36:07 INFO [http_cmd]: http status code OK 06:36:07 DEBUG 06:36:07 DEBUG 1713591367, 06:36:07 INFO [http_cmd]: http status code OK 06:36:07 INFO [http_cmd]: http status code OK 06:36:07 INFO [http_cmd]: http status code OK 06:36:07 INFO [http_cmd]: http status code OK 06:36:07 DEBUG 1713591367, 06:36:07 INFO [http_cmd]: http status code OK 06:36:07 INFO [http_cmd]: http status code OK 06:36:07 DEBUG --- status code --- 06:36:07 INFO [http_cmd]: http status code OK 06:36:07 DEBUG 06:36:07 INFO [http_cmd]: http status code OK 06:36:07 DEBUG "0" 06:36:07 DEBUG --- status code --- 06:36:07 DEBUG --- status code --- 06:36:07 DEBUG --- status code --- 06:36:07 DEBUG --- status code --- 06:36:07 INFO [http_cmd]: http status code OK 06:36:07 DEBUG "63655031.96666667" 06:36:07 INFO [http_cmd]: http status code OK 06:36:07 DEBUG --- status code --- 06:36:07 DEBUG --- status code --- 06:36:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:36:07 DEBUG --- status code --- 06:36:07 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 06:36:07 DEBUG --- status code --- 06:36:07 DEBUG ] 06:36:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:36:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:36:07 INFO [http_cmd]: http status code OK 06:36:07 INFO [http_cmd]: http status code OK 06:36:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:36:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:36:07 DEBUG --- status code --- 06:36:07 DEBUG ] 06:36:07 DEBUG --- status code --- 06:36:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:36:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:36:07 DEBUG --- http response --- 06:36:07 DEBUG --- http response --- 06:36:07 DEBUG }, 06:36:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:36:07 DEBUG } 06:36:07 DEBUG --- http response --- 06:36:07 DEBUG --- http response --- 06:36:07 DEBUG --- status code --- 06:36:07 DEBUG --- status code --- 06:36:07 DEBUG --- http response --- 06:36:07 DEBUG --- http response --- 06:36:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:36:07 DEBUG } 06:36:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:36:07 DEBUG --- http response --- 06:36:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:36:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:36:07 DEBUG "value": [ 06:36:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:36:07 DEBUG --- http response --- 06:36:07 DEBUG ] 06:36:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:36:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:36:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:36:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:36:07 INFO [http_cmd]: http status code OK 06:36:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:36:07 DEBUG --- http response --- 06:36:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 06:36:07 DEBUG ] 06:36:07 DEBUG --- http response --- 06:36:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 06:36:07 DEBUG --- http response --- 06:36:07 DEBUG ----- output ----- 06:36:07 DEBUG 1713591367, 06:36:07 DEBUG ----- output ----- 06:36:07 DEBUG } 06:36:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:36:07 DEBUG ----- output ----- 06:36:07 DEBUG --- http response --- 06:36:07 DEBUG ----- output ----- 06:36:07 DEBUG --- http response --- 06:36:07 DEBUG --- status code --- 06:36:07 DEBUG ----- output ----- 06:36:07 DEBUG ----- output ----- 06:36:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:36:07 DEBUG } 06:36:07 DEBUG ----- output ----- 06:36:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:36:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:36:07 DEBUG { 06:36:07 DEBUG "0" 06:36:07 DEBUG { 06:36:07 DEBUG } 06:36:07 DEBUG ----- output ----- 06:36:07 DEBUG { 06:36:07 DEBUG { 06:36:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:36:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:36:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:36:07 DEBUG { 06:36:07 DEBUG { 06:36:07 DEBUG ----- output ----- 06:36:07 INFO [loop_until]: OK (rc = 0) 06:36:07 DEBUG } 06:36:07 DEBUG { 06:36:07 DEBUG ----- output ----- 06:36:07 DEBUG ----- output ----- 06:36:07 DEBUG "status": "success", 06:36:07 DEBUG ] 06:36:07 DEBUG "status": "success", 06:36:07 DEBUG { 06:36:07 DEBUG "status": "success", 06:36:07 DEBUG "status": "success", 06:36:07 DEBUG ----- output ----- 06:36:07 DEBUG ----- output ----- 06:36:07 DEBUG --- http response --- 06:36:07 DEBUG "status": "success", 06:36:07 DEBUG "status": "success", 06:36:07 DEBUG { 06:36:07 DEBUG --- stdout --- 06:36:07 DEBUG "status": "success", 06:36:07 DEBUG { 06:36:07 DEBUG { 06:36:07 DEBUG "data": { 06:36:07 DEBUG } 06:36:07 DEBUG "data": { 06:36:07 DEBUG "status": "success", 06:36:07 DEBUG "data": { 06:36:07 DEBUG "data": { 06:36:07 DEBUG { 06:36:07 DEBUG { 06:36:07 DEBUG "data": { 06:36:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:36:07 DEBUG "data": { 06:36:07 DEBUG "status": "success", 06:36:07 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 14287m 8619Mi am-767f6b67b5-qcqz2 14062m 8677Mi am-767f6b67b5-vbltz 15063m 8625Mi ds-cts-0 8281m 10403Mi ds-cts-1 7535m 10680Mi ds-cts-2 8326m 10050Mi ds-idrepo-0 6775m 3728Mi ds-idrepo-1 6766m 3731Mi ds-idrepo-2 7116m 3695Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1177Mi idm-b85f9d978-wnhxb 3m 1236Mi lodemon-7cb5b9456d-dskfn 1m 62Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 8047m 2895Mi 06:36:07 DEBUG "data": { 06:36:07 DEBUG "status": "success", 06:36:07 DEBUG "status": "success", 06:36:07 DEBUG "resultType": "vector", 06:36:07 DEBUG ] 06:36:07 DEBUG "resultType": "vector", 06:36:07 DEBUG "data": { 06:36:07 DEBUG "resultType": "vector", 06:36:07 DEBUG "resultType": "vector", 06:36:07 DEBUG "status": "success", 06:36:07 DEBUG "status": "success", 06:36:07 DEBUG "resultType": "vector", 06:36:07 DEBUG ----- output ----- 06:36:07 DEBUG "resultType": "vector", 06:36:07 DEBUG "data": { 06:36:07 DEBUG --- stderr --- 06:36:07 DEBUG "resultType": "vector", 06:36:07 DEBUG "data": { 06:36:07 DEBUG "data": { 06:36:07 DEBUG "result": [ 06:36:07 DEBUG } 06:36:07 DEBUG "result": [ 06:36:07 DEBUG "resultType": "vector", 06:36:07 DEBUG "result": [ 06:36:07 DEBUG "result": [ 06:36:07 DEBUG "data": { 06:36:07 DEBUG "data": { 06:36:07 DEBUG "result": [ 06:36:07 DEBUG { 06:36:07 DEBUG "result": [ 06:36:07 DEBUG "resultType": "vector", 06:36:07 DEBUG 06:36:07 DEBUG "result": [ 06:36:07 DEBUG "resultType": "vector", 06:36:07 DEBUG "resultType": "vector", 06:36:07 DEBUG { 06:36:07 DEBUG } 06:36:07 DEBUG { 06:36:07 DEBUG "result": [ 06:36:07 DEBUG { 06:36:07 DEBUG { 06:36:07 DEBUG "resultType": "vector", 06:36:07 DEBUG "resultType": "vector", 06:36:07 DEBUG { 06:36:07 DEBUG "status": "success", 06:36:07 DEBUG { 06:36:07 DEBUG "result": [ 06:36:07 DEBUG { 06:36:07 DEBUG "result": [ 06:36:07 DEBUG "result": [ 06:36:07 DEBUG "metric": { 06:36:07 DEBUG "metric": { 06:36:07 DEBUG { 06:36:07 DEBUG "metric": { 06:36:07 DEBUG "metric": { 06:36:07 DEBUG "result": [ 06:36:07 DEBUG "result": [ 06:36:07 DEBUG "metric": { 06:36:07 DEBUG "data": { 06:36:07 DEBUG "metric": { 06:36:07 DEBUG { 06:36:07 DEBUG "metric": { 06:36:07 DEBUG { 06:36:07 DEBUG { 06:36:07 DEBUG "instance": "10.142.0.107:9100" 06:36:07 DEBUG "pod": "am-767f6b67b5-9zw22" 06:36:07 DEBUG "metric": { 06:36:07 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 06:36:07 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 06:36:07 DEBUG { 06:36:07 DEBUG { 06:36:07 DEBUG "instance": "10.142.0.107:9100" 06:36:07 DEBUG "resultType": "vector", 06:36:07 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:36:07 DEBUG "metric": { 06:36:07 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:36:07 DEBUG "metric": { 06:36:07 DEBUG "metric": { 06:36:07 DEBUG }, 06:36:07 DEBUG }, 06:36:07 DEBUG "pod": "ds-idrepo-0" 06:36:07 DEBUG }, 06:36:07 DEBUG }, 06:36:07 DEBUG "metric": { 06:36:07 DEBUG "metric": { 06:36:07 DEBUG }, 06:36:07 DEBUG "result": [ 06:36:07 DEBUG }, 06:36:07 DEBUG "pod": "ds-cts-0" 06:36:07 DEBUG }, 06:36:07 DEBUG "pod": "am-767f6b67b5-9zw22" 06:36:07 DEBUG "pod": "am-767f6b67b5-9zw22" 06:36:07 DEBUG "value": [ 06:36:07 DEBUG "value": [ 06:36:07 DEBUG }, 06:36:07 DEBUG "value": [ 06:36:07 DEBUG "value": [ 06:36:07 DEBUG "pod": "am-767f6b67b5-9zw22" 06:36:07 DEBUG "instance": "10.142.0.107:9100" 06:36:07 DEBUG "value": [ 06:36:07 DEBUG { 06:36:07 DEBUG "value": [ 06:36:07 DEBUG }, 06:36:07 DEBUG "value": [ 06:36:07 DEBUG }, 06:36:07 DEBUG }, 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG "value": [ 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG }, 06:36:07 DEBUG }, 06:36:07 DEBUG 06:36:07 DEBUG "metric": { 06:36:07 DEBUG 06:36:07 DEBUG "value": [ 06:36:07 DEBUG 06:36:07 DEBUG "value": [ 06:36:07 DEBUG "value": [ 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG "value": [ 06:36:07 DEBUG "value": [ 06:36:07 DEBUG 06:36:07 DEBUG "pod": "am-767f6b67b5-9zw22" 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG }, 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG [...] 06:36:07 DEBUG [...] 06:36:07 DEBUG 06:36:07 DEBUG [...] 06:36:07 DEBUG [...] 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG [...] 06:36:07 DEBUG "value": [ 06:36:07 DEBUG [...] 06:36:07 DEBUG 06:36:07 DEBUG [...] 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG [...] 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG [...] 06:36:07 DEBUG 06:36:07 DEBUG [...] 06:36:07 DEBUG [...] 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG [...] 06:36:07 DEBUG [...] 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG "instance": "10.142.0.43:9100" 06:36:07 DEBUG "pod": "am-767f6b67b5-vbltz" 06:36:07 DEBUG 06:36:07 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 06:36:07 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG "instance": "10.142.0.43:9100" 06:36:07 DEBUG [...] 06:36:07 DEBUG "pod": "ds-idrepo-2" 06:36:07 DEBUG 06:36:07 DEBUG "pod": "ds-idrepo-2" 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG }, 06:36:07 DEBUG }, 06:36:07 DEBUG "pod": "ds-idrepo-2" 06:36:07 DEBUG }, 06:36:07 DEBUG }, 06:36:07 DEBUG 06:36:07 DEBUG 06:36:07 DEBUG }, 06:36:07 DEBUG 06:36:07 DEBUG }, 06:36:07 DEBUG "pod": "ds-cts-2" 06:36:07 DEBUG }, 06:36:07 DEBUG "pod": "am-767f6b67b5-vbltz" 06:36:07 DEBUG "pod": "am-767f6b67b5-vbltz" 06:36:07 DEBUG "value": [ 06:36:07 DEBUG "value": [ 06:36:07 DEBUG }, 06:36:07 DEBUG "value": [ 06:36:07 DEBUG "value": [ 06:36:07 DEBUG "pod": "am-767f6b67b5-vbltz" 06:36:07 DEBUG "instance": "10.142.0.43:9100" 06:36:07 DEBUG "value": [ 06:36:07 DEBUG 06:36:07 DEBUG "value": [ 06:36:07 DEBUG }, 06:36:07 DEBUG "value": [ 06:36:07 DEBUG }, 06:36:07 DEBUG }, 06:36:07 DEBUG 1713591367, 06:36:07 DEBUG 1713591367, 06:36:07 DEBUG "value": [ 06:36:07 DEBUG 1713591367, 06:36:07 DEBUG 1713591367, 06:36:07 DEBUG }, 06:36:07 DEBUG }, 06:36:07 DEBUG 1713591367, 06:36:07 DEBUG 06:36:07 DEBUG 1713591367, 06:36:07 DEBUG "value": [ 06:36:07 DEBUG 1713591367, 06:36:07 DEBUG "value": [ 06:36:07 INFO [loop_until]: OK (rc = 0) 06:36:07 DEBUG --- stdout --- 06:36:07 DEBUG "0.03833333333332689" 06:36:07 DEBUG "0" 06:36:07 DEBUG 1713591367, 06:36:07 DEBUG "8.040513438428487" 06:36:07 DEBUG "3124801536" 06:36:07 DEBUG "value": [ 06:36:07 DEBUG "value": [ 06:36:07 DEBUG "0.05473333333336067" 06:36:07 DEBUG "pod": "am-767f6b67b5-vbltz" 06:36:07 DEBUG "0" 06:36:07 DEBUG 1713591367, 06:36:07 DEBUG "0" 06:36:07 DEBUG 1713591367, 06:36:07 DEBUG "value": [ 06:36:07 DEBUG ] 06:36:07 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 72m 0% 6279Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7127m 23% 7393Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 13931m 46% 11326Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 84m 0% 3382Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 8230m 27% 13572Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 148m 0% 5195Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 14211m 47% 11540Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 17716m 59% 3126Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 15845m 53% 10830Mi 9% gke-xlou-cdm-ds-0f601751-j488 7420m 24% 7007Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 7597m 25% 15633Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 7212m 24% 7635Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 8241m 27% 12692Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 8182m 51% 6746Mi 11% 06:36:07 DEBUG ] 06:36:07 DEBUG "0" 06:36:07 DEBUG ] 06:36:07 DEBUG ] 06:36:07 DEBUG 1713591367, 06:36:07 DEBUG 1713591367, 06:36:07 DEBUG ] 06:36:07 DEBUG }, 06:36:07 DEBUG ] 06:36:07 DEBUG "0.00003333333333333331" 06:36:07 DEBUG ] 06:36:07 DEBUG "0" 06:36:07 DEBUG 1713591367, 06:36:07 DEBUG } 06:36:07 DEBUG --- stderr --- 06:36:07 DEBUG } 06:36:07 DEBUG ] 06:36:07 DEBUG } 06:36:07 DEBUG } 06:36:07 DEBUG "2662.8333333333335" 06:36:07 DEBUG "63649639.233333334" 06:36:07 DEBUG } 06:36:07 DEBUG "value": [ 06:36:07 DEBUG } 06:36:07 DEBUG ] 06:36:07 DEBUG } 06:36:07 DEBUG ] 06:36:07 DEBUG "2663" 06:36:07 DEBUG ] 06:36:07 DEBUG 06:36:07 DEBUG ] 06:36:07 DEBUG } 06:36:07 DEBUG ] 06:36:07 DEBUG ] 06:36:07 DEBUG ] 06:36:07 DEBUG ] 06:36:07 DEBUG ] 06:36:07 DEBUG } 06:36:07 DEBUG ] 06:36:07 DEBUG } 06:36:07 DEBUG ] 06:36:07 DEBUG } 06:36:07 DEBUG ] 06:36:07 DEBUG } 06:36:07 DEBUG } 06:36:07 DEBUG ] 06:36:07 DEBUG } 06:36:07 DEBUG } 06:36:07 DEBUG } 06:36:07 DEBUG } 06:36:07 DEBUG } 06:36:07 DEBUG 1713591367, 06:36:07 DEBUG } 06:36:07 DEBUG ] 06:36:07 DEBUG } 06:36:07 DEBUG ] 06:36:07 DEBUG } 06:36:07 DEBUG } 06:36:07 DEBUG } 06:36:07 DEBUG } 06:36:07 DEBUG } 06:36:07 DEBUG } 06:36:07 DEBUG ] 06:36:07 DEBUG } 06:36:07 DEBUG "0.0945" 06:36:07 DEBUG } 06:36:07 DEBUG } 06:36:07 DEBUG } 06:36:07 DEBUG } 06:36:07 DEBUG ] 06:36:07 DEBUG } 06:36:07 DEBUG ] 06:36:07 DEBUG } 06:36:07 DEBUG ] 06:36:07 DEBUG } 06:36:07 DEBUG } 06:36:07 DEBUG } 06:36:07 DEBUG } 06:36:07 DEBUG } 06:36:07 DEBUG } 06:36:07 DEBUG } 06:36:07 DEBUG ] 06:36:07 DEBUG } 06:36:07 DEBUG } 06:37:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591427" 06:37:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591427" 06:37:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591427" 06:37:07 INFO [http_cmd]: http status code OK 06:37:07 DEBUG --- status code --- 06:37:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:37:07 DEBUG --- http response --- 06:37:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:37:07 DEBUG ----- output ----- 06:37:07 DEBUG { 06:37:07 DEBUG "status": "success", 06:37:07 DEBUG "data": { 06:37:07 DEBUG "resultType": "vector", 06:37:07 DEBUG "result": [ 06:37:07 DEBUG { 06:37:07 DEBUG "metric": { 06:37:07 DEBUG "pod": "am-767f6b67b5-9zw22" 06:37:07 DEBUG }, 06:37:07 DEBUG "value": [ 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG [...] 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG "pod": "am-767f6b67b5-vbltz" 06:37:07 DEBUG }, 06:37:07 DEBUG "value": [ 06:37:07 DEBUG 1713591427, 06:37:07 DEBUG "444409" 06:37:07 DEBUG ] 06:37:07 DEBUG } 06:37:07 DEBUG ] 06:37:07 DEBUG } 06:37:07 DEBUG } 06:37:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713591427" 06:37:07 INFO [http_cmd]: http status code OK 06:37:07 DEBUG --- status code --- 06:37:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:37:07 DEBUG --- http response --- 06:37:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 06:37:07 DEBUG ----- output ----- 06:37:07 DEBUG { 06:37:07 DEBUG "status": "success", 06:37:07 DEBUG "data": { 06:37:07 DEBUG "resultType": "vector", 06:37:07 DEBUG "result": [ 06:37:07 DEBUG { 06:37:07 DEBUG "metric": { 06:37:07 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:37:07 DEBUG }, 06:37:07 DEBUG "value": [ 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG [...] 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 06:37:07 DEBUG }, 06:37:07 DEBUG "value": [ 06:37:07 DEBUG 1713591427, 06:37:07 DEBUG "0" 06:37:07 DEBUG ] 06:37:07 DEBUG } 06:37:07 DEBUG ] 06:37:07 DEBUG } 06:37:07 DEBUG } 06:37:07 INFO [http_cmd]: http status code OK 06:37:07 DEBUG --- status code --- 06:37:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:37:07 DEBUG --- http response --- 06:37:07 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 06:37:07 ERROR [http_cmd]: ERROR 06:37:07 ERROR --- status code --- 06:37:07 ERROR http status code is 400 (expected [200, 401, 403, 404]) 06:37:07 ERROR --- http response --- 06:37:07 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 06:37:07 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 06:37:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591427" 06:37:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713591427" 06:37:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591427" 06:37:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591427" 06:37:07 INFO [http_cmd]: http status code OK 06:37:07 INFO [http_cmd]: http status code OK 06:37:07 DEBUG --- status code --- 06:37:07 DEBUG --- status code --- 06:37:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:37:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:37:07 DEBUG --- http response --- 06:37:07 DEBUG --- http response --- 06:37:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:37:07 DEBUG ----- output ----- 06:37:07 DEBUG { 06:37:07 DEBUG "status": "success", 06:37:07 DEBUG "data": { 06:37:07 DEBUG "resultType": "vector", 06:37:07 DEBUG "result": [ 06:37:07 DEBUG { 06:37:07 DEBUG "metric": { 06:37:07 DEBUG "pod": "ds-idrepo-0" 06:37:07 DEBUG }, 06:37:07 DEBUG "value": [ 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG [...] 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG "pod": "ds-idrepo-2" 06:37:07 DEBUG }, 06:37:07 DEBUG "value": [ 06:37:07 DEBUG 1713591427, 06:37:07 DEBUG "0" 06:37:07 DEBUG ] 06:37:07 DEBUG } 06:37:07 DEBUG ] 06:37:07 INFO [http_cmd]: http status code OK 06:37:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:37:07 DEBUG } 06:37:07 DEBUG --- status code --- 06:37:07 DEBUG ----- output ----- 06:37:07 DEBUG } 06:37:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:37:07 DEBUG { 06:37:07 DEBUG --- http response --- 06:37:07 DEBUG "status": "success", 06:37:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:37:07 DEBUG "data": { 06:37:07 DEBUG ----- output ----- 06:37:07 DEBUG "resultType": "vector", 06:37:07 DEBUG { 06:37:07 DEBUG "result": [ 06:37:07 DEBUG "status": "success", 06:37:07 DEBUG { 06:37:07 DEBUG "data": { 06:37:07 DEBUG "metric": { 06:37:07 DEBUG "resultType": "vector", 06:37:07 DEBUG "instance": "10.142.0.107:9100" 06:37:07 DEBUG "result": [ 06:37:07 DEBUG }, 06:37:07 DEBUG { 06:37:07 DEBUG "value": [ 06:37:07 DEBUG "metric": { 06:37:07 DEBUG 06:37:07 DEBUG "pod": "am-767f6b67b5-9zw22" 06:37:07 DEBUG 06:37:07 DEBUG }, 06:37:07 DEBUG 06:37:07 DEBUG "value": [ 06:37:07 DEBUG [...] 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591427" 06:37:07 DEBUG [...] 06:37:07 DEBUG "instance": "10.142.0.43:9100" 06:37:07 DEBUG 06:37:07 DEBUG }, 06:37:07 DEBUG 06:37:07 DEBUG "value": [ 06:37:07 DEBUG 06:37:07 DEBUG 1713591427, 06:37:07 DEBUG "pod": "am-767f6b67b5-vbltz" 06:37:07 DEBUG "3169404052480" 06:37:07 DEBUG }, 06:37:07 DEBUG ] 06:37:07 DEBUG "value": [ 06:37:07 DEBUG } 06:37:07 DEBUG 1713591427, 06:37:07 DEBUG ] 06:37:07 DEBUG "2666.1666666666665" 06:37:07 DEBUG } 06:37:07 INFO [http_cmd]: http status code OK 06:37:07 DEBUG ] 06:37:07 DEBUG } 06:37:07 DEBUG --- status code --- 06:37:07 DEBUG } 06:37:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:37:07 DEBUG ] 06:37:07 DEBUG --- http response --- 06:37:07 DEBUG } 06:37:07 DEBUG } 06:37:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:37:07 DEBUG ----- output ----- 06:37:07 DEBUG { 06:37:07 DEBUG "status": "success", 06:37:07 DEBUG "data": { 06:37:07 DEBUG "resultType": "vector", 06:37:07 DEBUG "result": [ 06:37:07 DEBUG { 06:37:07 DEBUG "metric": { 06:37:07 DEBUG "pod": "ds-cts-0" 06:37:07 DEBUG }, 06:37:07 DEBUG "value": [ 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG [...] 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG "pod": "ds-cts-2" 06:37:07 DEBUG }, 06:37:07 DEBUG "value": [ 06:37:07 DEBUG 1713591427, 06:37:07 DEBUG "0" 06:37:07 DEBUG ] 06:37:07 DEBUG } 06:37:07 DEBUG ] 06:37:07 DEBUG } 06:37:07 DEBUG } 06:37:07 INFO [http_cmd]: http status code OK 06:37:07 DEBUG --- status code --- 06:37:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:37:07 DEBUG --- http response --- 06:37:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:37:07 DEBUG ----- output ----- 06:37:07 DEBUG { 06:37:07 DEBUG "status": "success", 06:37:07 DEBUG "data": { 06:37:07 DEBUG "resultType": "vector", 06:37:07 DEBUG "result": [ 06:37:07 DEBUG { 06:37:07 DEBUG "metric": { 06:37:07 DEBUG "pod": "ds-idrepo-0" 06:37:07 DEBUG }, 06:37:07 DEBUG "value": [ 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG [...] 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG "pod": "ds-idrepo-2" 06:37:07 DEBUG }, 06:37:07 DEBUG "value": [ 06:37:07 DEBUG 1713591427, 06:37:07 DEBUG "0" 06:37:07 DEBUG ] 06:37:07 DEBUG } 06:37:07 DEBUG ] 06:37:07 DEBUG } 06:37:07 DEBUG } 06:37:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713591427" 06:37:07 INFO 06:37:07 INFO [loop_until]: kubectl --namespace=xlou top node 06:37:07 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:37:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591427" 06:37:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591427" 06:37:07 INFO [http_cmd]: http status code OK 06:37:07 DEBUG --- status code --- 06:37:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:37:07 DEBUG --- http response --- 06:37:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:37:07 DEBUG ----- output ----- 06:37:07 DEBUG { 06:37:07 DEBUG "status": "success", 06:37:07 DEBUG "data": { 06:37:07 DEBUG "resultType": "vector", 06:37:07 DEBUG "result": [ 06:37:07 DEBUG { 06:37:07 DEBUG "metric": { 06:37:07 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 06:37:07 DEBUG }, 06:37:07 DEBUG "value": [ 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG [...] 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 06:37:07 DEBUG }, 06:37:07 DEBUG "value": [ 06:37:07 DEBUG 1713591427, 06:37:07 DEBUG "8.0263196802854" 06:37:07 DEBUG ] 06:37:07 DEBUG } 06:37:07 DEBUG ] 06:37:07 DEBUG } 06:37:07 DEBUG } 06:37:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713591427" 06:37:07 INFO [http_cmd]: http status code OK 06:37:07 DEBUG --- status code --- 06:37:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:37:07 DEBUG --- http response --- 06:37:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:37:07 DEBUG ----- output ----- 06:37:07 DEBUG { 06:37:07 DEBUG "status": "success", 06:37:07 DEBUG "data": { 06:37:07 DEBUG "resultType": "vector", 06:37:07 DEBUG "result": [ 06:37:07 DEBUG { 06:37:07 DEBUG "metric": { 06:37:07 DEBUG "pod": "ds-cts-0" 06:37:07 DEBUG }, 06:37:07 DEBUG "value": [ 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG [...] 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG "pod": "ds-cts-2" 06:37:07 DEBUG }, 06:37:07 DEBUG "value": [ 06:37:07 DEBUG 1713591427, 06:37:07 DEBUG "0" 06:37:07 DEBUG ] 06:37:07 DEBUG } 06:37:07 DEBUG ] 06:37:07 DEBUG } 06:37:07 DEBUG } 06:37:07 INFO [http_cmd]: http status code OK 06:37:07 DEBUG --- status code --- 06:37:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:37:07 DEBUG --- http response --- 06:37:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 06:37:07 DEBUG ----- output ----- 06:37:07 DEBUG { 06:37:07 DEBUG "status": "success", 06:37:07 DEBUG "data": { 06:37:07 DEBUG "resultType": "vector", 06:37:07 DEBUG "result": [ 06:37:07 DEBUG { 06:37:07 DEBUG "metric": { 06:37:07 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:37:07 DEBUG }, 06:37:07 DEBUG "value": [ 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG [...] 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 06:37:07 DEBUG }, 06:37:07 DEBUG "value": [ 06:37:07 DEBUG 1713591427, 06:37:07 DEBUG "0" 06:37:07 DEBUG ] 06:37:07 DEBUG } 06:37:07 DEBUG ] 06:37:07 DEBUG } 06:37:07 DEBUG } 06:37:07 INFO [http_cmd]: http status code OK 06:37:07 DEBUG --- status code --- 06:37:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:37:07 DEBUG --- http response --- 06:37:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:37:07 DEBUG ----- output ----- 06:37:07 DEBUG { 06:37:07 DEBUG "status": "success", 06:37:07 DEBUG "data": { 06:37:07 DEBUG "resultType": "vector", 06:37:07 DEBUG "result": [ 06:37:07 DEBUG { 06:37:07 DEBUG "metric": { 06:37:07 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 06:37:07 DEBUG }, 06:37:07 DEBUG "value": [ 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG [...] 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 06:37:07 DEBUG }, 06:37:07 DEBUG "value": [ 06:37:07 DEBUG 1713591427, 06:37:07 DEBUG "5026025472" 06:37:07 DEBUG ] 06:37:07 DEBUG } 06:37:07 DEBUG ] 06:37:07 DEBUG } 06:37:07 DEBUG } 06:37:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713591427" 06:37:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713591427" 06:37:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713591427" 06:37:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713591427" 06:37:07 INFO [http_cmd]: http status code OK 06:37:07 DEBUG --- status code --- 06:37:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:37:07 DEBUG --- http response --- 06:37:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:37:07 DEBUG ----- output ----- 06:37:07 DEBUG { 06:37:07 DEBUG "status": "success", 06:37:07 DEBUG "data": { 06:37:07 DEBUG "resultType": "vector", 06:37:07 DEBUG "result": [ 06:37:07 DEBUG { 06:37:07 DEBUG "metric": { 06:37:07 DEBUG "instance": "10.142.0.107:9100" 06:37:07 DEBUG }, 06:37:07 DEBUG "value": [ 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG [...] 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG "instance": "10.142.0.43:9100" 06:37:07 DEBUG }, 06:37:07 DEBUG "value": [ 06:37:07 DEBUG 1713591427, 06:37:07 DEBUG "63117382.16666667" 06:37:07 DEBUG ] 06:37:07 DEBUG } 06:37:07 DEBUG ] 06:37:07 DEBUG } 06:37:07 DEBUG } 06:37:07 INFO 06:37:07 INFO [loop_until]: kubectl --namespace=xlou top pods 06:37:07 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:37:07 INFO [http_cmd]: http status code OK 06:37:07 DEBUG --- status code --- 06:37:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:37:07 DEBUG --- http response --- 06:37:07 INFO [http_cmd]: http status code OK 06:37:07 INFO [http_cmd]: http status code OK 06:37:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:37:07 DEBUG --- status code --- 06:37:07 DEBUG --- status code --- 06:37:07 DEBUG ----- output ----- 06:37:07 DEBUG { 06:37:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:37:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:37:07 DEBUG "status": "success", 06:37:07 DEBUG --- http response --- 06:37:07 DEBUG --- http response --- 06:37:07 DEBUG "data": { 06:37:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:37:07 DEBUG "resultType": "vector", 06:37:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:37:07 DEBUG ----- output ----- 06:37:07 DEBUG "result": [ 06:37:07 DEBUG ----- output ----- 06:37:07 DEBUG { 06:37:07 DEBUG { 06:37:07 DEBUG { 06:37:07 DEBUG "status": "success", 06:37:07 DEBUG "metric": { 06:37:07 DEBUG "status": "success", 06:37:07 DEBUG "data": { 06:37:07 DEBUG "pod": "am-767f6b67b5-9zw22" 06:37:07 DEBUG "data": { 06:37:07 DEBUG "resultType": "vector", 06:37:07 DEBUG }, 06:37:07 DEBUG "resultType": "vector", 06:37:07 DEBUG "result": [ 06:37:07 DEBUG "value": [ 06:37:07 DEBUG "result": [ 06:37:07 DEBUG { 06:37:07 DEBUG 06:37:07 DEBUG { 06:37:07 DEBUG "metric": { 06:37:07 DEBUG 06:37:07 DEBUG "metric": { 06:37:07 DEBUG "instance": "10.142.0.107:9100" 06:37:07 DEBUG 06:37:07 DEBUG "pod": "ds-idrepo-0" 06:37:07 DEBUG }, 06:37:07 DEBUG [...] 06:37:07 DEBUG }, 06:37:07 DEBUG "value": [ 06:37:07 DEBUG 06:37:07 DEBUG "value": [ 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG "pod": "am-767f6b67b5-vbltz" 06:37:07 DEBUG 06:37:07 DEBUG [...] 06:37:07 DEBUG }, 06:37:07 DEBUG [...] 06:37:07 DEBUG 06:37:07 DEBUG "value": [ 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG 1713591427, 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG "0.095" 06:37:07 DEBUG 06:37:07 DEBUG "instance": "10.142.0.43:9100" 06:37:07 DEBUG ] 06:37:07 DEBUG "pod": "ds-cts-2" 06:37:07 DEBUG }, 06:37:07 DEBUG } 06:37:07 DEBUG }, 06:37:07 DEBUG "value": [ 06:37:07 DEBUG ] 06:37:07 DEBUG "value": [ 06:37:07 DEBUG 1713591427, 06:37:07 DEBUG } 06:37:07 DEBUG 1713591427, 06:37:07 DEBUG "63123371.46666666" 06:37:07 DEBUG } 06:37:07 DEBUG "0" 06:37:07 DEBUG ] 06:37:07 DEBUG ] 06:37:07 DEBUG } 06:37:07 DEBUG } 06:37:07 DEBUG ] 06:37:07 DEBUG ] 06:37:07 DEBUG } 06:37:07 DEBUG } 06:37:07 DEBUG } 06:37:07 DEBUG } 06:37:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591427" 06:37:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713591427" 06:37:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591427" 06:37:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713591427" 06:37:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591427" 06:37:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591427" 06:37:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591427" 06:37:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591427" 06:37:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591427" 06:37:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591427" 06:37:07 INFO [http_cmd]: http status code OK 06:37:07 DEBUG --- status code --- 06:37:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:37:07 DEBUG --- http response --- 06:37:07 INFO [loop_until]: OK (rc = 0) 06:37:07 DEBUG --- stdout --- 06:37:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:37:07 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 72m 0% 6262Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7284m 24% 7376Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 15149m 50% 11336Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 77m 0% 3378Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 10135m 33% 13543Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 151m 0% 5189Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 15607m 52% 11567Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 18312m 61% 3510Mi 3% gke-xlou-cdm-ds-0f601751-hmm7 17070m 57% 10850Mi 9% gke-xlou-cdm-ds-0f601751-j488 7666m 25% 6999Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 10840m 36% 15600Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 7452m 24% 7642Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 12013m 40% 12659Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 8171m 51% 8538Mi 14% 06:37:07 INFO [http_cmd]: http status code OK 06:37:07 DEBUG ----- output ----- 06:37:07 DEBUG --- stderr --- 06:37:07 DEBUG --- status code --- 06:37:07 DEBUG { 06:37:07 DEBUG 06:37:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:37:07 DEBUG "status": "success", 06:37:07 DEBUG --- http response --- 06:37:07 INFO [http_cmd]: http status code OK 06:37:07 INFO [http_cmd]: http status code OK 06:37:07 INFO [http_cmd]: http status code OK 06:37:07 DEBUG "data": { 06:37:07 INFO [http_cmd]: http status code OK 06:37:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:37:07 DEBUG --- status code --- 06:37:07 INFO [http_cmd]: http status code OK 06:37:07 DEBUG --- status code --- 06:37:07 INFO [http_cmd]: http status code OK 06:37:07 INFO [http_cmd]: http status code OK 06:37:07 DEBUG --- status code --- 06:37:07 DEBUG "resultType": "vector", 06:37:07 DEBUG --- status code --- 06:37:07 DEBUG ----- output ----- 06:37:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:37:07 DEBUG --- status code --- 06:37:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:37:07 DEBUG --- status code --- 06:37:07 DEBUG --- status code --- 06:37:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:37:07 DEBUG "result": [ 06:37:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:37:07 DEBUG { 06:37:07 DEBUG --- http response --- 06:37:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:37:07 DEBUG --- http response --- 06:37:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:37:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:37:07 DEBUG --- http response --- 06:37:07 DEBUG { 06:37:07 INFO [http_cmd]: http status code OK 06:37:07 DEBUG --- http response --- 06:37:07 DEBUG "status": "success", 06:37:07 DEBUG --- http response --- 06:37:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:37:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:37:07 DEBUG --- http response --- 06:37:07 DEBUG --- http response --- 06:37:07 DEBUG "metric": { 06:37:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:37:07 DEBUG --- status code --- 06:37:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:37:07 DEBUG "data": { 06:37:07 DEBUG ----- output ----- 06:37:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:37:07 DEBUG ----- output ----- 06:37:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:37:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:37:07 DEBUG "instance": "10.142.0.107:9100" 06:37:07 DEBUG ----- output ----- 06:37:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:37:07 DEBUG ----- output ----- 06:37:07 DEBUG "resultType": "vector", 06:37:07 DEBUG { 06:37:07 DEBUG ----- output ----- 06:37:07 DEBUG { 06:37:07 DEBUG ----- output ----- 06:37:07 DEBUG ----- output ----- 06:37:07 DEBUG }, 06:37:07 DEBUG { 06:37:07 DEBUG --- http response --- 06:37:07 DEBUG { 06:37:07 DEBUG "result": [ 06:37:07 DEBUG "status": "success", 06:37:07 DEBUG { 06:37:07 DEBUG "status": "success", 06:37:07 DEBUG { 06:37:07 DEBUG { 06:37:07 DEBUG "value": [ 06:37:07 DEBUG "status": "success", 06:37:07 DEBUG "status": "success", 06:37:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:37:07 DEBUG { 06:37:07 DEBUG "data": { 06:37:07 DEBUG "status": "success", 06:37:07 DEBUG "data": { 06:37:07 DEBUG "status": "success", 06:37:07 DEBUG "status": "success", 06:37:07 DEBUG 06:37:07 DEBUG "data": { 06:37:07 DEBUG "data": { 06:37:07 DEBUG ----- output ----- 06:37:07 DEBUG "metric": { 06:37:07 DEBUG "resultType": "vector", 06:37:07 DEBUG "data": { 06:37:07 DEBUG "resultType": "vector", 06:37:07 DEBUG "data": { 06:37:07 DEBUG "data": { 06:37:07 DEBUG 06:37:07 DEBUG "resultType": "vector", 06:37:07 DEBUG "resultType": "vector", 06:37:07 DEBUG { 06:37:07 DEBUG "pod": "am-767f6b67b5-9zw22" 06:37:07 DEBUG "result": [ 06:37:07 DEBUG "resultType": "vector", 06:37:07 DEBUG "result": [ 06:37:07 DEBUG "resultType": "vector", 06:37:07 DEBUG "resultType": "vector", 06:37:07 DEBUG 06:37:07 DEBUG "result": [ 06:37:07 DEBUG "result": [ 06:37:07 DEBUG "status": "success", 06:37:07 DEBUG }, 06:37:07 DEBUG { 06:37:07 DEBUG "result": [ 06:37:07 DEBUG { 06:37:07 DEBUG "result": [ 06:37:07 DEBUG "result": [ 06:37:07 DEBUG [...] 06:37:07 DEBUG { 06:37:07 DEBUG { 06:37:07 DEBUG "data": { 06:37:07 DEBUG "value": [ 06:37:07 DEBUG "metric": { 06:37:07 DEBUG { 06:37:07 DEBUG "metric": { 06:37:07 DEBUG { 06:37:07 DEBUG { 06:37:07 DEBUG 06:37:07 DEBUG "metric": { 06:37:07 DEBUG "metric": { 06:37:07 DEBUG "resultType": "vector", 06:37:07 DEBUG 06:37:07 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:37:07 DEBUG "metric": { 06:37:07 DEBUG "pod": "am-767f6b67b5-9zw22" 06:37:07 DEBUG "metric": { 06:37:07 DEBUG "metric": { 06:37:07 DEBUG 06:37:07 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:37:07 DEBUG "pod": "ds-idrepo-0" 06:37:07 DEBUG "result": [ 06:37:07 DEBUG 06:37:07 DEBUG }, 06:37:07 DEBUG "instance": "10.142.0.107:9100" 06:37:07 DEBUG }, 06:37:07 DEBUG "pod": "am-767f6b67b5-9zw22" 06:37:07 DEBUG "pod": "am-767f6b67b5-9zw22" 06:37:07 DEBUG 06:37:07 DEBUG }, 06:37:07 DEBUG }, 06:37:07 DEBUG { 06:37:07 DEBUG 06:37:07 DEBUG "value": [ 06:37:07 DEBUG }, 06:37:07 DEBUG "value": [ 06:37:07 DEBUG }, 06:37:07 DEBUG }, 06:37:07 DEBUG "instance": "10.142.0.43:9100" 06:37:07 DEBUG "value": [ 06:37:07 DEBUG "value": [ 06:37:07 DEBUG "metric": { 06:37:07 DEBUG [...] 06:37:07 DEBUG 06:37:07 DEBUG "value": [ 06:37:07 DEBUG 06:37:07 DEBUG "value": [ 06:37:07 DEBUG "value": [ 06:37:07 DEBUG }, 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG "pod": "ds-cts-0" 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG "value": [ 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG }, 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG 1713591427, 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG "value": [ 06:37:07 DEBUG 06:37:07 DEBUG [...] 06:37:07 DEBUG 06:37:07 DEBUG [...] 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG "0.05013333333329892" 06:37:07 DEBUG [...] 06:37:07 DEBUG [...] 06:37:07 DEBUG 06:37:07 DEBUG "pod": "am-767f6b67b5-vbltz" 06:37:07 DEBUG 06:37:07 DEBUG [...] 06:37:07 DEBUG 06:37:07 DEBUG [...] 06:37:07 DEBUG [...] 06:37:07 DEBUG ] 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG }, 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG } 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG "value": [ 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG ] 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG [...] 06:37:07 DEBUG 1713591427, 06:37:07 DEBUG "pod": "am-767f6b67b5-qcqz2" 06:37:07 DEBUG 06:37:07 DEBUG "pod": "am-767f6b67b5-vbltz" 06:37:07 DEBUG 06:37:07 DEBUG 06:37:07 DEBUG } 06:37:07 DEBUG "pod": "am-767f6b67b5-qcqz2" 06:37:07 DEBUG "pod": "ds-idrepo-2" 06:37:07 DEBUG 06:37:07 DEBUG "697.8333333333334" 06:37:07 DEBUG }, 06:37:07 DEBUG "instance": "10.142.0.43:9100" 06:37:07 DEBUG }, 06:37:07 DEBUG "pod": "am-767f6b67b5-vbltz" 06:37:07 DEBUG "pod": "am-767f6b67b5-vbltz" 06:37:07 DEBUG } 06:37:07 DEBUG }, 06:37:07 DEBUG }, 06:37:07 DEBUG 06:37:07 DEBUG ] 06:37:07 DEBUG "value": [ 06:37:07 DEBUG }, 06:37:07 DEBUG "value": [ 06:37:07 DEBUG }, 06:37:07 DEBUG }, 06:37:07 DEBUG "value": [ 06:37:07 DEBUG "value": [ 06:37:07 DEBUG 06:37:07 DEBUG } 06:37:07 DEBUG 1713591427, 06:37:07 DEBUG "value": [ 06:37:07 DEBUG 1713591427, 06:37:07 DEBUG "value": [ 06:37:07 DEBUG "value": [ 06:37:07 DEBUG 1713591427, 06:37:07 DEBUG 1713591427, 06:37:07 DEBUG "pod": "ds-cts-2" 06:37:07 DEBUG ] 06:37:07 DEBUG "0" 06:37:07 DEBUG 1713591427, 06:37:07 DEBUG "2669.733333333333" 06:37:07 DEBUG 1713591427, 06:37:07 DEBUG 1713591427, 06:37:07 DEBUG "0" 06:37:07 DEBUG "0" 06:37:07 DEBUG }, 06:37:07 DEBUG } 06:37:07 DEBUG ] 06:37:07 DEBUG "0.033000000000005976" 06:37:07 DEBUG ] 06:37:07 DEBUG "2670.5" 06:37:07 DEBUG "0" 06:37:07 DEBUG ] 06:37:07 DEBUG ] 06:37:07 DEBUG "value": [ 06:37:07 DEBUG } 06:37:07 DEBUG } 06:37:07 DEBUG ] 06:37:07 DEBUG } 06:37:07 DEBUG ] 06:37:07 DEBUG ] 06:37:07 DEBUG } 06:37:07 DEBUG } 06:37:07 DEBUG 1713591427, 06:37:07 DEBUG ] 06:37:07 DEBUG } 06:37:07 DEBUG ] 06:37:07 DEBUG } 06:37:07 DEBUG } 06:37:07 DEBUG ] 06:37:07 DEBUG ] 06:37:07 DEBUG "0.00018695333333333336" 06:37:07 DEBUG } 06:37:07 DEBUG ] 06:37:07 DEBUG } 06:37:07 DEBUG ] 06:37:07 DEBUG ] 06:37:07 DEBUG } 06:37:07 DEBUG } 06:37:07 DEBUG ] 06:37:07 DEBUG } 06:37:07 DEBUG } 06:37:07 DEBUG } 06:37:07 DEBUG } 06:37:07 DEBUG } 06:37:07 DEBUG } 06:37:07 DEBUG } 06:37:07 DEBUG } 06:37:07 DEBUG } 06:37:07 DEBUG } 06:37:07 DEBUG } 06:37:07 DEBUG ] 06:37:07 DEBUG } 06:37:07 DEBUG } 06:37:07 INFO [loop_until]: OK (rc = 0) 06:37:07 DEBUG --- stdout --- 06:37:07 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 15260m 8638Mi am-767f6b67b5-qcqz2 15609m 8697Mi am-767f6b67b5-vbltz 16429m 8633Mi ds-cts-0 11819m 10404Mi ds-cts-1 9710m 10680Mi ds-cts-2 11303m 10058Mi ds-idrepo-0 6950m 3714Mi ds-idrepo-1 7155m 3747Mi ds-idrepo-2 6961m 3679Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 3m 1177Mi idm-b85f9d978-wnhxb 3m 1236Mi lodemon-7cb5b9456d-dskfn 2m 62Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7964m 4714Mi 06:37:07 DEBUG --- stderr --- 06:37:07 DEBUG 06:38:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591487" 06:38:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591487" 06:38:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591487" 06:38:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713591487" 06:38:07 INFO [http_cmd]: http status code OK 06:38:07 INFO [http_cmd]: http status code OK 06:38:07 INFO [http_cmd]: http status code OK 06:38:07 DEBUG --- status code --- 06:38:07 DEBUG --- status code --- 06:38:07 DEBUG --- status code --- 06:38:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:38:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:38:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:38:07 DEBUG --- http response --- 06:38:07 DEBUG --- http response --- 06:38:07 DEBUG --- http response --- 06:38:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:38:07 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 06:38:07 DEBUG ----- output ----- 06:38:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 06:38:07 DEBUG { 06:38:07 DEBUG ----- output ----- 06:38:07 DEBUG "status": "success", 06:38:07 DEBUG { 06:38:07 DEBUG "data": { 06:38:07 DEBUG "status": "success", 06:38:07 DEBUG "resultType": "vector", 06:38:07 DEBUG "data": { 06:38:07 DEBUG "result": [ 06:38:07 DEBUG "resultType": "vector", 06:38:07 DEBUG { 06:38:07 DEBUG "result": [ 06:38:07 DEBUG "metric": { 06:38:07 DEBUG { 06:38:07 DEBUG "pod": "am-767f6b67b5-9zw22" 06:38:07 DEBUG "metric": { 06:38:07 DEBUG }, 06:38:07 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:38:07 DEBUG "value": [ 06:38:07 DEBUG }, 06:38:07 DEBUG 06:38:07 DEBUG "value": [ 06:38:07 DEBUG 06:38:07 DEBUG 06:38:07 DEBUG 06:38:07 DEBUG 06:38:07 DEBUG [...] 06:38:07 DEBUG 06:38:07 DEBUG 06:38:07 DEBUG [...] 06:38:07 DEBUG 06:38:07 DEBUG 06:38:07 DEBUG 06:38:07 DEBUG 06:38:07 DEBUG "pod": "am-767f6b67b5-vbltz" 06:38:07 DEBUG 06:38:07 DEBUG }, 06:38:07 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 06:38:07 DEBUG "value": [ 06:38:07 DEBUG }, 06:38:07 DEBUG 1713591487, 06:38:07 DEBUG "value": [ 06:38:07 DEBUG "479331" 06:38:07 DEBUG 1713591487, 06:38:07 DEBUG ] 06:38:07 DEBUG "0" 06:38:07 DEBUG } 06:38:07 DEBUG ] 06:38:07 DEBUG ] 06:38:07 DEBUG } 06:38:07 DEBUG } 06:38:07 DEBUG ] 06:38:07 DEBUG } 06:38:07 DEBUG } 06:38:07 DEBUG } 06:38:07 ERROR [http_cmd]: ERROR 06:38:07 ERROR --- status code --- 06:38:07 ERROR http status code is 400 (expected [200, 401, 403, 404]) 06:38:07 ERROR --- http response --- 06:38:07 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 06:38:07 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 06:38:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713591487" 06:38:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591487" 06:38:07 INFO [http_cmd]: http status code OK 06:38:07 DEBUG --- status code --- 06:38:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:38:07 DEBUG --- http response --- 06:38:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:38:07 DEBUG ----- output ----- 06:38:07 DEBUG { 06:38:07 DEBUG "status": "success", 06:38:07 DEBUG "data": { 06:38:07 DEBUG "resultType": "vector", 06:38:07 DEBUG "result": [ 06:38:07 DEBUG { 06:38:07 DEBUG "metric": { 06:38:07 DEBUG "instance": "10.142.0.107:9100" 06:38:07 DEBUG }, 06:38:07 DEBUG "value": [ 06:38:07 DEBUG 06:38:07 DEBUG 06:38:07 DEBUG 06:38:07 DEBUG [...] 06:38:07 DEBUG 06:38:07 DEBUG 06:38:07 DEBUG 06:38:07 DEBUG "instance": "10.142.0.43:9100" 06:38:07 DEBUG }, 06:38:07 DEBUG "value": [ 06:38:07 DEBUG 1713591487, 06:38:07 DEBUG "3172603284480" 06:38:07 DEBUG ] 06:38:07 DEBUG } 06:38:07 DEBUG ] 06:38:07 DEBUG } 06:38:07 DEBUG } 06:38:07 INFO [http_cmd]: http status code OK 06:38:07 DEBUG --- status code --- 06:38:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:38:07 DEBUG --- http response --- 06:38:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 06:38:07 DEBUG ----- output ----- 06:38:07 DEBUG { 06:38:07 DEBUG "status": "success", 06:38:07 DEBUG "data": { 06:38:07 DEBUG "resultType": "vector", 06:38:07 DEBUG "result": [ 06:38:07 DEBUG { 06:38:07 DEBUG "metric": { 06:38:07 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:38:07 DEBUG }, 06:38:07 DEBUG "value": [ 06:38:07 DEBUG 06:38:07 DEBUG 06:38:07 DEBUG 06:38:07 DEBUG [...] 06:38:07 DEBUG 06:38:07 DEBUG 06:38:07 DEBUG 06:38:07 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 06:38:07 DEBUG }, 06:38:07 DEBUG "value": [ 06:38:07 DEBUG 1713591487, 06:38:07 DEBUG "0" 06:38:07 DEBUG ] 06:38:07 DEBUG } 06:38:07 DEBUG ] 06:38:07 DEBUG } 06:38:07 DEBUG } 06:38:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591487" 06:38:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591487" 06:38:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591487" 06:38:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591487" 06:38:07 INFO [http_cmd]: http status code OK 06:38:07 DEBUG --- status code --- 06:38:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:38:07 DEBUG --- http response --- 06:38:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:38:07 DEBUG ----- output ----- 06:38:07 DEBUG { 06:38:07 DEBUG "status": "success", 06:38:07 DEBUG "data": { 06:38:07 DEBUG "resultType": "vector", 06:38:07 DEBUG "result": [ 06:38:07 DEBUG { 06:38:07 DEBUG "metric": { 06:38:07 DEBUG "pod": "ds-cts-0" 06:38:07 DEBUG }, 06:38:07 DEBUG "value": [ 06:38:07 DEBUG 06:38:07 DEBUG 06:38:07 DEBUG 06:38:07 DEBUG [...] 06:38:07 DEBUG 06:38:07 DEBUG 06:38:07 DEBUG 06:38:07 DEBUG "pod": "ds-cts-2" 06:38:07 DEBUG }, 06:38:07 DEBUG "value": [ 06:38:07 DEBUG 1713591487, 06:38:07 DEBUG "0" 06:38:07 DEBUG ] 06:38:07 DEBUG } 06:38:07 DEBUG ] 06:38:07 DEBUG } 06:38:07 DEBUG } 06:38:07 INFO [http_cmd]: http status code OK 06:38:07 DEBUG --- status code --- 06:38:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:38:07 DEBUG --- http response --- 06:38:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:38:07 DEBUG ----- output ----- 06:38:07 DEBUG { 06:38:07 DEBUG "status": "success", 06:38:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713591487" 06:38:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591487" 06:38:07 DEBUG "data": { 06:38:07 DEBUG "resultType": "vector", 06:38:07 DEBUG "result": [ 06:38:07 DEBUG { 06:38:07 DEBUG "metric": { 06:38:07 DEBUG "pod": "ds-idrepo-0" 06:38:07 DEBUG }, 06:38:07 DEBUG "value": [ 06:38:07 DEBUG 06:38:07 DEBUG 06:38:07 DEBUG 06:38:07 DEBUG [...] 06:38:07 DEBUG 06:38:07 DEBUG 06:38:07 DEBUG 06:38:07 DEBUG "pod": "ds-idrepo-2" 06:38:07 DEBUG }, 06:38:07 DEBUG "value": [ 06:38:07 DEBUG 1713591487, 06:38:07 DEBUG "0" 06:38:07 DEBUG ] 06:38:07 DEBUG } 06:38:07 DEBUG ] 06:38:07 DEBUG } 06:38:07 DEBUG } 06:38:07 INFO [http_cmd]: http status code OK 06:38:07 INFO [http_cmd]: http status code OK 06:38:07 DEBUG --- status code --- 06:38:07 DEBUG --- status code --- 06:38:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:38:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:38:07 DEBUG --- http response --- 06:38:07 DEBUG --- http response --- 06:38:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:38:07 DEBUG ----- output ----- 06:38:07 DEBUG { 06:38:07 DEBUG "status": "success", 06:38:07 DEBUG "data": { 06:38:07 DEBUG "resultType": "vector", 06:38:07 DEBUG "result": [ 06:38:07 DEBUG { 06:38:07 DEBUG "metric": { 06:38:07 DEBUG "pod": "am-767f6b67b5-9zw22" 06:38:07 DEBUG }, 06:38:07 DEBUG "value": [ 06:38:07 DEBUG 06:38:07 DEBUG 06:38:07 DEBUG 06:38:07 DEBUG [...] 06:38:07 DEBUG 06:38:07 DEBUG 06:38:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:38:07 DEBUG 06:38:07 DEBUG ----- output ----- 06:38:07 DEBUG "pod": "am-767f6b67b5-vbltz" 06:38:07 DEBUG { 06:38:07 DEBUG }, 06:38:07 DEBUG "status": "success", 06:38:07 DEBUG "value": [ 06:38:07 DEBUG "data": { 06:38:07 DEBUG 1713591487, 06:38:07 DEBUG "resultType": "vector", 06:38:07 DEBUG "2699.866666666667" 06:38:07 DEBUG "result": [ 06:38:07 DEBUG ] 06:38:07 DEBUG { 06:38:07 DEBUG } 06:38:07 DEBUG "metric": { 06:38:07 DEBUG ] 06:38:07 DEBUG "pod": "ds-cts-0" 06:38:07 DEBUG } 06:38:07 DEBUG }, 06:38:07 DEBUG } 06:38:07 DEBUG "value": [ 06:38:07 DEBUG 06:38:07 DEBUG 06:38:07 DEBUG 06:38:07 DEBUG [...] 06:38:07 DEBUG 06:38:07 DEBUG 06:38:07 DEBUG 06:38:07 DEBUG "pod": "ds-cts-2" 06:38:07 DEBUG }, 06:38:07 DEBUG "value": [ 06:38:07 DEBUG 1713591487, 06:38:07 DEBUG "0" 06:38:07 DEBUG ] 06:38:07 DEBUG } 06:38:07 DEBUG ] 06:38:07 DEBUG } 06:38:07 DEBUG } 06:38:07 INFO [http_cmd]: http status code OK 06:38:07 DEBUG --- status code --- 06:38:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:38:07 DEBUG --- http response --- 06:38:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:38:07 DEBUG ----- output ----- 06:38:07 DEBUG { 06:38:07 DEBUG "status": "success", 06:38:07 DEBUG "data": { 06:38:07 DEBUG "resultType": "vector", 06:38:07 DEBUG "result": [ 06:38:07 DEBUG { 06:38:07 DEBUG "metric": { 06:38:07 DEBUG "pod": "ds-idrepo-0" 06:38:07 DEBUG }, 06:38:07 DEBUG "value": [ 06:38:07 DEBUG 06:38:07 DEBUG 06:38:07 DEBUG 06:38:07 DEBUG [...] 06:38:07 DEBUG 06:38:07 DEBUG 06:38:07 DEBUG 06:38:07 DEBUG "pod": "ds-idrepo-2" 06:38:07 DEBUG }, 06:38:07 DEBUG "value": [ 06:38:07 DEBUG 1713591487, 06:38:07 DEBUG "0" 06:38:07 DEBUG ] 06:38:07 DEBUG } 06:38:07 DEBUG ] 06:38:07 DEBUG } 06:38:07 DEBUG } 06:38:07 INFO [http_cmd]: http status code OK 06:38:07 DEBUG --- status code --- 06:38:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:38:07 DEBUG --- http response --- 06:38:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:38:07 DEBUG ----- output ----- 06:38:07 DEBUG { 06:38:07 DEBUG "status": "success", 06:38:07 DEBUG "data": { 06:38:07 DEBUG "resultType": "vector", 06:38:07 DEBUG "result": [ 06:38:07 DEBUG { 06:38:07 DEBUG "metric": { 06:38:07 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 06:38:07 DEBUG }, 06:38:07 DEBUG "value": [ 06:38:07 DEBUG 06:38:07 DEBUG 06:38:07 DEBUG 06:38:07 DEBUG [...] 06:38:07 DEBUG 06:38:07 DEBUG 06:38:07 DEBUG 06:38:07 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 06:38:07 DEBUG }, 06:38:07 DEBUG "value": [ 06:38:07 DEBUG 1713591487, 06:38:07 DEBUG "5047558144" 06:38:07 DEBUG ] 06:38:07 DEBUG } 06:38:07 DEBUG ] 06:38:07 DEBUG } 06:38:07 DEBUG } 06:38:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713591487" 06:38:07 INFO [http_cmd]: http status code OK 06:38:07 DEBUG --- status code --- 06:38:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:38:07 DEBUG --- http response --- 06:38:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:38:07 DEBUG ----- output ----- 06:38:07 DEBUG { 06:38:07 DEBUG "status": "success", 06:38:07 DEBUG "data": { 06:38:07 DEBUG "resultType": "vector", 06:38:07 DEBUG "result": [ 06:38:07 DEBUG { 06:38:07 DEBUG "metric": { 06:38:07 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 06:38:07 DEBUG }, 06:38:07 DEBUG "value": [ 06:38:07 DEBUG 06:38:07 DEBUG 06:38:07 DEBUG 06:38:07 DEBUG [...] 06:38:07 DEBUG 06:38:07 DEBUG 06:38:07 DEBUG 06:38:07 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 06:38:07 DEBUG }, 06:38:07 DEBUG "value": [ 06:38:07 DEBUG 1713591487, 06:38:07 DEBUG "8.030709142290958" 06:38:07 DEBUG ] 06:38:07 DEBUG } 06:38:07 DEBUG ] 06:38:07 DEBUG } 06:38:07 DEBUG } 06:38:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713591487" 06:38:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713591487" 06:38:07 INFO [http_cmd]: http status code OK 06:38:07 DEBUG --- status code --- 06:38:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:38:07 DEBUG --- http response --- 06:38:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:38:07 DEBUG ----- output ----- 06:38:07 DEBUG { 06:38:07 DEBUG "status": "success", 06:38:07 DEBUG "data": { 06:38:07 DEBUG "resultType": "vector", 06:38:07 DEBUG "result": [ 06:38:07 DEBUG { 06:38:07 DEBUG "metric": { 06:38:07 DEBUG "pod": "am-767f6b67b5-9zw22" 06:38:07 DEBUG }, 06:38:07 DEBUG "value": [ 06:38:07 DEBUG 06:38:07 DEBUG 06:38:07 DEBUG 06:38:07 DEBUG [...] 06:38:07 DEBUG 06:38:07 DEBUG 06:38:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591487" 06:38:07 INFO [http_cmd]: http status code OK 06:38:07 DEBUG 06:38:07 DEBUG --- status code --- 06:38:07 DEBUG "pod": "am-767f6b67b5-vbltz" 06:38:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:38:07 DEBUG }, 06:38:07 DEBUG --- http response --- 06:38:07 DEBUG "value": [ 06:38:07 DEBUG 1713591487, 06:38:07 DEBUG "0.095" 06:38:07 DEBUG ] 06:38:07 DEBUG } 06:38:07 DEBUG ] 06:38:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:38:07 DEBUG } 06:38:07 DEBUG ----- output ----- 06:38:07 DEBUG } 06:38:07 DEBUG { 06:38:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713591487" 06:38:07 DEBUG "status": "success", 06:38:07 DEBUG "data": { 06:38:07 DEBUG "resultType": "vector", 06:38:07 DEBUG "result": [ 06:38:07 DEBUG { 06:38:07 DEBUG "metric": { 06:38:07 DEBUG "instance": "10.142.0.107:9100" 06:38:07 DEBUG }, 06:38:07 DEBUG "value": [ 06:38:07 DEBUG 06:38:07 DEBUG 06:38:07 DEBUG 06:38:07 DEBUG [...] 06:38:07 DEBUG 06:38:07 DEBUG 06:38:07 DEBUG 06:38:07 DEBUG "instance": "10.142.0.43:9100" 06:38:07 DEBUG }, 06:38:07 DEBUG "value": [ 06:38:07 DEBUG 1713591487, 06:38:07 DEBUG "81344002.4" 06:38:07 DEBUG ] 06:38:07 DEBUG } 06:38:07 DEBUG ] 06:38:07 DEBUG } 06:38:07 DEBUG } 06:38:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713591487" 06:38:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591487" 06:38:07 INFO [http_cmd]: http status code OK 06:38:07 DEBUG --- status code --- 06:38:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:38:07 DEBUG --- http response --- 06:38:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:38:07 DEBUG ----- output ----- 06:38:07 DEBUG { 06:38:07 DEBUG "status": "success", 06:38:07 DEBUG "data": { 06:38:07 DEBUG "resultType": "vector", 06:38:07 DEBUG "result": [ 06:38:07 DEBUG { 06:38:07 DEBUG "metric": { 06:38:07 DEBUG "pod": "ds-idrepo-0" 06:38:07 DEBUG }, 06:38:07 DEBUG "value": [ 06:38:07 DEBUG 06:38:07 DEBUG 06:38:07 DEBUG 06:38:07 DEBUG [...] 06:38:07 DEBUG 06:38:07 DEBUG 06:38:07 DEBUG 06:38:07 DEBUG "pod": "ds-idrepo-2" 06:38:07 DEBUG }, 06:38:07 DEBUG "value": [ 06:38:07 DEBUG 1713591487, 06:38:07 DEBUG "0" 06:38:07 DEBUG ] 06:38:07 DEBUG } 06:38:07 DEBUG ] 06:38:07 DEBUG } 06:38:07 DEBUG } 06:38:07 INFO [http_cmd]: http status code OK 06:38:07 DEBUG --- status code --- 06:38:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:38:07 DEBUG --- http response --- 06:38:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:38:07 INFO [http_cmd]: http status code OK 06:38:07 INFO 06:38:07 DEBUG ----- output ----- 06:38:07 DEBUG --- status code --- 06:38:07 INFO [loop_until]: kubectl --namespace=xlou top node 06:38:07 DEBUG { 06:38:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:38:07 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:38:07 DEBUG "status": "success", 06:38:07 DEBUG --- http response --- 06:38:07 DEBUG "data": { 06:38:07 DEBUG "resultType": "vector", 06:38:07 DEBUG "result": [ 06:38:07 DEBUG { 06:38:07 DEBUG "metric": { 06:38:07 DEBUG "pod": "am-767f6b67b5-9zw22" 06:38:07 DEBUG }, 06:38:07 DEBUG "value": [ 06:38:07 DEBUG 06:38:07 DEBUG 06:38:07 DEBUG 06:38:07 DEBUG [...] 06:38:07 DEBUG 06:38:07 DEBUG 06:38:07 DEBUG 06:38:07 DEBUG "pod": "am-767f6b67b5-vbltz" 06:38:07 DEBUG }, 06:38:07 DEBUG "value": [ 06:38:07 DEBUG 1713591487, 06:38:07 DEBUG "2670.4333333333334" 06:38:07 DEBUG ] 06:38:07 DEBUG } 06:38:07 DEBUG ] 06:38:07 DEBUG } 06:38:07 DEBUG } 06:38:07 INFO [http_cmd]: http status code OK 06:38:07 DEBUG --- status code --- 06:38:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:38:07 DEBUG --- http response --- 06:38:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:38:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:38:07 DEBUG ----- output ----- 06:38:07 DEBUG ----- output ----- 06:38:07 DEBUG { 06:38:07 DEBUG { 06:38:07 DEBUG "status": "success", 06:38:07 DEBUG "status": "success", 06:38:07 DEBUG "data": { 06:38:07 DEBUG "data": { 06:38:07 DEBUG "resultType": "vector", 06:38:07 DEBUG "resultType": "vector", 06:38:07 DEBUG "result": [ 06:38:07 DEBUG "result": [ 06:38:07 DEBUG { 06:38:07 DEBUG { 06:38:07 DEBUG "metric": { 06:38:07 DEBUG "metric": { 06:38:07 DEBUG "instance": "10.142.0.107:9100" 06:38:07 DEBUG "pod": "ds-idrepo-0" 06:38:07 DEBUG }, 06:38:07 DEBUG }, 06:38:07 DEBUG "value": [ 06:38:07 DEBUG "value": [ 06:38:07 DEBUG 06:38:07 DEBUG 06:38:07 DEBUG 06:38:07 DEBUG 06:38:07 DEBUG 06:38:07 DEBUG 06:38:07 DEBUG [...] 06:38:07 DEBUG [...] 06:38:07 DEBUG 06:38:07 DEBUG 06:38:07 DEBUG 06:38:07 DEBUG 06:38:07 DEBUG 06:38:07 DEBUG 06:38:07 DEBUG "instance": "10.142.0.43:9100" 06:38:07 DEBUG "pod": "ds-cts-2" 06:38:07 DEBUG }, 06:38:07 DEBUG }, 06:38:07 DEBUG "value": [ 06:38:07 DEBUG "value": [ 06:38:07 DEBUG 1713591487, 06:38:07 DEBUG 1713591487, 06:38:07 DEBUG "81349201.26666667" 06:38:07 DEBUG "0" 06:38:07 DEBUG ] 06:38:07 DEBUG ] 06:38:07 DEBUG } 06:38:07 DEBUG } 06:38:07 DEBUG ] 06:38:07 DEBUG ] 06:38:07 DEBUG } 06:38:07 DEBUG } 06:38:07 DEBUG } 06:38:07 DEBUG } 06:38:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713591487" 06:38:07 INFO [http_cmd]: http status code OK 06:38:07 DEBUG --- status code --- 06:38:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:38:07 DEBUG --- http response --- 06:38:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:38:07 DEBUG ----- output ----- 06:38:07 DEBUG { 06:38:07 DEBUG "status": "success", 06:38:07 DEBUG "data": { 06:38:07 DEBUG "resultType": "vector", 06:38:07 DEBUG "result": [ 06:38:07 DEBUG { 06:38:07 DEBUG "metric": { 06:38:07 DEBUG "instance": "10.142.0.107:9100" 06:38:07 DEBUG }, 06:38:07 DEBUG "value": [ 06:38:07 DEBUG 06:38:07 DEBUG 06:38:07 DEBUG 06:38:07 DEBUG [...] 06:38:07 DEBUG 06:38:07 DEBUG 06:38:07 DEBUG 06:38:07 DEBUG "instance": "10.142.0.43:9100" 06:38:07 DEBUG }, 06:38:07 DEBUG "value": [ 06:38:07 DEBUG 1713591487, 06:38:07 DEBUG "0.16856666666662837" 06:38:07 DEBUG ] 06:38:07 DEBUG } 06:38:07 DEBUG ] 06:38:07 DEBUG } 06:38:07 DEBUG } 06:38:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591488" 06:38:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591488" 06:38:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713591488" 06:38:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591488" 06:38:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591488" 06:38:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591488" 06:38:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591488" 06:38:08 INFO [http_cmd]: http status code OK 06:38:08 INFO [http_cmd]: http status code OK 06:38:08 DEBUG --- status code --- 06:38:08 DEBUG --- status code --- 06:38:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:38:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:38:08 DEBUG --- http response --- 06:38:08 DEBUG --- http response --- 06:38:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:38:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:38:08 DEBUG ----- output ----- 06:38:08 INFO [http_cmd]: http status code OK 06:38:08 DEBUG ----- output ----- 06:38:08 INFO [http_cmd]: http status code OK 06:38:08 DEBUG { 06:38:08 DEBUG --- status code --- 06:38:08 DEBUG { 06:38:08 INFO [http_cmd]: http status code OK 06:38:08 DEBUG --- status code --- 06:38:08 DEBUG "status": "success", 06:38:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:38:08 DEBUG "status": "success", 06:38:08 DEBUG --- status code --- 06:38:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:38:08 DEBUG "data": { 06:38:08 DEBUG --- http response --- 06:38:08 DEBUG "data": { 06:38:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:38:08 DEBUG --- http response --- 06:38:08 DEBUG "resultType": "vector", 06:38:08 DEBUG "resultType": "vector", 06:38:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:38:08 DEBUG --- http response --- 06:38:08 DEBUG "result": [ 06:38:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:38:08 INFO [http_cmd]: http status code OK 06:38:08 INFO [http_cmd]: http status code OK 06:38:08 DEBUG "result": [ 06:38:08 DEBUG ----- output ----- 06:38:08 DEBUG { 06:38:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:38:08 DEBUG ----- output ----- 06:38:08 DEBUG --- status code --- 06:38:08 DEBUG --- status code --- 06:38:08 DEBUG { 06:38:08 DEBUG { 06:38:08 DEBUG "metric": { 06:38:08 DEBUG ----- output ----- 06:38:08 DEBUG { 06:38:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:38:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:38:08 DEBUG "metric": { 06:38:08 DEBUG "status": "success", 06:38:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:38:08 DEBUG { 06:38:08 DEBUG "status": "success", 06:38:08 DEBUG --- http response --- 06:38:08 DEBUG --- http response --- 06:38:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:38:08 DEBUG "data": { 06:38:08 DEBUG }, 06:38:08 DEBUG "status": "success", 06:38:08 DEBUG "data": { 06:38:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:38:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:38:08 DEBUG }, 06:38:08 DEBUG "resultType": "vector", 06:38:08 DEBUG "value": [ 06:38:08 DEBUG "data": { 06:38:08 DEBUG "resultType": "vector", 06:38:08 DEBUG ----- output ----- 06:38:08 DEBUG ----- output ----- 06:38:08 DEBUG "value": [ 06:38:08 DEBUG "result": [ 06:38:08 DEBUG 06:38:08 DEBUG "resultType": "vector", 06:38:08 DEBUG "result": [ 06:38:08 DEBUG { 06:38:08 DEBUG { 06:38:08 DEBUG 06:38:08 DEBUG { 06:38:08 DEBUG 06:38:08 DEBUG "result": [ 06:38:08 DEBUG { 06:38:08 DEBUG "status": "success", 06:38:08 DEBUG "status": "success", 06:38:08 DEBUG 06:38:08 DEBUG "metric": { 06:38:08 DEBUG 06:38:08 DEBUG { 06:38:08 DEBUG "metric": { 06:38:08 DEBUG "data": { 06:38:08 DEBUG "data": { 06:38:08 DEBUG 06:38:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:38:08 DEBUG [...] 06:38:08 DEBUG "metric": { 06:38:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:38:08 DEBUG "resultType": "vector", 06:38:08 DEBUG "resultType": "vector", 06:38:08 DEBUG [...] 06:38:08 DEBUG }, 06:38:08 DEBUG 06:38:08 DEBUG "instance": "10.142.0.107:9100" 06:38:08 DEBUG }, 06:38:08 DEBUG "result": [ 06:38:08 DEBUG "result": [ 06:38:08 DEBUG 06:38:08 DEBUG "value": [ 06:38:08 DEBUG 06:38:08 DEBUG }, 06:38:08 DEBUG "value": [ 06:38:08 DEBUG { 06:38:08 DEBUG { 06:38:08 DEBUG 06:38:08 DEBUG 06:38:08 DEBUG 06:38:08 DEBUG "value": [ 06:38:08 DEBUG 06:38:08 DEBUG "metric": { 06:38:08 DEBUG "metric": { 06:38:08 DEBUG 06:38:08 DEBUG 06:38:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:38:08 DEBUG 06:38:08 DEBUG 06:38:08 DEBUG "pod": "ds-cts-0" 06:38:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:38:08 DEBUG "pod": "ds-idrepo-2" 06:38:08 DEBUG 06:38:08 DEBUG }, 06:38:08 DEBUG 06:38:08 DEBUG 06:38:08 DEBUG }, 06:38:08 DEBUG }, 06:38:08 DEBUG }, 06:38:08 DEBUG [...] 06:38:08 DEBUG "value": [ 06:38:08 DEBUG 06:38:08 DEBUG [...] 06:38:08 DEBUG "value": [ 06:38:08 DEBUG "value": [ 06:38:08 DEBUG "value": [ 06:38:08 DEBUG 06:38:08 DEBUG 1713591488, 06:38:08 DEBUG [...] 06:38:08 DEBUG 06:38:08 DEBUG 06:38:08 DEBUG 06:38:08 DEBUG 1713591488, 06:38:08 DEBUG 06:38:08 DEBUG "2697.633333333333" 06:38:08 DEBUG 06:38:08 DEBUG 06:38:08 DEBUG 06:38:08 DEBUG 06:38:08 DEBUG "0" 06:38:08 DEBUG 06:38:08 DEBUG ] 06:38:08 DEBUG 06:38:08 DEBUG 06:38:08 DEBUG 06:38:08 DEBUG 06:38:08 DEBUG ] 06:38:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:38:08 DEBUG } 06:38:08 DEBUG 06:38:08 DEBUG "pod": "ds-idrepo-2" 06:38:08 DEBUG [...] 06:38:08 DEBUG [...] 06:38:08 DEBUG } 06:38:08 DEBUG }, 06:38:08 DEBUG ] 06:38:08 DEBUG "instance": "10.142.0.43:9100" 06:38:08 DEBUG }, 06:38:08 DEBUG 06:38:08 DEBUG 06:38:08 DEBUG ] 06:38:08 DEBUG "value": [ 06:38:08 DEBUG } 06:38:08 DEBUG }, 06:38:08 DEBUG "value": [ 06:38:08 DEBUG 06:38:08 DEBUG 06:38:08 DEBUG 06:38:08 DEBUG } 06:38:08 DEBUG } 06:38:08 DEBUG "value": [ 06:38:08 DEBUG 1713591488, 06:38:08 DEBUG 06:38:08 DEBUG 1713591488, 06:38:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:38:08 DEBUG } 06:38:08 DEBUG 1713591488, 06:38:08 DEBUG "0" 06:38:08 DEBUG "pod": "ds-cts-2" 06:38:08 DEBUG "0" 06:38:08 DEBUG }, 06:38:08 DEBUG "0.06133333333334388" 06:38:08 DEBUG ] 06:38:08 DEBUG }, 06:38:08 DEBUG ] 06:38:08 DEBUG "value": [ 06:38:08 DEBUG ] 06:38:08 DEBUG } 06:38:08 DEBUG "value": [ 06:38:08 DEBUG } 06:38:08 DEBUG 1713591488, 06:38:08 DEBUG } 06:38:08 DEBUG ] 06:38:08 DEBUG 1713591488, 06:38:08 DEBUG ] 06:38:08 DEBUG "2696.633333333333" 06:38:08 DEBUG ] 06:38:08 DEBUG } 06:38:08 DEBUG "0.00046666666666666666" 06:38:08 DEBUG } 06:38:08 DEBUG ] 06:38:08 DEBUG } 06:38:08 DEBUG } 06:38:08 DEBUG ] 06:38:08 DEBUG } 06:38:08 DEBUG } 06:38:08 DEBUG } 06:38:08 DEBUG } 06:38:08 DEBUG ] 06:38:08 DEBUG ] 06:38:08 DEBUG } 06:38:08 DEBUG } 06:38:08 DEBUG } 06:38:08 DEBUG } 06:38:08 INFO [loop_until]: OK (rc = 0) 06:38:08 DEBUG --- stdout --- 06:38:08 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 69m 0% 6264Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 8047m 26% 7389Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 17382m 58% 11351Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 78m 0% 3369Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 18170m 60% 13535Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 150m 0% 5200Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 17723m 59% 11582Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 18349m 61% 3417Mi 3% gke-xlou-cdm-ds-0f601751-hmm7 19583m 65% 10857Mi 9% gke-xlou-cdm-ds-0f601751-j488 8205m 27% 7013Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 16849m 56% 15586Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 8170m 27% 7664Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 18719m 62% 12642Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 7920m 49% 8542Mi 14% 06:38:08 DEBUG --- stderr --- 06:38:08 DEBUG 06:38:08 INFO 06:38:08 INFO [loop_until]: kubectl --namespace=xlou top pods 06:38:08 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:38:08 INFO [loop_until]: OK (rc = 0) 06:38:08 DEBUG --- stdout --- 06:38:08 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 17515m 8641Mi am-767f6b67b5-qcqz2 17708m 8708Mi am-767f6b67b5-vbltz 19275m 8652Mi ds-cts-0 18395m 10380Mi ds-cts-1 16820m 10661Mi ds-cts-2 19091m 10048Mi ds-idrepo-0 7821m 3716Mi ds-idrepo-1 7688m 3757Mi ds-idrepo-2 7978m 3685Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1177Mi idm-b85f9d978-wnhxb 3m 1236Mi lodemon-7cb5b9456d-dskfn 2m 62Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 8181m 4721Mi 06:38:08 DEBUG --- stderr --- 06:38:08 DEBUG 06:39:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713591547" 06:39:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591547" 06:39:07 ERROR [http_cmd]: ERROR 06:39:07 ERROR --- status code --- 06:39:07 ERROR http status code is 400 (expected [200, 401, 403, 404]) 06:39:07 ERROR --- http response --- 06:39:07 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 06:39:07 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 06:39:07 INFO [http_cmd]: http status code OK 06:39:07 DEBUG --- status code --- 06:39:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:39:07 DEBUG --- http response --- 06:39:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 06:39:07 DEBUG ----- output ----- 06:39:07 DEBUG { 06:39:07 DEBUG "status": "success", 06:39:07 DEBUG "data": { 06:39:07 DEBUG "resultType": "vector", 06:39:07 DEBUG "result": [ 06:39:07 DEBUG { 06:39:07 DEBUG "metric": { 06:39:07 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:39:07 DEBUG }, 06:39:07 DEBUG "value": [ 06:39:07 DEBUG 06:39:07 DEBUG 06:39:07 DEBUG 06:39:07 DEBUG [...] 06:39:07 DEBUG 06:39:07 DEBUG 06:39:07 DEBUG 06:39:07 DEBUG "pod": "am-767f6b67b5-qcqz2" 06:39:07 DEBUG }, 06:39:07 DEBUG "value": [ 06:39:07 DEBUG 1713591547, 06:39:07 DEBUG "2127.1179279987027" 06:39:07 DEBUG ] 06:39:07 DEBUG } 06:39:07 DEBUG ] 06:39:07 DEBUG } 06:39:07 DEBUG } 06:39:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591547" 06:39:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591547" 06:39:07 INFO [http_cmd]: http status code OK 06:39:07 INFO [http_cmd]: http status code OK 06:39:07 DEBUG --- status code --- 06:39:07 DEBUG --- status code --- 06:39:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:39:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:39:07 DEBUG --- http response --- 06:39:07 DEBUG --- http response --- 06:39:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:39:07 DEBUG ----- output ----- 06:39:07 DEBUG { 06:39:07 DEBUG "status": "success", 06:39:07 DEBUG "data": { 06:39:07 DEBUG "resultType": "vector", 06:39:07 DEBUG "result": [ 06:39:07 DEBUG { 06:39:07 DEBUG "metric": { 06:39:07 DEBUG "pod": "am-767f6b67b5-9zw22" 06:39:07 DEBUG }, 06:39:07 DEBUG "value": [ 06:39:07 DEBUG 06:39:07 DEBUG 06:39:07 DEBUG 06:39:07 DEBUG [...] 06:39:07 DEBUG 06:39:07 DEBUG 06:39:07 DEBUG 06:39:07 DEBUG "pod": "am-767f6b67b5-vbltz" 06:39:07 DEBUG }, 06:39:07 DEBUG "value": [ 06:39:07 DEBUG 1713591547, 06:39:07 DEBUG "481043" 06:39:07 DEBUG ] 06:39:07 DEBUG } 06:39:07 DEBUG ] 06:39:07 DEBUG } 06:39:07 DEBUG } 06:39:07 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 06:39:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713591547" 06:39:07 INFO [http_cmd]: http status code OK 06:39:07 DEBUG --- status code --- 06:39:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:39:07 DEBUG --- http response --- 06:39:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:39:07 DEBUG ----- output ----- 06:39:07 DEBUG { 06:39:07 DEBUG "status": "success", 06:39:07 DEBUG "data": { 06:39:07 DEBUG "resultType": "vector", 06:39:07 DEBUG "result": [ 06:39:07 DEBUG { 06:39:07 DEBUG "metric": { 06:39:07 DEBUG "instance": "10.142.0.107:9100" 06:39:07 DEBUG }, 06:39:07 DEBUG "value": [ 06:39:07 DEBUG 06:39:07 DEBUG 06:39:07 DEBUG 06:39:07 DEBUG [...] 06:39:07 DEBUG 06:39:07 DEBUG 06:39:07 DEBUG 06:39:07 DEBUG "instance": "10.142.0.43:9100" 06:39:07 DEBUG }, 06:39:07 DEBUG "value": [ 06:39:07 DEBUG 1713591547, 06:39:07 DEBUG "3176009196544" 06:39:07 DEBUG ] 06:39:07 DEBUG } 06:39:07 DEBUG ] 06:39:07 DEBUG } 06:39:07 DEBUG } 06:39:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713591547" 06:39:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591547" 06:39:07 INFO [http_cmd]: http status code OK 06:39:07 DEBUG --- status code --- 06:39:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:39:07 DEBUG --- http response --- 06:39:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:39:07 DEBUG ----- output ----- 06:39:07 DEBUG { 06:39:07 DEBUG "status": "success", 06:39:07 DEBUG "data": { 06:39:07 DEBUG "resultType": "vector", 06:39:07 DEBUG "result": [ 06:39:07 DEBUG { 06:39:07 DEBUG "metric": { 06:39:07 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 06:39:07 DEBUG }, 06:39:07 DEBUG "value": [ 06:39:07 DEBUG 06:39:07 DEBUG 06:39:07 DEBUG 06:39:07 DEBUG [...] 06:39:07 DEBUG 06:39:07 DEBUG 06:39:07 DEBUG 06:39:07 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 06:39:07 DEBUG }, 06:39:07 DEBUG "value": [ 06:39:07 DEBUG 1713591547, 06:39:07 DEBUG "5031825408" 06:39:07 DEBUG ] 06:39:07 DEBUG } 06:39:07 DEBUG ] 06:39:07 DEBUG } 06:39:07 DEBUG } 06:39:07 INFO [http_cmd]: http status code OK 06:39:07 DEBUG --- status code --- 06:39:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:39:07 DEBUG --- http response --- 06:39:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 06:39:07 DEBUG ----- output ----- 06:39:07 DEBUG { 06:39:07 DEBUG "status": "success", 06:39:07 DEBUG "data": { 06:39:07 DEBUG "resultType": "vector", 06:39:07 DEBUG "result": [ 06:39:07 DEBUG { 06:39:07 DEBUG "metric": { 06:39:07 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:39:07 DEBUG }, 06:39:07 DEBUG "value": [ 06:39:07 DEBUG 06:39:07 DEBUG 06:39:07 DEBUG 06:39:07 DEBUG [...] 06:39:07 DEBUG 06:39:07 DEBUG 06:39:07 DEBUG 06:39:07 DEBUG "pod": "am-767f6b67b5-qcqz2" 06:39:07 DEBUG }, 06:39:07 DEBUG "value": [ 06:39:07 DEBUG 1713591547, 06:39:07 DEBUG "0" 06:39:07 DEBUG ] 06:39:07 DEBUG } 06:39:07 DEBUG ] 06:39:07 DEBUG } 06:39:07 DEBUG } 06:39:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591547" 06:39:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591547" 06:39:07 INFO [http_cmd]: http status code OK 06:39:07 DEBUG --- status code --- 06:39:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:39:07 DEBUG --- http response --- 06:39:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591547" 06:39:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591547" 06:39:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:39:07 DEBUG ----- output ----- 06:39:07 DEBUG { 06:39:07 DEBUG "status": "success", 06:39:07 DEBUG "data": { 06:39:07 DEBUG "resultType": "vector", 06:39:07 DEBUG "result": [ 06:39:07 DEBUG { 06:39:07 DEBUG "metric": { 06:39:07 DEBUG "pod": "ds-cts-0" 06:39:07 DEBUG }, 06:39:07 DEBUG "value": [ 06:39:07 DEBUG 06:39:07 DEBUG 06:39:07 DEBUG 06:39:07 DEBUG [...] 06:39:07 DEBUG 06:39:07 DEBUG 06:39:07 DEBUG 06:39:07 DEBUG "pod": "ds-cts-2" 06:39:07 DEBUG }, 06:39:07 DEBUG "value": [ 06:39:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591547" 06:39:07 DEBUG 1713591547, 06:39:07 DEBUG "0" 06:39:07 DEBUG ] 06:39:07 DEBUG } 06:39:07 DEBUG ] 06:39:07 DEBUG } 06:39:07 DEBUG } 06:39:07 INFO [http_cmd]: http status code OK 06:39:07 DEBUG --- status code --- 06:39:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:39:07 DEBUG --- http response --- 06:39:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:39:07 DEBUG ----- output ----- 06:39:07 DEBUG { 06:39:07 DEBUG "status": "success", 06:39:07 DEBUG "data": { 06:39:07 DEBUG "resultType": "vector", 06:39:07 DEBUG "result": [ 06:39:07 DEBUG { 06:39:07 DEBUG "metric": { 06:39:07 DEBUG "pod": "ds-idrepo-0" 06:39:07 DEBUG }, 06:39:07 DEBUG "value": [ 06:39:07 DEBUG 06:39:07 DEBUG 06:39:07 DEBUG 06:39:07 DEBUG [...] 06:39:07 DEBUG 06:39:07 DEBUG 06:39:07 DEBUG 06:39:07 DEBUG "pod": "ds-idrepo-2" 06:39:07 DEBUG }, 06:39:07 DEBUG "value": [ 06:39:07 DEBUG 1713591547, 06:39:07 DEBUG "0" 06:39:07 DEBUG ] 06:39:07 DEBUG } 06:39:07 DEBUG ] 06:39:07 DEBUG } 06:39:07 DEBUG } 06:39:07 INFO [http_cmd]: http status code OK 06:39:07 DEBUG --- status code --- 06:39:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:39:07 DEBUG --- http response --- 06:39:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:39:07 DEBUG ----- output ----- 06:39:07 DEBUG { 06:39:07 DEBUG "status": "success", 06:39:07 DEBUG "data": { 06:39:07 DEBUG "resultType": "vector", 06:39:07 DEBUG "result": [ 06:39:07 DEBUG { 06:39:07 DEBUG "metric": { 06:39:07 DEBUG "pod": "am-767f6b67b5-9zw22" 06:39:07 DEBUG }, 06:39:07 DEBUG "value": [ 06:39:07 DEBUG 06:39:07 DEBUG 06:39:07 DEBUG 06:39:07 DEBUG [...] 06:39:07 DEBUG 06:39:07 DEBUG 06:39:07 DEBUG 06:39:07 DEBUG "pod": "am-767f6b67b5-vbltz" 06:39:07 DEBUG }, 06:39:07 DEBUG "value": [ 06:39:07 DEBUG 1713591547, 06:39:07 DEBUG "2690.7" 06:39:07 DEBUG ] 06:39:07 DEBUG } 06:39:07 DEBUG ] 06:39:07 DEBUG } 06:39:07 DEBUG } 06:39:07 INFO [http_cmd]: http status code OK 06:39:07 DEBUG --- status code --- 06:39:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:39:07 DEBUG --- http response --- 06:39:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:39:07 DEBUG ----- output ----- 06:39:07 DEBUG { 06:39:07 DEBUG "status": "success", 06:39:07 DEBUG "data": { 06:39:07 DEBUG "resultType": "vector", 06:39:07 DEBUG "result": [ 06:39:07 DEBUG { 06:39:07 DEBUG "metric": { 06:39:07 DEBUG "pod": "ds-cts-0" 06:39:07 DEBUG }, 06:39:07 DEBUG "value": [ 06:39:07 DEBUG 06:39:07 DEBUG 06:39:07 DEBUG 06:39:07 DEBUG [...] 06:39:07 DEBUG 06:39:07 DEBUG 06:39:07 DEBUG 06:39:07 DEBUG "pod": "ds-cts-2" 06:39:07 DEBUG }, 06:39:07 DEBUG "value": [ 06:39:07 DEBUG 1713591547, 06:39:07 DEBUG "0" 06:39:07 DEBUG ] 06:39:07 DEBUG } 06:39:07 DEBUG ] 06:39:07 DEBUG } 06:39:07 DEBUG } 06:39:07 INFO [http_cmd]: http status code OK 06:39:07 DEBUG --- status code --- 06:39:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:39:07 DEBUG --- http response --- 06:39:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:39:07 DEBUG ----- output ----- 06:39:07 DEBUG { 06:39:07 DEBUG "status": "success", 06:39:07 DEBUG "data": { 06:39:07 DEBUG "resultType": "vector", 06:39:07 DEBUG "result": [ 06:39:07 DEBUG { 06:39:07 DEBUG "metric": { 06:39:07 DEBUG "pod": "ds-idrepo-0" 06:39:07 DEBUG }, 06:39:07 DEBUG "value": [ 06:39:07 DEBUG 06:39:07 DEBUG 06:39:07 DEBUG 06:39:07 DEBUG [...] 06:39:07 DEBUG 06:39:07 DEBUG 06:39:07 DEBUG 06:39:07 DEBUG "pod": "ds-idrepo-2" 06:39:07 DEBUG }, 06:39:07 DEBUG "value": [ 06:39:07 DEBUG 1713591547, 06:39:07 DEBUG "0" 06:39:07 DEBUG ] 06:39:07 DEBUG } 06:39:07 DEBUG ] 06:39:07 DEBUG } 06:39:07 DEBUG } 06:39:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713591547" 06:39:08 INFO [http_cmd]: http status code OK 06:39:08 DEBUG --- status code --- 06:39:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:39:08 DEBUG --- http response --- 06:39:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:39:08 DEBUG ----- output ----- 06:39:08 DEBUG { 06:39:08 DEBUG "status": "success", 06:39:08 DEBUG "data": { 06:39:08 DEBUG "resultType": "vector", 06:39:08 DEBUG "result": [ 06:39:08 DEBUG { 06:39:08 DEBUG "metric": { 06:39:08 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 06:39:08 DEBUG }, 06:39:08 DEBUG "value": [ 06:39:08 DEBUG 06:39:08 DEBUG 06:39:08 DEBUG 06:39:08 DEBUG [...] 06:39:08 DEBUG 06:39:08 DEBUG 06:39:08 DEBUG 06:39:08 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 06:39:08 DEBUG }, 06:39:08 DEBUG "value": [ 06:39:08 DEBUG 1713591547, 06:39:08 DEBUG "8.031739018150025" 06:39:08 DEBUG ] 06:39:08 DEBUG } 06:39:08 DEBUG ] 06:39:08 DEBUG } 06:39:08 DEBUG } 06:39:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713591548" 06:39:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713591548" 06:39:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713591548" 06:39:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591548" 06:39:08 INFO [http_cmd]: http status code OK 06:39:08 INFO [http_cmd]: http status code OK 06:39:08 DEBUG --- status code --- 06:39:08 DEBUG --- status code --- 06:39:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:39:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:39:08 DEBUG --- http response --- 06:39:08 DEBUG --- http response --- 06:39:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:39:08 DEBUG ----- output ----- 06:39:08 DEBUG { 06:39:08 DEBUG "status": "success", 06:39:08 DEBUG "data": { 06:39:08 DEBUG "resultType": "vector", 06:39:08 DEBUG "result": [ 06:39:08 DEBUG { 06:39:08 DEBUG "metric": { 06:39:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:39:08 DEBUG }, 06:39:08 DEBUG "value": [ 06:39:08 DEBUG 06:39:08 DEBUG 06:39:08 DEBUG 06:39:08 DEBUG [...] 06:39:08 DEBUG 06:39:08 DEBUG 06:39:08 DEBUG 06:39:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:39:08 DEBUG }, 06:39:08 DEBUG "value": [ 06:39:08 DEBUG 1713591548, 06:39:08 DEBUG "0.095" 06:39:08 DEBUG ] 06:39:08 DEBUG } 06:39:08 DEBUG ] 06:39:08 DEBUG } 06:39:08 DEBUG } 06:39:08 INFO [http_cmd]: http status code OK 06:39:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:39:08 DEBUG --- status code --- 06:39:08 INFO [http_cmd]: http status code OK 06:39:08 DEBUG ----- output ----- 06:39:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:39:08 DEBUG --- status code --- 06:39:08 DEBUG { 06:39:08 DEBUG --- http response --- 06:39:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:39:08 DEBUG "status": "success", 06:39:08 DEBUG --- http response --- 06:39:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:39:08 DEBUG "data": { 06:39:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:39:08 DEBUG ----- output ----- 06:39:08 DEBUG "resultType": "vector", 06:39:08 DEBUG ----- output ----- 06:39:08 DEBUG { 06:39:08 DEBUG "result": [ 06:39:08 DEBUG { 06:39:08 DEBUG "status": "success", 06:39:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591548" 06:39:08 DEBUG { 06:39:08 DEBUG "status": "success", 06:39:08 DEBUG "data": { 06:39:08 DEBUG "resultType": "vector", 06:39:08 DEBUG "data": { 06:39:08 DEBUG "result": [ 06:39:08 DEBUG "metric": { 06:39:08 DEBUG "resultType": "vector", 06:39:08 DEBUG { 06:39:08 DEBUG "instance": "10.142.0.107:9100" 06:39:08 DEBUG "result": [ 06:39:08 DEBUG "metric": { 06:39:08 DEBUG }, 06:39:08 DEBUG { 06:39:08 DEBUG "pod": "ds-idrepo-0" 06:39:08 DEBUG "value": [ 06:39:08 DEBUG "metric": { 06:39:08 DEBUG }, 06:39:08 DEBUG 06:39:08 DEBUG "instance": "10.142.0.107:9100" 06:39:08 DEBUG "value": [ 06:39:08 DEBUG 06:39:08 DEBUG }, 06:39:08 DEBUG 06:39:08 DEBUG 06:39:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713591548" 06:39:08 DEBUG 06:39:08 DEBUG "value": [ 06:39:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713591548" 06:39:08 DEBUG 06:39:08 DEBUG [...] 06:39:08 DEBUG 06:39:08 DEBUG [...] 06:39:08 DEBUG 06:39:08 DEBUG 06:39:08 DEBUG 06:39:08 DEBUG 06:39:08 DEBUG 06:39:08 DEBUG 06:39:08 DEBUG 06:39:08 DEBUG [...] 06:39:08 DEBUG 06:39:08 DEBUG "instance": "10.142.0.43:9100" 06:39:08 DEBUG 06:39:08 DEBUG "pod": "ds-idrepo-2" 06:39:08 DEBUG }, 06:39:08 DEBUG 06:39:08 DEBUG }, 06:39:08 DEBUG "value": [ 06:39:08 DEBUG 06:39:08 DEBUG "value": [ 06:39:08 DEBUG 1713591548, 06:39:08 DEBUG "instance": "10.142.0.43:9100" 06:39:08 DEBUG 1713591548, 06:39:08 DEBUG "0.07146666666659295" 06:39:08 DEBUG }, 06:39:08 DEBUG "0" 06:39:08 DEBUG ] 06:39:08 DEBUG "value": [ 06:39:08 DEBUG ] 06:39:08 DEBUG } 06:39:08 DEBUG 1713591548, 06:39:08 DEBUG } 06:39:08 DEBUG ] 06:39:08 DEBUG "81421018.66666666" 06:39:08 DEBUG ] 06:39:08 DEBUG } 06:39:08 DEBUG ] 06:39:08 DEBUG } 06:39:08 DEBUG } 06:39:08 DEBUG } 06:39:08 DEBUG } 06:39:08 DEBUG ] 06:39:08 DEBUG } 06:39:08 DEBUG } 06:39:08 INFO [http_cmd]: http status code OK 06:39:08 DEBUG --- status code --- 06:39:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:39:08 DEBUG --- http response --- 06:39:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:39:08 DEBUG ----- output ----- 06:39:08 DEBUG { 06:39:08 DEBUG "status": "success", 06:39:08 DEBUG "data": { 06:39:08 DEBUG "resultType": "vector", 06:39:08 DEBUG "result": [ 06:39:08 DEBUG { 06:39:08 DEBUG "metric": { 06:39:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:39:08 DEBUG }, 06:39:08 DEBUG "value": [ 06:39:08 DEBUG 06:39:08 DEBUG 06:39:08 DEBUG 06:39:08 DEBUG [...] 06:39:08 DEBUG 06:39:08 DEBUG 06:39:08 DEBUG 06:39:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:39:08 DEBUG }, 06:39:08 DEBUG "value": [ 06:39:08 DEBUG 1713591548, 06:39:08 DEBUG "2660.6666666666665" 06:39:08 DEBUG ] 06:39:08 DEBUG } 06:39:08 DEBUG ] 06:39:08 DEBUG } 06:39:08 DEBUG } 06:39:08 INFO [http_cmd]: http status code OK 06:39:08 DEBUG --- status code --- 06:39:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:39:08 DEBUG --- http response --- 06:39:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:39:08 DEBUG ----- output ----- 06:39:08 DEBUG { 06:39:08 DEBUG "status": "success", 06:39:08 DEBUG "data": { 06:39:08 DEBUG "resultType": "vector", 06:39:08 DEBUG "result": [ 06:39:08 DEBUG { 06:39:08 DEBUG "metric": { 06:39:08 DEBUG "instance": "10.142.0.107:9100" 06:39:08 DEBUG }, 06:39:08 DEBUG "value": [ 06:39:08 DEBUG 06:39:08 DEBUG 06:39:08 DEBUG 06:39:08 DEBUG [...] 06:39:08 DEBUG 06:39:08 DEBUG 06:39:08 DEBUG 06:39:08 DEBUG "instance": "10.142.0.43:9100" 06:39:08 DEBUG }, 06:39:08 DEBUG "value": [ 06:39:08 DEBUG 1713591548, 06:39:08 DEBUG "81427296.76666667" 06:39:08 DEBUG ] 06:39:08 DEBUG } 06:39:08 DEBUG ] 06:39:08 DEBUG } 06:39:08 DEBUG } 06:39:08 INFO [http_cmd]: http status code OK 06:39:08 DEBUG --- status code --- 06:39:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:39:08 DEBUG --- http response --- 06:39:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:39:08 DEBUG ----- output ----- 06:39:08 DEBUG { 06:39:08 DEBUG "status": "success", 06:39:08 DEBUG "data": { 06:39:08 DEBUG "resultType": "vector", 06:39:08 DEBUG "result": [ 06:39:08 DEBUG { 06:39:08 DEBUG "metric": { 06:39:08 DEBUG "pod": "ds-idrepo-0" 06:39:08 DEBUG }, 06:39:08 DEBUG "value": [ 06:39:08 DEBUG 06:39:08 DEBUG 06:39:08 DEBUG 06:39:08 DEBUG [...] 06:39:08 DEBUG 06:39:08 DEBUG 06:39:08 DEBUG 06:39:08 DEBUG "pod": "ds-cts-2" 06:39:08 DEBUG }, 06:39:08 DEBUG "value": [ 06:39:08 DEBUG 1713591548, 06:39:08 DEBUG "0" 06:39:08 DEBUG ] 06:39:08 DEBUG } 06:39:08 DEBUG ] 06:39:08 DEBUG } 06:39:08 DEBUG } 06:39:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591548" 06:39:08 INFO [http_cmd]: http status code OK 06:39:08 DEBUG --- status code --- 06:39:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:39:08 DEBUG --- http response --- 06:39:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:39:08 DEBUG ----- output ----- 06:39:08 DEBUG { 06:39:08 DEBUG "status": "success", 06:39:08 DEBUG "data": { 06:39:08 DEBUG "resultType": "vector", 06:39:08 DEBUG "result": [ 06:39:08 DEBUG { 06:39:08 DEBUG "metric": { 06:39:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:39:08 DEBUG }, 06:39:08 DEBUG "value": [ 06:39:08 DEBUG 06:39:08 DEBUG 06:39:08 DEBUG 06:39:08 DEBUG [...] 06:39:08 DEBUG 06:39:08 DEBUG 06:39:08 DEBUG 06:39:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:39:08 DEBUG }, 06:39:08 DEBUG "value": [ 06:39:08 DEBUG 1713591548, 06:39:08 DEBUG "0" 06:39:08 DEBUG ] 06:39:08 DEBUG } 06:39:08 DEBUG ] 06:39:08 DEBUG } 06:39:08 DEBUG } 06:39:08 INFO 06:39:08 INFO [loop_until]: kubectl --namespace=xlou top node 06:39:08 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:39:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591548" 06:39:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591548" 06:39:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713591548" 06:39:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591548" 06:39:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591548" 06:39:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591548" 06:39:08 INFO [http_cmd]: http status code OK 06:39:08 DEBUG --- status code --- 06:39:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:39:08 DEBUG --- http response --- 06:39:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:39:08 DEBUG ----- output ----- 06:39:08 DEBUG { 06:39:08 DEBUG "status": "success", 06:39:08 DEBUG "data": { 06:39:08 DEBUG "resultType": "vector", 06:39:08 DEBUG "result": [ 06:39:08 DEBUG { 06:39:08 DEBUG "metric": { 06:39:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:39:08 DEBUG }, 06:39:08 DEBUG "value": [ 06:39:08 DEBUG 06:39:08 DEBUG 06:39:08 DEBUG 06:39:08 DEBUG [...] 06:39:08 DEBUG 06:39:08 DEBUG 06:39:08 DEBUG 06:39:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:39:08 DEBUG }, 06:39:08 DEBUG "value": [ 06:39:08 DEBUG 1713591548, 06:39:08 DEBUG "2694.7666666666664" 06:39:08 DEBUG ] 06:39:08 DEBUG } 06:39:08 DEBUG ] 06:39:08 DEBUG } 06:39:08 DEBUG } 06:39:08 INFO [http_cmd]: http status code OK 06:39:08 DEBUG --- status code --- 06:39:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:39:08 DEBUG --- http response --- 06:39:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 06:39:08 DEBUG ----- output ----- 06:39:08 DEBUG { 06:39:08 DEBUG "status": "success", 06:39:08 DEBUG "data": { 06:39:08 DEBUG "resultType": "vector", 06:39:08 DEBUG "result": [ 06:39:08 DEBUG { 06:39:08 DEBUG "metric": { 06:39:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:39:08 DEBUG }, 06:39:08 DEBUG "value": [ 06:39:08 DEBUG 06:39:08 DEBUG 06:39:08 DEBUG 06:39:08 DEBUG [...] 06:39:08 DEBUG 06:39:08 DEBUG 06:39:08 DEBUG 06:39:08 DEBUG "pod": "am-767f6b67b5-qcqz2" 06:39:08 DEBUG }, 06:39:08 DEBUG "value": [ 06:39:08 DEBUG 1713591548, 06:39:08 DEBUG "11.93780537388412" 06:39:08 DEBUG ] 06:39:08 DEBUG } 06:39:08 DEBUG ] 06:39:08 DEBUG } 06:39:08 DEBUG } 06:39:08 INFO [http_cmd]: http status code OK 06:39:08 DEBUG --- status code --- 06:39:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:39:08 DEBUG --- http response --- 06:39:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:39:08 DEBUG ----- output ----- 06:39:08 DEBUG { 06:39:08 DEBUG "status": "success", 06:39:08 DEBUG "data": { 06:39:08 DEBUG "resultType": "vector", 06:39:08 DEBUG "result": [ 06:39:08 DEBUG { 06:39:08 DEBUG "metric": { 06:39:08 DEBUG "instance": "10.142.0.107:9100" 06:39:08 DEBUG }, 06:39:08 DEBUG "value": [ 06:39:08 DEBUG 06:39:08 DEBUG 06:39:08 DEBUG 06:39:08 DEBUG [...] 06:39:08 DEBUG 06:39:08 DEBUG 06:39:08 DEBUG 06:39:08 DEBUG "instance": "10.142.0.43:9100" 06:39:08 DEBUG }, 06:39:08 DEBUG "value": [ 06:39:08 DEBUG 1713591548, 06:39:08 DEBUG "0.039666666666664685" 06:39:08 DEBUG ] 06:39:08 DEBUG } 06:39:08 DEBUG ] 06:39:08 DEBUG } 06:39:08 DEBUG } 06:39:08 INFO [http_cmd]: http status code OK 06:39:08 DEBUG --- status code --- 06:39:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:39:08 DEBUG --- http response --- 06:39:08 INFO [http_cmd]: http status code OK 06:39:08 DEBUG --- status code --- 06:39:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:39:08 DEBUG --- http response --- 06:39:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 06:39:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:39:08 INFO [http_cmd]: http status code OK 06:39:08 DEBUG ----- output ----- 06:39:08 DEBUG ----- output ----- 06:39:08 DEBUG --- status code --- 06:39:08 DEBUG { 06:39:08 DEBUG { 06:39:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:39:08 DEBUG "status": "success", 06:39:08 DEBUG "status": "success", 06:39:08 DEBUG --- http response --- 06:39:08 DEBUG "data": { 06:39:08 DEBUG "data": { 06:39:08 DEBUG "resultType": "vector", 06:39:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:39:08 DEBUG "resultType": "vector", 06:39:08 DEBUG "result": [ 06:39:08 DEBUG ----- output ----- 06:39:08 DEBUG "result": [ 06:39:08 DEBUG { 06:39:08 DEBUG { 06:39:08 DEBUG { 06:39:08 DEBUG "metric": { 06:39:08 DEBUG "status": "success", 06:39:08 DEBUG "metric": { 06:39:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:39:08 DEBUG "data": { 06:39:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:39:08 DEBUG }, 06:39:08 DEBUG "resultType": "vector", 06:39:08 DEBUG }, 06:39:08 DEBUG "value": [ 06:39:08 DEBUG "result": [ 06:39:08 DEBUG "value": [ 06:39:08 DEBUG 06:39:08 DEBUG { 06:39:08 DEBUG 06:39:08 DEBUG 06:39:08 DEBUG "metric": { 06:39:08 DEBUG 06:39:08 DEBUG 06:39:08 DEBUG "pod": "ds-cts-0" 06:39:08 DEBUG 06:39:08 DEBUG [...] 06:39:08 DEBUG }, 06:39:08 DEBUG [...] 06:39:08 DEBUG 06:39:08 DEBUG "value": [ 06:39:08 DEBUG 06:39:08 DEBUG 06:39:08 DEBUG 06:39:08 DEBUG 06:39:08 DEBUG 06:39:08 DEBUG 06:39:08 DEBUG 06:39:08 DEBUG "pod": "am-767f6b67b5-qcqz2" 06:39:08 DEBUG 06:39:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:39:08 DEBUG }, 06:39:08 DEBUG [...] 06:39:08 DEBUG }, 06:39:08 DEBUG "value": [ 06:39:08 DEBUG 06:39:08 DEBUG "value": [ 06:39:08 DEBUG 1713591548, 06:39:08 DEBUG 06:39:08 DEBUG 1713591548, 06:39:08 DEBUG "231435614.25389245" 06:39:08 DEBUG 06:39:08 DEBUG "2694.5666666666666" 06:39:08 DEBUG ] 06:39:08 DEBUG "pod": "ds-cts-2" 06:39:08 DEBUG ] 06:39:08 DEBUG } 06:39:08 DEBUG }, 06:39:08 DEBUG } 06:39:08 DEBUG ] 06:39:08 DEBUG "value": [ 06:39:08 DEBUG ] 06:39:08 DEBUG } 06:39:08 DEBUG 1713591548, 06:39:08 DEBUG } 06:39:08 DEBUG } 06:39:08 DEBUG "0.00033215555555555555" 06:39:08 DEBUG } 06:39:08 DEBUG ] 06:39:08 DEBUG } 06:39:08 DEBUG ] 06:39:08 DEBUG } 06:39:08 DEBUG } 06:39:08 INFO [loop_until]: OK (rc = 0) 06:39:08 DEBUG --- stdout --- 06:39:08 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 69m 0% 6271Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 8036m 26% 7399Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 17503m 58% 11354Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 79m 0% 3382Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 18199m 60% 13533Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 161m 0% 5212Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 17766m 59% 11587Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 17951m 60% 3094Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 19039m 63% 10859Mi 9% gke-xlou-cdm-ds-0f601751-j488 8268m 27% 7026Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 16893m 56% 15584Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 8169m 27% 7640Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 18790m 62% 12651Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 8166m 51% 8538Mi 14% 06:39:08 DEBUG --- stderr --- 06:39:08 DEBUG 06:39:08 INFO 06:39:08 INFO [loop_until]: kubectl --namespace=xlou top pods 06:39:08 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:39:08 INFO [loop_until]: OK (rc = 0) 06:39:08 DEBUG --- stdout --- 06:39:08 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 17711m 8642Mi am-767f6b67b5-qcqz2 18060m 8709Mi am-767f6b67b5-vbltz 19078m 8652Mi ds-cts-0 18263m 10381Mi ds-cts-1 16517m 10657Mi ds-cts-2 19062m 10046Mi ds-idrepo-0 7658m 3728Mi ds-idrepo-1 7625m 3758Mi ds-idrepo-2 7842m 3701Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1177Mi idm-b85f9d978-wnhxb 4m 1236Mi lodemon-7cb5b9456d-dskfn 2m 62Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7827m 4699Mi 06:39:08 DEBUG --- stderr --- 06:39:08 DEBUG 06:40:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713591607" 06:40:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591607" 06:40:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591607" 06:40:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591607" 06:40:07 ERROR [http_cmd]: ERROR 06:40:07 ERROR --- status code --- 06:40:07 ERROR http status code is 400 (expected [200, 401, 403, 404]) 06:40:07 ERROR --- http response --- 06:40:07 INFO [http_cmd]: http status code OK 06:40:07 INFO [http_cmd]: http status code OK 06:40:07 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 06:40:07 DEBUG --- status code --- 06:40:07 DEBUG --- status code --- 06:40:07 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 06:40:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:40:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:40:07 DEBUG --- http response --- 06:40:07 DEBUG --- http response --- 06:40:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 06:40:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:40:07 DEBUG ----- output ----- 06:40:07 DEBUG ----- output ----- 06:40:07 DEBUG { 06:40:07 DEBUG { 06:40:07 DEBUG "status": "success", 06:40:07 DEBUG "status": "success", 06:40:07 DEBUG "data": { 06:40:07 DEBUG "data": { 06:40:07 DEBUG "resultType": "vector", 06:40:07 DEBUG "resultType": "vector", 06:40:07 DEBUG "result": [ 06:40:07 DEBUG "result": [ 06:40:07 DEBUG { 06:40:07 DEBUG { 06:40:07 DEBUG "metric": { 06:40:07 DEBUG "metric": { 06:40:07 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:40:07 DEBUG "pod": "am-767f6b67b5-9zw22" 06:40:07 DEBUG }, 06:40:07 DEBUG }, 06:40:07 DEBUG "value": [ 06:40:07 DEBUG "value": [ 06:40:07 DEBUG 06:40:07 DEBUG 06:40:07 DEBUG 06:40:07 DEBUG 06:40:07 DEBUG 06:40:07 DEBUG 06:40:07 DEBUG [...] 06:40:07 DEBUG [...] 06:40:07 DEBUG 06:40:07 DEBUG 06:40:07 DEBUG 06:40:07 DEBUG 06:40:07 DEBUG 06:40:07 DEBUG 06:40:07 DEBUG "pod": "ds-idrepo-2" 06:40:07 DEBUG "pod": "am-767f6b67b5-vbltz" 06:40:07 DEBUG }, 06:40:07 DEBUG }, 06:40:07 DEBUG "value": [ 06:40:07 DEBUG "value": [ 06:40:07 DEBUG 1713591607, 06:40:07 DEBUG 1713591607, 06:40:07 DEBUG "3082.079776684052" 06:40:07 DEBUG "483805.5" 06:40:07 DEBUG ] 06:40:07 DEBUG ] 06:40:07 DEBUG } 06:40:07 DEBUG } 06:40:07 DEBUG ] 06:40:07 DEBUG ] 06:40:07 DEBUG } 06:40:07 DEBUG } 06:40:07 DEBUG } 06:40:07 DEBUG } 06:40:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591607" 06:40:07 INFO [http_cmd]: http status code OK 06:40:07 DEBUG --- status code --- 06:40:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:40:07 DEBUG --- http response --- 06:40:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 06:40:07 DEBUG ----- output ----- 06:40:07 DEBUG { 06:40:07 DEBUG "status": "success", 06:40:07 DEBUG "data": { 06:40:07 DEBUG "resultType": "vector", 06:40:07 DEBUG "result": [ 06:40:07 DEBUG { 06:40:07 DEBUG "metric": { 06:40:07 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:40:07 DEBUG }, 06:40:07 DEBUG "value": [ 06:40:07 DEBUG 06:40:07 DEBUG 06:40:07 DEBUG 06:40:07 DEBUG [...] 06:40:07 DEBUG 06:40:07 DEBUG 06:40:07 DEBUG 06:40:07 DEBUG "pod": "ds-idrepo-2" 06:40:07 DEBUG }, 06:40:07 DEBUG "value": [ 06:40:07 DEBUG 1713591607, 06:40:07 DEBUG "0" 06:40:07 DEBUG ] 06:40:07 DEBUG } 06:40:07 DEBUG ] 06:40:07 DEBUG } 06:40:07 DEBUG } 06:40:07 INFO [http_cmd]: http status code OK 06:40:07 DEBUG --- status code --- 06:40:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:40:07 DEBUG --- http response --- 06:40:07 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 06:40:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713591607" 06:40:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591607" 06:40:07 INFO [http_cmd]: http status code OK 06:40:07 DEBUG --- status code --- 06:40:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:40:07 DEBUG --- http response --- 06:40:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:40:07 DEBUG ----- output ----- 06:40:07 DEBUG { 06:40:07 DEBUG "status": "success", 06:40:07 DEBUG "data": { 06:40:07 DEBUG "resultType": "vector", 06:40:07 DEBUG "result": [ 06:40:07 DEBUG { 06:40:07 DEBUG "metric": { 06:40:07 DEBUG "instance": "10.142.0.107:9100" 06:40:07 DEBUG }, 06:40:07 DEBUG "value": [ 06:40:07 DEBUG 06:40:07 DEBUG 06:40:07 DEBUG 06:40:07 DEBUG [...] 06:40:07 DEBUG 06:40:07 DEBUG 06:40:07 DEBUG 06:40:07 DEBUG "instance": "10.142.0.43:9100" 06:40:07 DEBUG }, 06:40:07 DEBUG "value": [ 06:40:07 DEBUG 1713591607, 06:40:07 DEBUG "3179437151232" 06:40:07 DEBUG ] 06:40:07 DEBUG } 06:40:07 DEBUG ] 06:40:07 DEBUG } 06:40:07 DEBUG } 06:40:07 INFO [http_cmd]: http status code OK 06:40:07 DEBUG --- status code --- 06:40:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:40:07 DEBUG --- http response --- 06:40:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:40:07 DEBUG ----- output ----- 06:40:07 DEBUG { 06:40:07 DEBUG "status": "success", 06:40:07 DEBUG "data": { 06:40:07 DEBUG "resultType": "vector", 06:40:07 DEBUG "result": [ 06:40:07 DEBUG { 06:40:07 DEBUG "metric": { 06:40:07 DEBUG "pod": "ds-idrepo-0" 06:40:07 DEBUG }, 06:40:07 DEBUG "value": [ 06:40:07 DEBUG 06:40:07 DEBUG 06:40:07 DEBUG 06:40:07 DEBUG [...] 06:40:07 DEBUG 06:40:07 DEBUG 06:40:07 DEBUG 06:40:07 DEBUG "pod": "ds-idrepo-2" 06:40:07 DEBUG }, 06:40:07 DEBUG "value": [ 06:40:07 DEBUG 1713591607, 06:40:07 DEBUG "0" 06:40:07 DEBUG ] 06:40:07 DEBUG } 06:40:07 DEBUG ] 06:40:07 DEBUG } 06:40:07 DEBUG } 06:40:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591607" 06:40:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713591607" 06:40:08 INFO [http_cmd]: http status code OK 06:40:08 DEBUG --- status code --- 06:40:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:40:08 DEBUG --- http response --- 06:40:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:40:08 DEBUG ----- output ----- 06:40:08 DEBUG { 06:40:08 DEBUG "status": "success", 06:40:08 DEBUG "data": { 06:40:08 DEBUG "resultType": "vector", 06:40:08 DEBUG "result": [ 06:40:08 DEBUG { 06:40:08 DEBUG "metric": { 06:40:08 DEBUG "pod": "ds-idrepo-0" 06:40:08 DEBUG }, 06:40:08 DEBUG "value": [ 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG [...] 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG "pod": "ds-idrepo-2" 06:40:08 DEBUG }, 06:40:08 DEBUG "value": [ 06:40:08 DEBUG 1713591607, 06:40:08 DEBUG "0" 06:40:08 DEBUG ] 06:40:08 DEBUG } 06:40:08 DEBUG ] 06:40:08 DEBUG } 06:40:08 DEBUG } 06:40:08 INFO [http_cmd]: http status code OK 06:40:08 DEBUG --- status code --- 06:40:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:40:08 DEBUG --- http response --- 06:40:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:40:08 DEBUG ----- output ----- 06:40:08 DEBUG { 06:40:08 DEBUG "status": "success", 06:40:08 DEBUG "data": { 06:40:08 DEBUG "resultType": "vector", 06:40:08 DEBUG "result": [ 06:40:08 DEBUG { 06:40:08 DEBUG "metric": { 06:40:08 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 06:40:08 DEBUG }, 06:40:08 DEBUG "value": [ 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG [...] 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 06:40:08 DEBUG }, 06:40:08 DEBUG "value": [ 06:40:08 DEBUG 1713591607, 06:40:08 DEBUG "5000507392" 06:40:08 DEBUG ] 06:40:08 DEBUG } 06:40:08 DEBUG ] 06:40:08 DEBUG } 06:40:08 DEBUG } 06:40:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591608" 06:40:08 INFO [http_cmd]: http status code OK 06:40:08 DEBUG --- status code --- 06:40:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:40:08 DEBUG --- http response --- 06:40:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:40:08 DEBUG ----- output ----- 06:40:08 DEBUG { 06:40:08 DEBUG "status": "success", 06:40:08 DEBUG "data": { 06:40:08 DEBUG "resultType": "vector", 06:40:08 DEBUG "result": [ 06:40:08 DEBUG { 06:40:08 DEBUG "metric": { 06:40:08 DEBUG "pod": "ds-cts-0" 06:40:08 DEBUG }, 06:40:08 DEBUG "value": [ 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG [...] 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG "pod": "ds-cts-2" 06:40:08 DEBUG }, 06:40:08 DEBUG "value": [ 06:40:08 DEBUG 1713591608, 06:40:08 DEBUG "0" 06:40:08 DEBUG ] 06:40:08 DEBUG } 06:40:08 DEBUG ] 06:40:08 DEBUG } 06:40:08 DEBUG } 06:40:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591608" 06:40:08 INFO [http_cmd]: http status code OK 06:40:08 DEBUG --- status code --- 06:40:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:40:08 DEBUG --- http response --- 06:40:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:40:08 DEBUG ----- output ----- 06:40:08 DEBUG { 06:40:08 DEBUG "status": "success", 06:40:08 DEBUG "data": { 06:40:08 DEBUG "resultType": "vector", 06:40:08 DEBUG "result": [ 06:40:08 DEBUG { 06:40:08 DEBUG "metric": { 06:40:08 DEBUG "pod": "ds-cts-0" 06:40:08 DEBUG }, 06:40:08 DEBUG "value": [ 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG [...] 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG "pod": "ds-cts-2" 06:40:08 DEBUG }, 06:40:08 DEBUG "value": [ 06:40:08 DEBUG 1713591608, 06:40:08 DEBUG "0" 06:40:08 DEBUG ] 06:40:08 DEBUG } 06:40:08 DEBUG ] 06:40:08 DEBUG } 06:40:08 DEBUG } 06:40:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591608" 06:40:08 INFO [http_cmd]: http status code OK 06:40:08 DEBUG --- status code --- 06:40:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:40:08 DEBUG --- http response --- 06:40:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:40:08 DEBUG ----- output ----- 06:40:08 DEBUG { 06:40:08 DEBUG "status": "success", 06:40:08 DEBUG "data": { 06:40:08 DEBUG "resultType": "vector", 06:40:08 DEBUG "result": [ 06:40:08 DEBUG { 06:40:08 DEBUG "metric": { 06:40:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:40:08 DEBUG }, 06:40:08 DEBUG "value": [ 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG [...] 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:40:08 DEBUG }, 06:40:08 DEBUG "value": [ 06:40:08 DEBUG 1713591608, 06:40:08 DEBUG "2669.1666666666665" 06:40:08 DEBUG ] 06:40:08 DEBUG } 06:40:08 DEBUG ] 06:40:08 DEBUG } 06:40:08 DEBUG } 06:40:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713591608" 06:40:08 INFO [http_cmd]: http status code OK 06:40:08 DEBUG --- status code --- 06:40:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:40:08 DEBUG --- http response --- 06:40:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:40:08 DEBUG ----- output ----- 06:40:08 DEBUG { 06:40:08 DEBUG "status": "success", 06:40:08 DEBUG "data": { 06:40:08 DEBUG "resultType": "vector", 06:40:08 DEBUG "result": [ 06:40:08 DEBUG { 06:40:08 DEBUG "metric": { 06:40:08 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 06:40:08 DEBUG }, 06:40:08 DEBUG "value": [ 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG [...] 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 06:40:08 DEBUG }, 06:40:08 DEBUG "value": [ 06:40:08 DEBUG 1713591608, 06:40:08 DEBUG "8.019716548102595" 06:40:08 DEBUG ] 06:40:08 DEBUG } 06:40:08 DEBUG ] 06:40:08 DEBUG } 06:40:08 DEBUG } 06:40:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591608" 06:40:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713591608" 06:40:08 INFO [http_cmd]: http status code OK 06:40:08 DEBUG --- status code --- 06:40:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:40:08 DEBUG --- http response --- 06:40:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:40:08 DEBUG ----- output ----- 06:40:08 DEBUG { 06:40:08 DEBUG "status": "success", 06:40:08 DEBUG "data": { 06:40:08 DEBUG "resultType": "vector", 06:40:08 DEBUG "result": [ 06:40:08 DEBUG { 06:40:08 DEBUG "metric": { 06:40:08 DEBUG "pod": "ds-idrepo-0" 06:40:08 DEBUG }, 06:40:08 DEBUG "value": [ 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG [...] 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG "pod": "ds-idrepo-2" 06:40:08 DEBUG }, 06:40:08 DEBUG "value": [ 06:40:08 DEBUG 1713591608, 06:40:08 DEBUG "0" 06:40:08 DEBUG ] 06:40:08 DEBUG } 06:40:08 DEBUG ] 06:40:08 DEBUG } 06:40:08 DEBUG } 06:40:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713591608" 06:40:08 INFO [http_cmd]: http status code OK 06:40:08 DEBUG --- status code --- 06:40:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:40:08 DEBUG --- http response --- 06:40:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:40:08 DEBUG ----- output ----- 06:40:08 DEBUG { 06:40:08 DEBUG "status": "success", 06:40:08 DEBUG "data": { 06:40:08 DEBUG "resultType": "vector", 06:40:08 DEBUG "result": [ 06:40:08 DEBUG { 06:40:08 DEBUG "metric": { 06:40:08 DEBUG "pod": "ds-idrepo-0" 06:40:08 DEBUG }, 06:40:08 DEBUG "value": [ 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG [...] 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG "pod": "ds-cts-2" 06:40:08 DEBUG }, 06:40:08 DEBUG "value": [ 06:40:08 DEBUG 1713591608, 06:40:08 DEBUG "0" 06:40:08 DEBUG ] 06:40:08 DEBUG } 06:40:08 DEBUG ] 06:40:08 DEBUG } 06:40:08 DEBUG } 06:40:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713591608" 06:40:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713591608" 06:40:08 INFO [http_cmd]: http status code OK 06:40:08 DEBUG --- status code --- 06:40:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:40:08 DEBUG --- http response --- 06:40:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:40:08 DEBUG ----- output ----- 06:40:08 DEBUG { 06:40:08 DEBUG "status": "success", 06:40:08 DEBUG "data": { 06:40:08 DEBUG "resultType": "vector", 06:40:08 DEBUG "result": [ 06:40:08 DEBUG { 06:40:08 DEBUG "metric": { 06:40:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:40:08 DEBUG }, 06:40:08 DEBUG "value": [ 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG [...] 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:40:08 DEBUG }, 06:40:08 DEBUG "value": [ 06:40:08 DEBUG 1713591608, 06:40:08 DEBUG "0.095" 06:40:08 DEBUG ] 06:40:08 DEBUG } 06:40:08 DEBUG ] 06:40:08 DEBUG } 06:40:08 DEBUG } 06:40:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713591608" 06:40:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591608" 06:40:08 INFO [http_cmd]: http status code OK 06:40:08 DEBUG --- status code --- 06:40:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:40:08 DEBUG --- http response --- 06:40:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:40:08 DEBUG ----- output ----- 06:40:08 DEBUG { 06:40:08 DEBUG "status": "success", 06:40:08 DEBUG "data": { 06:40:08 DEBUG "resultType": "vector", 06:40:08 DEBUG "result": [ 06:40:08 DEBUG { 06:40:08 DEBUG "metric": { 06:40:08 DEBUG "instance": "10.142.0.107:9100" 06:40:08 DEBUG }, 06:40:08 DEBUG "value": [ 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG [...] 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG "instance": "10.142.0.43:9100" 06:40:08 DEBUG }, 06:40:08 DEBUG "value": [ 06:40:08 DEBUG 1713591608, 06:40:08 DEBUG "0.17816666666658987" 06:40:08 DEBUG ] 06:40:08 DEBUG } 06:40:08 DEBUG ] 06:40:08 DEBUG } 06:40:08 DEBUG } 06:40:08 INFO [http_cmd]: http status code OK 06:40:08 DEBUG --- status code --- 06:40:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:40:08 DEBUG --- http response --- 06:40:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:40:08 DEBUG ----- output ----- 06:40:08 DEBUG { 06:40:08 DEBUG "status": "success", 06:40:08 DEBUG "data": { 06:40:08 DEBUG "resultType": "vector", 06:40:08 DEBUG "result": [ 06:40:08 DEBUG { 06:40:08 DEBUG "metric": { 06:40:08 DEBUG "instance": "10.142.0.107:9100" 06:40:08 DEBUG }, 06:40:08 DEBUG "value": [ 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG [...] 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG "instance": "10.142.0.43:9100" 06:40:08 DEBUG }, 06:40:08 DEBUG "value": [ 06:40:08 DEBUG 1713591608, 06:40:08 DEBUG "80338745.16666667" 06:40:08 DEBUG ] 06:40:08 DEBUG } 06:40:08 DEBUG ] 06:40:08 DEBUG } 06:40:08 DEBUG } 06:40:08 INFO [http_cmd]: http status code OK 06:40:08 DEBUG --- status code --- 06:40:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:40:08 DEBUG --- http response --- 06:40:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:40:08 DEBUG ----- output ----- 06:40:08 DEBUG { 06:40:08 DEBUG "status": "success", 06:40:08 DEBUG "data": { 06:40:08 DEBUG "resultType": "vector", 06:40:08 DEBUG "result": [ 06:40:08 DEBUG { 06:40:08 DEBUG "metric": { 06:40:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:40:08 DEBUG }, 06:40:08 DEBUG "value": [ 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG [...] 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:40:08 DEBUG }, 06:40:08 DEBUG "value": [ 06:40:08 DEBUG 1713591608, 06:40:08 DEBUG "2669.9" 06:40:08 DEBUG ] 06:40:08 DEBUG } 06:40:08 DEBUG ] 06:40:08 DEBUG } 06:40:08 DEBUG } 06:40:08 INFO [http_cmd]: http status code OK 06:40:08 DEBUG --- status code --- 06:40:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:40:08 DEBUG --- http response --- 06:40:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:40:08 DEBUG ----- output ----- 06:40:08 DEBUG { 06:40:08 DEBUG "status": "success", 06:40:08 DEBUG "data": { 06:40:08 DEBUG "resultType": "vector", 06:40:08 DEBUG "result": [ 06:40:08 DEBUG { 06:40:08 DEBUG "metric": { 06:40:08 DEBUG "instance": "10.142.0.107:9100" 06:40:08 DEBUG }, 06:40:08 DEBUG "value": [ 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG [...] 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG "instance": "10.142.0.43:9100" 06:40:08 DEBUG }, 06:40:08 DEBUG "value": [ 06:40:08 DEBUG 1713591608, 06:40:08 DEBUG "80342896.06666666" 06:40:08 DEBUG ] 06:40:08 DEBUG } 06:40:08 DEBUG ] 06:40:08 DEBUG } 06:40:08 DEBUG } 06:40:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591608" 06:40:08 INFO [http_cmd]: http status code OK 06:40:08 DEBUG --- status code --- 06:40:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:40:08 DEBUG --- http response --- 06:40:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:40:08 DEBUG ----- output ----- 06:40:08 DEBUG { 06:40:08 DEBUG "status": "success", 06:40:08 DEBUG "data": { 06:40:08 DEBUG "resultType": "vector", 06:40:08 DEBUG "result": [ 06:40:08 DEBUG { 06:40:08 DEBUG "metric": { 06:40:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:40:08 DEBUG }, 06:40:08 DEBUG "value": [ 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG [...] 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:40:08 DEBUG }, 06:40:08 DEBUG "value": [ 06:40:08 DEBUG 1713591608, 06:40:08 DEBUG "0" 06:40:08 DEBUG ] 06:40:08 DEBUG } 06:40:08 DEBUG ] 06:40:08 DEBUG } 06:40:08 DEBUG } 06:40:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591608" 06:40:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591608" 06:40:08 INFO [http_cmd]: http status code OK 06:40:08 DEBUG --- status code --- 06:40:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:40:08 DEBUG --- http response --- 06:40:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:40:08 DEBUG ----- output ----- 06:40:08 DEBUG { 06:40:08 DEBUG "status": "success", 06:40:08 DEBUG "data": { 06:40:08 DEBUG "resultType": "vector", 06:40:08 DEBUG "result": [ 06:40:08 DEBUG { 06:40:08 DEBUG "metric": { 06:40:08 DEBUG "pod": "ds-cts-0" 06:40:08 DEBUG }, 06:40:08 DEBUG "value": [ 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG [...] 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG "pod": "ds-cts-2" 06:40:08 DEBUG }, 06:40:08 DEBUG "value": [ 06:40:08 DEBUG 1713591608, 06:40:08 DEBUG "0.00009999999999999998" 06:40:08 DEBUG ] 06:40:08 DEBUG } 06:40:08 DEBUG ] 06:40:08 DEBUG } 06:40:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591608" 06:40:08 DEBUG } 06:40:08 INFO [http_cmd]: http status code OK 06:40:08 DEBUG --- status code --- 06:40:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:40:08 DEBUG --- http response --- 06:40:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:40:08 DEBUG ----- output ----- 06:40:08 DEBUG { 06:40:08 DEBUG "status": "success", 06:40:08 DEBUG "data": { 06:40:08 DEBUG "resultType": "vector", 06:40:08 DEBUG "result": [ 06:40:08 DEBUG { 06:40:08 DEBUG "metric": { 06:40:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:40:08 DEBUG }, 06:40:08 DEBUG "value": [ 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG [...] 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:40:08 DEBUG }, 06:40:08 DEBUG "value": [ 06:40:08 DEBUG 1713591608, 06:40:08 DEBUG "2668.633333333333" 06:40:08 DEBUG ] 06:40:08 DEBUG } 06:40:08 DEBUG ] 06:40:08 DEBUG } 06:40:08 DEBUG } 06:40:08 INFO [http_cmd]: http status code OK 06:40:08 DEBUG --- status code --- 06:40:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:40:08 DEBUG --- http response --- 06:40:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 06:40:08 DEBUG ----- output ----- 06:40:08 DEBUG { 06:40:08 DEBUG "status": "success", 06:40:08 DEBUG "data": { 06:40:08 DEBUG "resultType": "vector", 06:40:08 DEBUG "result": [ 06:40:08 DEBUG { 06:40:08 DEBUG "metric": { 06:40:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:40:08 DEBUG }, 06:40:08 DEBUG "value": [ 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG [...] 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG "pod": "ds-idrepo-2" 06:40:08 DEBUG }, 06:40:08 DEBUG "value": [ 06:40:08 DEBUG 1713591608, 06:40:08 DEBUG "0" 06:40:08 DEBUG ] 06:40:08 DEBUG } 06:40:08 DEBUG ] 06:40:08 DEBUG } 06:40:08 DEBUG } 06:40:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713591608" 06:40:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591608" 06:40:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591608" 06:40:08 INFO [http_cmd]: http status code OK 06:40:08 DEBUG --- status code --- 06:40:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:40:08 DEBUG --- http response --- 06:40:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:40:08 DEBUG ----- output ----- 06:40:08 DEBUG { 06:40:08 DEBUG "status": "success", 06:40:08 DEBUG "data": { 06:40:08 DEBUG "resultType": "vector", 06:40:08 DEBUG "result": [ 06:40:08 DEBUG { 06:40:08 DEBUG "metric": { 06:40:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:40:08 DEBUG }, 06:40:08 DEBUG "value": [ 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG [...] 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:40:08 DEBUG }, 06:40:08 DEBUG "value": [ 06:40:08 DEBUG 1713591608, 06:40:08 DEBUG "2668.866666666667" 06:40:08 DEBUG ] 06:40:08 DEBUG } 06:40:08 DEBUG ] 06:40:08 DEBUG } 06:40:08 DEBUG } 06:40:08 INFO [http_cmd]: http status code OK 06:40:08 DEBUG --- status code --- 06:40:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:40:08 DEBUG --- http response --- 06:40:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:40:08 DEBUG ----- output ----- 06:40:08 DEBUG { 06:40:08 DEBUG "status": "success", 06:40:08 DEBUG "data": { 06:40:08 DEBUG "resultType": "vector", 06:40:08 DEBUG "result": [ 06:40:08 DEBUG { 06:40:08 DEBUG "metric": { 06:40:08 DEBUG "instance": "10.142.0.107:9100" 06:40:08 DEBUG }, 06:40:08 DEBUG "value": [ 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG [...] 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG "instance": "10.142.0.43:9100" 06:40:08 DEBUG }, 06:40:08 DEBUG "value": [ 06:40:08 DEBUG 1713591608, 06:40:08 DEBUG "0.06399999999998728" 06:40:08 DEBUG ] 06:40:08 DEBUG } 06:40:08 DEBUG ] 06:40:08 DEBUG } 06:40:08 DEBUG } 06:40:08 INFO [http_cmd]: http status code OK 06:40:08 DEBUG --- status code --- 06:40:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:40:08 DEBUG --- http response --- 06:40:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 06:40:08 DEBUG ----- output ----- 06:40:08 DEBUG { 06:40:08 DEBUG "status": "success", 06:40:08 DEBUG "data": { 06:40:08 DEBUG "resultType": "vector", 06:40:08 DEBUG "result": [ 06:40:08 DEBUG { 06:40:08 DEBUG "metric": { 06:40:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:40:08 DEBUG }, 06:40:08 DEBUG "value": [ 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG [...] 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG 06:40:08 DEBUG "pod": "ds-idrepo-2" 06:40:08 DEBUG }, 06:40:08 DEBUG "value": [ 06:40:08 DEBUG 1713591608, 06:40:08 DEBUG "0" 06:40:08 DEBUG ] 06:40:08 DEBUG } 06:40:08 DEBUG ] 06:40:08 DEBUG } 06:40:08 DEBUG } 06:40:08 INFO 06:40:08 INFO [loop_until]: kubectl --namespace=xlou top node 06:40:08 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:40:08 INFO 06:40:08 INFO [loop_until]: kubectl --namespace=xlou top pods 06:40:08 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:40:08 INFO [loop_until]: OK (rc = 0) 06:40:08 DEBUG --- stdout --- 06:40:08 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 68m 0% 6282Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7993m 26% 7404Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 17077m 57% 11356Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 81m 0% 3379Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 18335m 61% 13526Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 156m 0% 5195Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 17437m 58% 11589Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 19237m 64% 3026Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 19333m 64% 10865Mi 9% gke-xlou-cdm-ds-0f601751-j488 8192m 27% 7020Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 17012m 56% 15586Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 8083m 27% 7665Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 18926m 63% 12656Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 8159m 51% 8516Mi 14% 06:40:08 DEBUG --- stderr --- 06:40:08 DEBUG 06:40:08 INFO [loop_until]: OK (rc = 0) 06:40:08 DEBUG --- stdout --- 06:40:08 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 17351m 8648Mi am-767f6b67b5-qcqz2 17132m 8709Mi am-767f6b67b5-vbltz 19028m 8650Mi ds-cts-0 18246m 10382Mi ds-cts-1 16672m 10658Mi ds-cts-2 19546m 10052Mi ds-idrepo-0 7942m 3738Mi ds-idrepo-1 7667m 3762Mi ds-idrepo-2 7659m 3695Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 3m 1177Mi idm-b85f9d978-wnhxb 3m 1236Mi lodemon-7cb5b9456d-dskfn 2m 62Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7961m 4672Mi 06:40:08 DEBUG --- stderr --- 06:40:08 DEBUG 06:41:07 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591667" 06:41:07 INFO [http_cmd]: http status code OK 06:41:07 DEBUG --- status code --- 06:41:07 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:41:07 DEBUG --- http response --- 06:41:07 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:41:07 DEBUG ----- output ----- 06:41:07 DEBUG { 06:41:07 DEBUG "status": "success", 06:41:07 DEBUG "data": { 06:41:07 DEBUG "resultType": "vector", 06:41:07 DEBUG "result": [ 06:41:07 DEBUG { 06:41:07 DEBUG "metric": { 06:41:07 DEBUG "pod": "am-767f6b67b5-9zw22" 06:41:07 DEBUG }, 06:41:07 DEBUG "value": [ 06:41:07 DEBUG 06:41:07 DEBUG 06:41:07 DEBUG 06:41:07 DEBUG [...] 06:41:07 DEBUG 06:41:07 DEBUG 06:41:07 DEBUG 06:41:07 DEBUG "pod": "am-767f6b67b5-vbltz" 06:41:07 DEBUG }, 06:41:07 DEBUG "value": [ 06:41:07 DEBUG 1713591667, 06:41:07 DEBUG "482508" 06:41:07 DEBUG ] 06:41:07 DEBUG } 06:41:07 DEBUG ] 06:41:07 DEBUG } 06:41:07 DEBUG } 06:41:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713591668" 06:41:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591668" 06:41:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591668" 06:41:08 ERROR [http_cmd]: ERROR 06:41:08 ERROR --- status code --- 06:41:08 ERROR http status code is 400 (expected [200, 401, 403, 404]) 06:41:08 ERROR --- http response --- 06:41:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713591668" 06:41:08 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 06:41:08 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 06:41:08 INFO [http_cmd]: http status code OK 06:41:08 DEBUG --- status code --- 06:41:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:41:08 DEBUG --- http response --- 06:41:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:41:08 DEBUG ----- output ----- 06:41:08 DEBUG { 06:41:08 DEBUG "status": "success", 06:41:08 DEBUG "data": { 06:41:08 DEBUG "resultType": "vector", 06:41:08 DEBUG "result": [ 06:41:08 DEBUG { 06:41:08 DEBUG "metric": { 06:41:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:41:08 DEBUG }, 06:41:08 DEBUG "value": [ 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG [...] 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG "pod": "am-767f6b67b5-qcqz2" 06:41:08 DEBUG }, 06:41:08 DEBUG "value": [ 06:41:08 DEBUG 1713591668, 06:41:08 DEBUG "0" 06:41:08 DEBUG ] 06:41:08 DEBUG } 06:41:08 DEBUG ] 06:41:08 DEBUG } 06:41:08 DEBUG } 06:41:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591668" 06:41:08 INFO [http_cmd]: http status code OK 06:41:08 DEBUG --- status code --- 06:41:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:41:08 DEBUG --- http response --- 06:41:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:41:08 DEBUG ----- output ----- 06:41:08 DEBUG { 06:41:08 DEBUG "status": "success", 06:41:08 DEBUG "data": { 06:41:08 DEBUG "resultType": "vector", 06:41:08 DEBUG "result": [ 06:41:08 DEBUG { 06:41:08 DEBUG "metric": { 06:41:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:41:08 DEBUG }, 06:41:08 DEBUG "value": [ 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG [...] 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG "pod": "am-767f6b67b5-qcqz2" 06:41:08 DEBUG }, 06:41:08 DEBUG "value": [ 06:41:08 DEBUG 1713591668, 06:41:08 DEBUG "0" 06:41:08 DEBUG ] 06:41:08 DEBUG } 06:41:08 DEBUG ] 06:41:08 DEBUG } 06:41:08 DEBUG } 06:41:08 INFO [http_cmd]: http status code OK 06:41:08 DEBUG --- status code --- 06:41:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:41:08 DEBUG --- http response --- 06:41:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:41:08 DEBUG ----- output ----- 06:41:08 DEBUG { 06:41:08 DEBUG "status": "success", 06:41:08 DEBUG "data": { 06:41:08 DEBUG "resultType": "vector", 06:41:08 DEBUG "result": [ 06:41:08 DEBUG { 06:41:08 DEBUG "metric": { 06:41:08 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 06:41:08 DEBUG }, 06:41:08 DEBUG "value": [ 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG [...] 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 06:41:08 DEBUG }, 06:41:08 DEBUG "value": [ 06:41:08 DEBUG 1713591668, 06:41:08 DEBUG "4972728320" 06:41:08 DEBUG ] 06:41:08 DEBUG } 06:41:08 DEBUG ] 06:41:08 DEBUG } 06:41:08 DEBUG } 06:41:08 INFO [http_cmd]: http status code OK 06:41:08 DEBUG --- status code --- 06:41:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:41:08 DEBUG --- http response --- 06:41:08 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 06:41:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713591668" 06:41:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591668" 06:41:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591668" 06:41:08 INFO [http_cmd]: http status code OK 06:41:08 INFO [http_cmd]: http status code OK 06:41:08 INFO [http_cmd]: http status code OK 06:41:08 DEBUG --- status code --- 06:41:08 DEBUG --- status code --- 06:41:08 DEBUG --- status code --- 06:41:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:41:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:41:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:41:08 DEBUG --- http response --- 06:41:08 DEBUG --- http response --- 06:41:08 DEBUG --- http response --- 06:41:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:41:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:41:08 DEBUG ----- output ----- 06:41:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:41:08 DEBUG ----- output ----- 06:41:08 DEBUG { 06:41:08 DEBUG ----- output ----- 06:41:08 DEBUG { 06:41:08 DEBUG "status": "success", 06:41:08 DEBUG { 06:41:08 DEBUG "status": "success", 06:41:08 DEBUG "data": { 06:41:08 DEBUG "status": "success", 06:41:08 DEBUG "data": { 06:41:08 DEBUG "resultType": "vector", 06:41:08 DEBUG "data": { 06:41:08 DEBUG "resultType": "vector", 06:41:08 DEBUG "result": [ 06:41:08 DEBUG "resultType": "vector", 06:41:08 DEBUG "result": [ 06:41:08 DEBUG { 06:41:08 DEBUG "result": [ 06:41:08 DEBUG { 06:41:08 DEBUG "metric": { 06:41:08 DEBUG { 06:41:08 DEBUG "metric": { 06:41:08 DEBUG "instance": "10.142.0.107:9100" 06:41:08 DEBUG "metric": { 06:41:08 DEBUG "pod": "ds-idrepo-0" 06:41:08 DEBUG }, 06:41:08 DEBUG "pod": "ds-cts-0" 06:41:08 DEBUG }, 06:41:08 DEBUG "value": [ 06:41:08 DEBUG }, 06:41:08 DEBUG "value": [ 06:41:08 DEBUG 06:41:08 DEBUG "value": [ 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG [...] 06:41:08 DEBUG 06:41:08 DEBUG [...] 06:41:08 DEBUG 06:41:08 DEBUG [...] 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG "instance": "10.142.0.43:9100" 06:41:08 DEBUG 06:41:08 DEBUG "pod": "ds-idrepo-2" 06:41:08 DEBUG }, 06:41:08 DEBUG "pod": "ds-cts-2" 06:41:08 DEBUG }, 06:41:08 DEBUG "value": [ 06:41:08 DEBUG }, 06:41:08 DEBUG "value": [ 06:41:08 DEBUG 1713591668, 06:41:08 DEBUG "value": [ 06:41:08 DEBUG 1713591668, 06:41:08 DEBUG "3183189371904" 06:41:08 DEBUG 1713591668, 06:41:08 DEBUG "0" 06:41:08 DEBUG ] 06:41:08 DEBUG "0" 06:41:08 DEBUG ] 06:41:08 DEBUG } 06:41:08 DEBUG ] 06:41:08 DEBUG } 06:41:08 DEBUG ] 06:41:08 DEBUG } 06:41:08 DEBUG ] 06:41:08 DEBUG } 06:41:08 DEBUG ] 06:41:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591668" 06:41:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591668" 06:41:08 DEBUG } 06:41:08 DEBUG } 06:41:08 DEBUG } 06:41:08 DEBUG } 06:41:08 DEBUG } 06:41:08 INFO [http_cmd]: http status code OK 06:41:08 INFO [http_cmd]: http status code OK 06:41:08 DEBUG --- status code --- 06:41:08 DEBUG --- status code --- 06:41:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:41:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:41:08 DEBUG --- http response --- 06:41:08 DEBUG --- http response --- 06:41:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:41:08 DEBUG ----- output ----- 06:41:08 DEBUG { 06:41:08 DEBUG "status": "success", 06:41:08 DEBUG "data": { 06:41:08 DEBUG "resultType": "vector", 06:41:08 DEBUG "result": [ 06:41:08 DEBUG { 06:41:08 DEBUG "metric": { 06:41:08 DEBUG "pod": "ds-cts-0" 06:41:08 DEBUG }, 06:41:08 DEBUG "value": [ 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG [...] 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG "pod": "ds-cts-2" 06:41:08 DEBUG }, 06:41:08 DEBUG "value": [ 06:41:08 DEBUG 1713591668, 06:41:08 DEBUG "0" 06:41:08 DEBUG ] 06:41:08 DEBUG } 06:41:08 DEBUG ] 06:41:08 DEBUG } 06:41:08 DEBUG } 06:41:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:41:08 DEBUG ----- output ----- 06:41:08 DEBUG { 06:41:08 DEBUG "status": "success", 06:41:08 DEBUG "data": { 06:41:08 DEBUG "resultType": "vector", 06:41:08 DEBUG "result": [ 06:41:08 DEBUG { 06:41:08 DEBUG "metric": { 06:41:08 DEBUG "pod": "ds-idrepo-0" 06:41:08 DEBUG }, 06:41:08 DEBUG "value": [ 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG [...] 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG "pod": "ds-idrepo-2" 06:41:08 DEBUG }, 06:41:08 DEBUG "value": [ 06:41:08 DEBUG 1713591668, 06:41:08 DEBUG "0" 06:41:08 DEBUG ] 06:41:08 DEBUG } 06:41:08 DEBUG ] 06:41:08 DEBUG } 06:41:08 DEBUG } 06:41:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591668" 06:41:08 INFO [http_cmd]: http status code OK 06:41:08 DEBUG --- status code --- 06:41:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:41:08 DEBUG --- http response --- 06:41:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:41:08 DEBUG ----- output ----- 06:41:08 DEBUG { 06:41:08 DEBUG "status": "success", 06:41:08 DEBUG "data": { 06:41:08 DEBUG "resultType": "vector", 06:41:08 DEBUG "result": [ 06:41:08 DEBUG { 06:41:08 DEBUG "metric": { 06:41:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:41:08 DEBUG }, 06:41:08 DEBUG "value": [ 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG [...] 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:41:08 DEBUG }, 06:41:08 DEBUG "value": [ 06:41:08 DEBUG 1713591668, 06:41:08 DEBUG "2675.8333333333335" 06:41:08 DEBUG ] 06:41:08 DEBUG } 06:41:08 DEBUG ] 06:41:08 DEBUG } 06:41:08 DEBUG } 06:41:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713591668" 06:41:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713591668" 06:41:08 INFO [http_cmd]: http status code OK 06:41:08 DEBUG --- status code --- 06:41:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:41:08 DEBUG --- http response --- 06:41:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:41:08 DEBUG ----- output ----- 06:41:08 DEBUG { 06:41:08 DEBUG "status": "success", 06:41:08 DEBUG "data": { 06:41:08 DEBUG "resultType": "vector", 06:41:08 DEBUG "result": [ 06:41:08 DEBUG { 06:41:08 DEBUG "metric": { 06:41:08 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 06:41:08 DEBUG }, 06:41:08 DEBUG "value": [ 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG [...] 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 06:41:08 DEBUG }, 06:41:08 DEBUG "value": [ 06:41:08 DEBUG 1713591668, 06:41:08 DEBUG "8.033671949326544" 06:41:08 DEBUG ] 06:41:08 DEBUG } 06:41:08 DEBUG ] 06:41:08 DEBUG } 06:41:08 DEBUG } 06:41:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591668" 06:41:08 INFO [http_cmd]: http status code OK 06:41:08 DEBUG --- status code --- 06:41:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:41:08 DEBUG --- http response --- 06:41:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:41:08 DEBUG ----- output ----- 06:41:08 DEBUG { 06:41:08 DEBUG "status": "success", 06:41:08 DEBUG "data": { 06:41:08 DEBUG "resultType": "vector", 06:41:08 DEBUG "result": [ 06:41:08 DEBUG { 06:41:08 DEBUG "metric": { 06:41:08 DEBUG "pod": "ds-idrepo-0" 06:41:08 DEBUG }, 06:41:08 DEBUG "value": [ 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG [...] 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG "pod": "ds-cts-2" 06:41:08 DEBUG }, 06:41:08 DEBUG "value": [ 06:41:08 DEBUG 1713591668, 06:41:08 DEBUG "0" 06:41:08 DEBUG ] 06:41:08 DEBUG } 06:41:08 DEBUG ] 06:41:08 DEBUG } 06:41:08 DEBUG } 06:41:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713591668" 06:41:08 INFO [http_cmd]: http status code OK 06:41:08 DEBUG --- status code --- 06:41:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:41:08 DEBUG --- http response --- 06:41:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:41:08 DEBUG ----- output ----- 06:41:08 DEBUG { 06:41:08 DEBUG "status": "success", 06:41:08 DEBUG "data": { 06:41:08 DEBUG "resultType": "vector", 06:41:08 DEBUG "result": [ 06:41:08 DEBUG { 06:41:08 DEBUG "metric": { 06:41:08 DEBUG "pod": "ds-idrepo-0" 06:41:08 DEBUG }, 06:41:08 DEBUG "value": [ 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG [...] 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG "pod": "ds-idrepo-2" 06:41:08 DEBUG }, 06:41:08 DEBUG "value": [ 06:41:08 DEBUG 1713591668, 06:41:08 DEBUG "0" 06:41:08 DEBUG ] 06:41:08 DEBUG } 06:41:08 DEBUG ] 06:41:08 DEBUG } 06:41:08 DEBUG } 06:41:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713591668" 06:41:08 INFO [http_cmd]: http status code OK 06:41:08 DEBUG --- status code --- 06:41:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:41:08 DEBUG --- http response --- 06:41:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:41:08 DEBUG ----- output ----- 06:41:08 DEBUG { 06:41:08 DEBUG "status": "success", 06:41:08 DEBUG "data": { 06:41:08 DEBUG "resultType": "vector", 06:41:08 DEBUG "result": [ 06:41:08 DEBUG { 06:41:08 DEBUG "metric": { 06:41:08 DEBUG "instance": "10.142.0.107:9100" 06:41:08 DEBUG }, 06:41:08 DEBUG "value": [ 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG [...] 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG "instance": "10.142.0.43:9100" 06:41:08 DEBUG }, 06:41:08 DEBUG "value": [ 06:41:08 DEBUG 1713591668, 06:41:08 DEBUG "81394870.23333332" 06:41:08 DEBUG ] 06:41:08 DEBUG } 06:41:08 DEBUG ] 06:41:08 DEBUG } 06:41:08 DEBUG } 06:41:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713591668" 06:41:08 INFO [http_cmd]: http status code OK 06:41:08 DEBUG --- status code --- 06:41:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:41:08 DEBUG --- http response --- 06:41:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:41:08 DEBUG ----- output ----- 06:41:08 DEBUG { 06:41:08 DEBUG "status": "success", 06:41:08 DEBUG "data": { 06:41:08 DEBUG "resultType": "vector", 06:41:08 DEBUG "result": [ 06:41:08 DEBUG { 06:41:08 DEBUG "metric": { 06:41:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:41:08 DEBUG }, 06:41:08 DEBUG "value": [ 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG [...] 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:41:08 DEBUG }, 06:41:08 DEBUG "value": [ 06:41:08 DEBUG 1713591668, 06:41:08 DEBUG "0.095" 06:41:08 DEBUG ] 06:41:08 DEBUG } 06:41:08 DEBUG ] 06:41:08 DEBUG } 06:41:08 DEBUG } 06:41:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591668" 06:41:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713591668" 06:41:08 INFO [http_cmd]: http status code OK 06:41:08 DEBUG --- status code --- 06:41:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:41:08 DEBUG --- http response --- 06:41:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:41:08 DEBUG ----- output ----- 06:41:08 DEBUG { 06:41:08 DEBUG "status": "success", 06:41:08 DEBUG "data": { 06:41:08 DEBUG "resultType": "vector", 06:41:08 DEBUG "result": [ 06:41:08 DEBUG { 06:41:08 DEBUG "metric": { 06:41:08 DEBUG "instance": "10.142.0.107:9100" 06:41:08 DEBUG }, 06:41:08 DEBUG "value": [ 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG [...] 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG "instance": "10.142.0.43:9100" 06:41:08 DEBUG }, 06:41:08 DEBUG "value": [ 06:41:08 DEBUG 1713591668, 06:41:08 DEBUG "0.20423333333336682" 06:41:08 DEBUG ] 06:41:08 DEBUG } 06:41:08 DEBUG ] 06:41:08 DEBUG } 06:41:08 DEBUG } 06:41:08 INFO [http_cmd]: http status code OK 06:41:08 DEBUG --- status code --- 06:41:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:41:08 DEBUG --- http response --- 06:41:08 INFO [http_cmd]: http status code OK 06:41:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:41:08 DEBUG --- status code --- 06:41:08 DEBUG ----- output ----- 06:41:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:41:08 DEBUG { 06:41:08 DEBUG --- http response --- 06:41:08 DEBUG "status": "success", 06:41:08 DEBUG "data": { 06:41:08 DEBUG "resultType": "vector", 06:41:08 DEBUG "result": [ 06:41:08 DEBUG { 06:41:08 DEBUG "metric": { 06:41:08 DEBUG "instance": "10.142.0.107:9100" 06:41:08 DEBUG }, 06:41:08 DEBUG "value": [ 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG [...] 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG "instance": "10.142.0.43:9100" 06:41:08 DEBUG }, 06:41:08 DEBUG "value": [ 06:41:08 DEBUG 1713591668, 06:41:08 DEBUG "81400695.76666667" 06:41:08 DEBUG ] 06:41:08 DEBUG } 06:41:08 DEBUG ] 06:41:08 DEBUG } 06:41:08 DEBUG } 06:41:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:41:08 DEBUG ----- output ----- 06:41:08 DEBUG { 06:41:08 DEBUG "status": "success", 06:41:08 DEBUG "data": { 06:41:08 DEBUG "resultType": "vector", 06:41:08 DEBUG "result": [ 06:41:08 DEBUG { 06:41:08 DEBUG "metric": { 06:41:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:41:08 DEBUG }, 06:41:08 DEBUG "value": [ 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG [...] 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:41:08 DEBUG }, 06:41:08 DEBUG "value": [ 06:41:08 DEBUG 1713591668, 06:41:08 DEBUG "2695.133333333333" 06:41:08 DEBUG ] 06:41:08 DEBUG } 06:41:08 DEBUG ] 06:41:08 DEBUG } 06:41:08 DEBUG } 06:41:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591668" 06:41:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591668" 06:41:08 INFO [http_cmd]: http status code OK 06:41:08 DEBUG --- status code --- 06:41:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:41:08 DEBUG --- http response --- 06:41:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:41:08 DEBUG ----- output ----- 06:41:08 DEBUG { 06:41:08 DEBUG "status": "success", 06:41:08 DEBUG "data": { 06:41:08 DEBUG "resultType": "vector", 06:41:08 DEBUG "result": [ 06:41:08 DEBUG { 06:41:08 DEBUG "metric": { 06:41:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:41:08 DEBUG }, 06:41:08 DEBUG "value": [ 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG [...] 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:41:08 DEBUG }, 06:41:08 DEBUG "value": [ 06:41:08 DEBUG 1713591668, 06:41:08 DEBUG "0" 06:41:08 DEBUG ] 06:41:08 DEBUG } 06:41:08 DEBUG ] 06:41:08 DEBUG } 06:41:08 DEBUG } 06:41:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591668" 06:41:08 INFO [http_cmd]: http status code OK 06:41:08 DEBUG --- status code --- 06:41:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:41:08 DEBUG --- http response --- 06:41:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:41:08 DEBUG ----- output ----- 06:41:08 DEBUG { 06:41:08 DEBUG "status": "success", 06:41:08 DEBUG "data": { 06:41:08 DEBUG "resultType": "vector", 06:41:08 DEBUG "result": [ 06:41:08 DEBUG { 06:41:08 DEBUG "metric": { 06:41:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:41:08 DEBUG }, 06:41:08 DEBUG "value": [ 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG [...] 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:41:08 DEBUG }, 06:41:08 DEBUG "value": [ 06:41:08 DEBUG 1713591668, 06:41:08 DEBUG "2675.7" 06:41:08 DEBUG ] 06:41:08 DEBUG } 06:41:08 DEBUG ] 06:41:08 DEBUG } 06:41:08 DEBUG } 06:41:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591668" 06:41:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591668" 06:41:08 INFO [http_cmd]: http status code OK 06:41:08 DEBUG --- status code --- 06:41:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:41:08 DEBUG --- http response --- 06:41:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:41:08 DEBUG ----- output ----- 06:41:08 DEBUG { 06:41:08 DEBUG "status": "success", 06:41:08 DEBUG "data": { 06:41:08 DEBUG "resultType": "vector", 06:41:08 DEBUG "result": [ 06:41:08 DEBUG { 06:41:08 DEBUG "metric": { 06:41:08 DEBUG "pod": "ds-cts-0" 06:41:08 DEBUG }, 06:41:08 DEBUG "value": [ 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG [...] 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG "pod": "ds-cts-2" 06:41:08 DEBUG }, 06:41:08 DEBUG "value": [ 06:41:08 DEBUG 1713591668, 06:41:08 DEBUG "0.00016666666666666682" 06:41:08 DEBUG ] 06:41:08 DEBUG } 06:41:08 DEBUG ] 06:41:08 DEBUG } 06:41:08 DEBUG } 06:41:08 INFO [http_cmd]: http status code OK 06:41:08 DEBUG --- status code --- 06:41:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:41:08 DEBUG --- http response --- 06:41:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:41:08 DEBUG ----- output ----- 06:41:08 DEBUG { 06:41:08 DEBUG "status": "success", 06:41:08 DEBUG "data": { 06:41:08 DEBUG "resultType": "vector", 06:41:08 DEBUG "result": [ 06:41:08 DEBUG { 06:41:08 DEBUG "metric": { 06:41:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:41:08 DEBUG }, 06:41:08 DEBUG "value": [ 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG [...] 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:41:08 DEBUG }, 06:41:08 DEBUG "value": [ 06:41:08 DEBUG 1713591668, 06:41:08 DEBUG "2675.6666666666665" 06:41:08 DEBUG ] 06:41:08 DEBUG } 06:41:08 DEBUG ] 06:41:08 DEBUG } 06:41:08 DEBUG } 06:41:08 INFO [http_cmd]: http status code OK 06:41:08 DEBUG --- status code --- 06:41:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:41:08 DEBUG --- http response --- 06:41:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:41:08 DEBUG ----- output ----- 06:41:08 DEBUG { 06:41:08 DEBUG "status": "success", 06:41:08 DEBUG "data": { 06:41:08 DEBUG "resultType": "vector", 06:41:08 DEBUG "result": [ 06:41:08 DEBUG { 06:41:08 DEBUG "metric": { 06:41:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:41:08 DEBUG }, 06:41:08 DEBUG "value": [ 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG [...] 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG "pod": "am-767f6b67b5-qcqz2" 06:41:08 DEBUG }, 06:41:08 DEBUG "value": [ 06:41:08 DEBUG 1713591668, 06:41:08 DEBUG "9.239981424574056" 06:41:08 DEBUG ] 06:41:08 DEBUG } 06:41:08 DEBUG ] 06:41:08 DEBUG } 06:41:08 DEBUG } 06:41:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713591668" 06:41:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591668" 06:41:08 INFO [http_cmd]: http status code OK 06:41:08 INFO [http_cmd]: http status code OK 06:41:08 DEBUG --- status code --- 06:41:08 DEBUG --- status code --- 06:41:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:41:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:41:08 DEBUG --- http response --- 06:41:08 DEBUG --- http response --- 06:41:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:41:08 DEBUG ----- output ----- 06:41:08 DEBUG { 06:41:08 DEBUG "status": "success", 06:41:08 DEBUG "data": { 06:41:08 DEBUG "resultType": "vector", 06:41:08 DEBUG "result": [ 06:41:08 DEBUG { 06:41:08 DEBUG "metric": { 06:41:08 DEBUG "instance": "10.142.0.107:9100" 06:41:08 DEBUG }, 06:41:08 DEBUG "value": [ 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG [...] 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG "instance": "10.142.0.43:9100" 06:41:08 DEBUG }, 06:41:08 DEBUG "value": [ 06:41:08 DEBUG 1713591668, 06:41:08 DEBUG "0.07533333333332733" 06:41:08 DEBUG ] 06:41:08 DEBUG } 06:41:08 DEBUG ] 06:41:08 DEBUG } 06:41:08 DEBUG } 06:41:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:41:08 DEBUG ----- output ----- 06:41:08 DEBUG { 06:41:08 DEBUG "status": "success", 06:41:08 DEBUG "data": { 06:41:08 DEBUG "resultType": "vector", 06:41:08 DEBUG "result": [ 06:41:08 DEBUG { 06:41:08 DEBUG "metric": { 06:41:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:41:08 DEBUG }, 06:41:08 DEBUG "value": [ 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG [...] 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG 06:41:08 DEBUG "pod": "am-767f6b67b5-qcqz2" 06:41:08 DEBUG }, 06:41:08 DEBUG "value": [ 06:41:08 DEBUG 1713591668, 06:41:08 DEBUG "254847635.74250686" 06:41:08 DEBUG ] 06:41:08 DEBUG } 06:41:08 DEBUG ] 06:41:08 DEBUG } 06:41:08 DEBUG } 06:41:08 INFO 06:41:08 INFO [loop_until]: kubectl --namespace=xlou top node 06:41:08 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:41:08 INFO 06:41:08 INFO [loop_until]: kubectl --namespace=xlou top pods 06:41:08 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:41:08 INFO [loop_until]: OK (rc = 0) 06:41:08 DEBUG --- stdout --- 06:41:08 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 17253m 8646Mi am-767f6b67b5-qcqz2 17694m 8709Mi am-767f6b67b5-vbltz 18998m 8650Mi ds-cts-0 18813m 10394Mi ds-cts-1 17481m 10662Mi ds-cts-2 19316m 10049Mi ds-idrepo-0 7587m 3740Mi ds-idrepo-1 7712m 3769Mi ds-idrepo-2 7649m 3695Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 3m 1177Mi idm-b85f9d978-wnhxb 3m 1236Mi lodemon-7cb5b9456d-dskfn 2m 62Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 8000m 4651Mi 06:41:08 DEBUG --- stderr --- 06:41:08 DEBUG 06:41:08 INFO [loop_until]: OK (rc = 0) 06:41:08 DEBUG --- stdout --- 06:41:08 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 72m 0% 6273Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 8011m 26% 7406Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 17133m 57% 11357Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 80m 0% 3378Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 18623m 62% 13526Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 161m 0% 5176Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 17364m 58% 11590Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 19012m 63% 3006Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 19533m 65% 10859Mi 9% gke-xlou-cdm-ds-0f601751-j488 8204m 27% 7021Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 17380m 58% 15583Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 8110m 27% 7673Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 19097m 63% 12637Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 7899m 49% 8499Mi 14% 06:41:08 DEBUG --- stderr --- 06:41:08 DEBUG 06:42:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713591728" 06:42:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591728" 06:42:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713591728" 06:42:08 ERROR [http_cmd]: ERROR 06:42:08 INFO [http_cmd]: http status code OK 06:42:08 ERROR --- status code --- 06:42:08 DEBUG --- status code --- 06:42:08 ERROR http status code is 400 (expected [200, 401, 403, 404]) 06:42:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:42:08 ERROR --- http response --- 06:42:08 DEBUG --- http response --- 06:42:08 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 06:42:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:42:08 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 06:42:08 DEBUG ----- output ----- 06:42:08 DEBUG { 06:42:08 DEBUG "status": "success", 06:42:08 DEBUG "data": { 06:42:08 DEBUG "resultType": "vector", 06:42:08 DEBUG "result": [ 06:42:08 DEBUG { 06:42:08 DEBUG "metric": { 06:42:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:42:08 DEBUG }, 06:42:08 DEBUG "value": [ 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG [...] 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:42:08 DEBUG }, 06:42:08 DEBUG "value": [ 06:42:08 DEBUG 1713591728, 06:42:08 DEBUG "480899.5" 06:42:08 DEBUG ] 06:42:08 DEBUG } 06:42:08 DEBUG ] 06:42:08 DEBUG } 06:42:08 DEBUG } 06:42:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591728" 06:42:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591728" 06:42:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591728" 06:42:08 INFO [http_cmd]: http status code OK 06:42:08 DEBUG --- status code --- 06:42:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:42:08 DEBUG --- http response --- 06:42:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:42:08 DEBUG ----- output ----- 06:42:08 DEBUG { 06:42:08 DEBUG "status": "success", 06:42:08 DEBUG "data": { 06:42:08 DEBUG "resultType": "vector", 06:42:08 DEBUG "result": [ 06:42:08 DEBUG { 06:42:08 DEBUG "metric": { 06:42:08 DEBUG "instance": "10.142.0.107:9100" 06:42:08 DEBUG }, 06:42:08 DEBUG "value": [ 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG [...] 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG "instance": "10.142.0.43:9100" 06:42:08 DEBUG }, 06:42:08 DEBUG "value": [ 06:42:08 DEBUG 1713591728, 06:42:08 DEBUG "3187512302592" 06:42:08 DEBUG ] 06:42:08 DEBUG } 06:42:08 DEBUG ] 06:42:08 DEBUG } 06:42:08 DEBUG } 06:42:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713591728" 06:42:08 INFO [http_cmd]: http status code OK 06:42:08 DEBUG --- status code --- 06:42:08 INFO [http_cmd]: http status code OK 06:42:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:42:08 DEBUG --- status code --- 06:42:08 DEBUG --- http response --- 06:42:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:42:08 DEBUG --- http response --- 06:42:08 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 06:42:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (98 lines): 06:42:08 DEBUG ----- output ----- 06:42:08 DEBUG { 06:42:08 DEBUG "status": "success", 06:42:08 DEBUG "data": { 06:42:08 DEBUG "resultType": "vector", 06:42:08 DEBUG "result": [ 06:42:08 DEBUG { 06:42:08 DEBUG "metric": { 06:42:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:42:08 DEBUG }, 06:42:08 DEBUG "value": [ 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG [...] 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 06:42:08 DEBUG }, 06:42:08 DEBUG "value": [ 06:42:08 DEBUG 1713591728, 06:42:08 DEBUG "0" 06:42:08 DEBUG ] 06:42:08 DEBUG } 06:42:08 DEBUG ] 06:42:08 DEBUG } 06:42:08 DEBUG } 06:42:08 INFO [http_cmd]: http status code OK 06:42:08 DEBUG --- status code --- 06:42:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:42:08 DEBUG --- http response --- 06:42:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (98 lines): 06:42:08 DEBUG ----- output ----- 06:42:08 DEBUG { 06:42:08 DEBUG "status": "success", 06:42:08 DEBUG "data": { 06:42:08 DEBUG "resultType": "vector", 06:42:08 DEBUG "result": [ 06:42:08 DEBUG { 06:42:08 DEBUG "metric": { 06:42:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:42:08 DEBUG }, 06:42:08 DEBUG "value": [ 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG [...] 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 06:42:08 DEBUG }, 06:42:08 DEBUG "value": [ 06:42:08 DEBUG 1713591728, 06:42:08 DEBUG "0" 06:42:08 DEBUG ] 06:42:08 DEBUG } 06:42:08 DEBUG ] 06:42:08 DEBUG } 06:42:08 DEBUG } 06:42:08 INFO [http_cmd]: http status code OK 06:42:08 DEBUG --- status code --- 06:42:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:42:08 DEBUG --- http response --- 06:42:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:42:08 DEBUG ----- output ----- 06:42:08 DEBUG { 06:42:08 DEBUG "status": "success", 06:42:08 DEBUG "data": { 06:42:08 DEBUG "resultType": "vector", 06:42:08 DEBUG "result": [ 06:42:08 DEBUG { 06:42:08 DEBUG "metric": { 06:42:08 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 06:42:08 DEBUG }, 06:42:08 DEBUG "value": [ 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG [...] 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 06:42:08 DEBUG }, 06:42:08 DEBUG "value": [ 06:42:08 DEBUG 1713591728, 06:42:08 DEBUG "4976594944" 06:42:08 DEBUG ] 06:42:08 DEBUG } 06:42:08 DEBUG ] 06:42:08 DEBUG } 06:42:08 DEBUG } 06:42:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591728" 06:42:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591728" 06:42:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591728" 06:42:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591728" 06:42:08 INFO [http_cmd]: http status code OK 06:42:08 INFO [http_cmd]: http status code OK 06:42:08 DEBUG --- status code --- 06:42:08 DEBUG --- status code --- 06:42:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:42:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:42:08 DEBUG --- http response --- 06:42:08 DEBUG --- http response --- 06:42:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:42:08 DEBUG ----- output ----- 06:42:08 DEBUG { 06:42:08 DEBUG "status": "success", 06:42:08 DEBUG "data": { 06:42:08 DEBUG "resultType": "vector", 06:42:08 DEBUG "result": [ 06:42:08 DEBUG { 06:42:08 DEBUG "metric": { 06:42:08 DEBUG "pod": "ds-idrepo-0" 06:42:08 DEBUG }, 06:42:08 DEBUG "value": [ 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG [...] 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG "pod": "ds-idrepo-2" 06:42:08 DEBUG }, 06:42:08 DEBUG "value": [ 06:42:08 DEBUG 1713591728, 06:42:08 DEBUG "0" 06:42:08 DEBUG ] 06:42:08 DEBUG } 06:42:08 DEBUG ] 06:42:08 DEBUG } 06:42:08 DEBUG } 06:42:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:42:08 DEBUG ----- output ----- 06:42:08 DEBUG { 06:42:08 DEBUG "status": "success", 06:42:08 DEBUG "data": { 06:42:08 DEBUG "resultType": "vector", 06:42:08 DEBUG "result": [ 06:42:08 DEBUG { 06:42:08 DEBUG "metric": { 06:42:08 DEBUG "pod": "ds-cts-0" 06:42:08 DEBUG }, 06:42:08 DEBUG "value": [ 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG [...] 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG "pod": "ds-cts-2" 06:42:08 DEBUG }, 06:42:08 DEBUG "value": [ 06:42:08 DEBUG 1713591728, 06:42:08 DEBUG "0" 06:42:08 DEBUG ] 06:42:08 DEBUG } 06:42:08 DEBUG ] 06:42:08 DEBUG } 06:42:08 DEBUG } 06:42:08 INFO [http_cmd]: http status code OK 06:42:08 INFO [http_cmd]: http status code OK 06:42:08 DEBUG --- status code --- 06:42:08 DEBUG --- status code --- 06:42:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:42:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:42:08 DEBUG --- http response --- 06:42:08 DEBUG --- http response --- 06:42:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:42:08 DEBUG ----- output ----- 06:42:08 DEBUG { 06:42:08 DEBUG "status": "success", 06:42:08 DEBUG "data": { 06:42:08 DEBUG "resultType": "vector", 06:42:08 DEBUG "result": [ 06:42:08 DEBUG { 06:42:08 DEBUG "metric": { 06:42:08 DEBUG "pod": "ds-idrepo-0" 06:42:08 DEBUG }, 06:42:08 DEBUG "value": [ 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG [...] 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG "pod": "ds-idrepo-2" 06:42:08 DEBUG }, 06:42:08 DEBUG "value": [ 06:42:08 DEBUG 1713591728, 06:42:08 DEBUG "0" 06:42:08 DEBUG ] 06:42:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:42:08 DEBUG } 06:42:08 DEBUG ----- output ----- 06:42:08 DEBUG ] 06:42:08 DEBUG { 06:42:08 DEBUG } 06:42:08 DEBUG "status": "success", 06:42:08 DEBUG } 06:42:08 DEBUG "data": { 06:42:08 DEBUG "resultType": "vector", 06:42:08 DEBUG "result": [ 06:42:08 DEBUG { 06:42:08 DEBUG "metric": { 06:42:08 DEBUG "pod": "ds-cts-0" 06:42:08 DEBUG }, 06:42:08 DEBUG "value": [ 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG [...] 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG "pod": "ds-cts-2" 06:42:08 DEBUG }, 06:42:08 DEBUG "value": [ 06:42:08 DEBUG 1713591728, 06:42:08 DEBUG "0" 06:42:08 DEBUG ] 06:42:08 DEBUG } 06:42:08 DEBUG ] 06:42:08 DEBUG } 06:42:08 DEBUG } 06:42:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591728" 06:42:08 INFO [http_cmd]: http status code OK 06:42:08 DEBUG --- status code --- 06:42:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:42:08 DEBUG --- http response --- 06:42:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:42:08 DEBUG ----- output ----- 06:42:08 DEBUG { 06:42:08 DEBUG "status": "success", 06:42:08 DEBUG "data": { 06:42:08 DEBUG "resultType": "vector", 06:42:08 DEBUG "result": [ 06:42:08 DEBUG { 06:42:08 DEBUG "metric": { 06:42:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:42:08 DEBUG }, 06:42:08 DEBUG "value": [ 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG [...] 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:42:08 DEBUG }, 06:42:08 DEBUG "value": [ 06:42:08 DEBUG 1713591728, 06:42:08 DEBUG "2669.6666666666665" 06:42:08 DEBUG ] 06:42:08 DEBUG } 06:42:08 DEBUG ] 06:42:08 DEBUG } 06:42:08 DEBUG } 06:42:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713591728" 06:42:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713591728" 06:42:08 INFO [http_cmd]: http status code OK 06:42:08 DEBUG --- status code --- 06:42:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:42:08 DEBUG --- http response --- 06:42:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713591728" 06:42:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:42:08 DEBUG ----- output ----- 06:42:08 DEBUG { 06:42:08 DEBUG "status": "success", 06:42:08 DEBUG "data": { 06:42:08 DEBUG "resultType": "vector", 06:42:08 DEBUG "result": [ 06:42:08 DEBUG { 06:42:08 DEBUG "metric": { 06:42:08 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 06:42:08 DEBUG }, 06:42:08 DEBUG "value": [ 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG [...] 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 06:42:08 DEBUG }, 06:42:08 DEBUG "value": [ 06:42:08 DEBUG 1713591728, 06:42:08 DEBUG "8.03841299310802" 06:42:08 DEBUG ] 06:42:08 DEBUG } 06:42:08 DEBUG ] 06:42:08 DEBUG } 06:42:08 DEBUG } 06:42:08 INFO [http_cmd]: http status code OK 06:42:08 DEBUG --- status code --- 06:42:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:42:08 DEBUG --- http response --- 06:42:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:42:08 DEBUG ----- output ----- 06:42:08 DEBUG { 06:42:08 DEBUG "status": "success", 06:42:08 DEBUG "data": { 06:42:08 DEBUG "resultType": "vector", 06:42:08 DEBUG "result": [ 06:42:08 DEBUG { 06:42:08 DEBUG "metric": { 06:42:08 DEBUG "instance": "10.142.0.107:9100" 06:42:08 DEBUG }, 06:42:08 DEBUG "value": [ 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG [...] 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG "instance": "10.142.0.43:9100" 06:42:08 DEBUG }, 06:42:08 DEBUG "value": [ 06:42:08 DEBUG 1713591728, 06:42:08 DEBUG "0.18176666666665633" 06:42:08 DEBUG ] 06:42:08 DEBUG } 06:42:08 DEBUG ] 06:42:08 DEBUG } 06:42:08 DEBUG } 06:42:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591728" 06:42:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713591728" 06:42:08 INFO [http_cmd]: http status code OK 06:42:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591728" 06:42:08 DEBUG --- status code --- 06:42:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:42:08 DEBUG --- http response --- 06:42:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:42:08 DEBUG ----- output ----- 06:42:08 DEBUG { 06:42:08 DEBUG "status": "success", 06:42:08 DEBUG "data": { 06:42:08 DEBUG "resultType": "vector", 06:42:08 DEBUG "result": [ 06:42:08 DEBUG { 06:42:08 DEBUG "metric": { 06:42:08 DEBUG "pod": "ds-idrepo-0" 06:42:08 DEBUG }, 06:42:08 DEBUG "value": [ 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG [...] 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG "pod": "ds-cts-2" 06:42:08 DEBUG }, 06:42:08 DEBUG "value": [ 06:42:08 DEBUG 1713591728, 06:42:08 DEBUG "0" 06:42:08 DEBUG ] 06:42:08 DEBUG } 06:42:08 DEBUG ] 06:42:08 DEBUG } 06:42:08 DEBUG } 06:42:08 INFO [http_cmd]: http status code OK 06:42:08 DEBUG --- status code --- 06:42:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:42:08 DEBUG --- http response --- 06:42:08 INFO [http_cmd]: http status code OK 06:42:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:42:08 DEBUG --- status code --- 06:42:08 DEBUG ----- output ----- 06:42:08 INFO [http_cmd]: http status code OK 06:42:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:42:08 DEBUG { 06:42:08 DEBUG --- status code --- 06:42:08 DEBUG --- http response --- 06:42:08 DEBUG "status": "success", 06:42:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:42:08 DEBUG "data": { 06:42:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:42:08 DEBUG --- http response --- 06:42:08 DEBUG "resultType": "vector", 06:42:08 DEBUG ----- output ----- 06:42:08 DEBUG "result": [ 06:42:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:42:08 DEBUG { 06:42:08 DEBUG { 06:42:08 DEBUG ----- output ----- 06:42:08 DEBUG "status": "success", 06:42:08 DEBUG "metric": { 06:42:08 DEBUG { 06:42:08 DEBUG "data": { 06:42:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:42:08 DEBUG "status": "success", 06:42:08 DEBUG "resultType": "vector", 06:42:08 DEBUG }, 06:42:08 DEBUG "data": { 06:42:08 DEBUG "result": [ 06:42:08 DEBUG "value": [ 06:42:08 DEBUG "resultType": "vector", 06:42:08 DEBUG { 06:42:08 DEBUG 06:42:08 DEBUG "result": [ 06:42:08 DEBUG "metric": { 06:42:08 DEBUG 06:42:08 DEBUG { 06:42:08 DEBUG "pod": "ds-idrepo-0" 06:42:08 DEBUG 06:42:08 DEBUG "metric": { 06:42:08 DEBUG }, 06:42:08 DEBUG [...] 06:42:08 DEBUG "instance": "10.142.0.107:9100" 06:42:08 DEBUG "value": [ 06:42:08 DEBUG 06:42:08 DEBUG }, 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG "value": [ 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:42:08 DEBUG 06:42:08 DEBUG [...] 06:42:08 DEBUG }, 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG "value": [ 06:42:08 DEBUG [...] 06:42:08 DEBUG 06:42:08 DEBUG 1713591728, 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG "0" 06:42:08 DEBUG 06:42:08 DEBUG "pod": "ds-idrepo-2" 06:42:08 DEBUG ] 06:42:08 DEBUG 06:42:08 DEBUG }, 06:42:08 DEBUG } 06:42:08 DEBUG "instance": "10.142.0.43:9100" 06:42:08 DEBUG "value": [ 06:42:08 DEBUG ] 06:42:08 DEBUG }, 06:42:08 DEBUG 1713591728, 06:42:08 DEBUG } 06:42:08 DEBUG "value": [ 06:42:08 DEBUG "0" 06:42:08 DEBUG } 06:42:08 DEBUG 1713591728, 06:42:08 DEBUG ] 06:42:08 DEBUG "80944632.23333333" 06:42:08 DEBUG } 06:42:08 DEBUG ] 06:42:08 DEBUG ] 06:42:08 DEBUG } 06:42:08 DEBUG } 06:42:08 DEBUG ] 06:42:08 DEBUG } 06:42:08 DEBUG } 06:42:08 DEBUG } 06:42:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713591728" 06:42:08 INFO [http_cmd]: http status code OK 06:42:08 DEBUG --- status code --- 06:42:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:42:08 DEBUG --- http response --- 06:42:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:42:08 DEBUG ----- output ----- 06:42:08 DEBUG { 06:42:08 DEBUG "status": "success", 06:42:08 DEBUG "data": { 06:42:08 DEBUG "resultType": "vector", 06:42:08 DEBUG "result": [ 06:42:08 DEBUG { 06:42:08 DEBUG "metric": { 06:42:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:42:08 DEBUG }, 06:42:08 DEBUG "value": [ 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG [...] 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:42:08 DEBUG }, 06:42:08 DEBUG "value": [ 06:42:08 DEBUG 1713591728, 06:42:08 DEBUG "0.095" 06:42:08 DEBUG ] 06:42:08 DEBUG } 06:42:08 DEBUG ] 06:42:08 DEBUG } 06:42:08 DEBUG } 06:42:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713591728" 06:42:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591728" 06:42:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591728" 06:42:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591728" 06:42:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591728" 06:42:08 INFO [http_cmd]: http status code OK 06:42:08 INFO [http_cmd]: http status code OK 06:42:08 INFO [http_cmd]: http status code OK 06:42:08 DEBUG --- status code --- 06:42:08 INFO [http_cmd]: http status code OK 06:42:08 DEBUG --- status code --- 06:42:08 DEBUG --- status code --- 06:42:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:42:08 DEBUG --- status code --- 06:42:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:42:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:42:08 DEBUG --- http response --- 06:42:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:42:08 DEBUG --- http response --- 06:42:08 DEBUG --- http response --- 06:42:08 DEBUG --- http response --- 06:42:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:42:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:42:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:42:08 DEBUG ----- output ----- 06:42:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 06:42:08 DEBUG ----- output ----- 06:42:08 DEBUG ----- output ----- 06:42:08 DEBUG { 06:42:08 DEBUG ----- output ----- 06:42:08 DEBUG { 06:42:08 DEBUG { 06:42:08 DEBUG "status": "success", 06:42:08 DEBUG { 06:42:08 DEBUG "status": "success", 06:42:08 DEBUG "status": "success", 06:42:08 DEBUG "data": { 06:42:08 DEBUG "status": "success", 06:42:08 DEBUG "data": { 06:42:08 DEBUG "data": { 06:42:08 DEBUG "resultType": "vector", 06:42:08 DEBUG "data": { 06:42:08 DEBUG "resultType": "vector", 06:42:08 DEBUG "resultType": "vector", 06:42:08 DEBUG "result": [ 06:42:08 DEBUG "resultType": "vector", 06:42:08 DEBUG "result": [ 06:42:08 DEBUG "result": [ 06:42:08 DEBUG { 06:42:08 DEBUG "result": [ 06:42:08 DEBUG { 06:42:08 DEBUG { 06:42:08 DEBUG "metric": { 06:42:08 DEBUG { 06:42:08 DEBUG "metric": { 06:42:08 DEBUG "metric": { 06:42:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:42:08 DEBUG "metric": { 06:42:08 DEBUG "pod": "ds-cts-0" 06:42:08 DEBUG "instance": "10.142.0.107:9100" 06:42:08 DEBUG }, 06:42:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:42:08 DEBUG }, 06:42:08 DEBUG }, 06:42:08 DEBUG "value": [ 06:42:08 DEBUG }, 06:42:08 DEBUG "value": [ 06:42:08 DEBUG "value": [ 06:42:08 DEBUG 06:42:08 DEBUG "value": [ 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 INFO [http_cmd]: http status code OK 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG [...] 06:42:08 DEBUG --- status code --- 06:42:08 DEBUG [...] 06:42:08 DEBUG [...] 06:42:08 DEBUG [...] 06:42:08 DEBUG 06:42:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG --- http response --- 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:42:08 DEBUG "pod": "idm-b85f9d978-wnhxb" 06:42:08 DEBUG ----- output ----- 06:42:08 DEBUG "pod": "ds-cts-2" 06:42:08 DEBUG "instance": "10.142.0.43:9100" 06:42:08 DEBUG }, 06:42:08 DEBUG }, 06:42:08 DEBUG { 06:42:08 DEBUG }, 06:42:08 DEBUG }, 06:42:08 DEBUG "value": [ 06:42:08 DEBUG "value": [ 06:42:08 DEBUG "status": "success", 06:42:08 DEBUG "value": [ 06:42:08 DEBUG "value": [ 06:42:08 DEBUG 1713591728, 06:42:08 DEBUG 1713591728, 06:42:08 DEBUG "data": { 06:42:08 DEBUG 1713591728, 06:42:08 DEBUG 1713591728, 06:42:08 DEBUG "2694.1" 06:42:08 DEBUG "0" 06:42:08 DEBUG "resultType": "vector", 06:42:08 DEBUG "0.00030000000000000003" 06:42:08 DEBUG "80950258.66666667" 06:42:08 DEBUG ] 06:42:08 DEBUG ] 06:42:08 DEBUG "result": [ 06:42:08 DEBUG ] 06:42:08 DEBUG ] 06:42:08 DEBUG } 06:42:08 DEBUG } 06:42:08 DEBUG { 06:42:08 DEBUG } 06:42:08 DEBUG } 06:42:08 DEBUG ] 06:42:08 DEBUG ] 06:42:08 DEBUG "metric": { 06:42:08 DEBUG ] 06:42:08 DEBUG ] 06:42:08 DEBUG } 06:42:08 DEBUG } 06:42:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:42:08 DEBUG } 06:42:08 DEBUG } 06:42:08 DEBUG } 06:42:08 DEBUG } 06:42:08 DEBUG }, 06:42:08 DEBUG } 06:42:08 DEBUG } 06:42:08 DEBUG "value": [ 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG [...] 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG "pod": "idm-b85f9d978-wnhxb" 06:42:08 DEBUG }, 06:42:08 DEBUG "value": [ 06:42:08 DEBUG 1713591728, 06:42:08 DEBUG "0" 06:42:08 DEBUG ] 06:42:08 DEBUG } 06:42:08 DEBUG ] 06:42:08 DEBUG } 06:42:08 DEBUG } 06:42:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591728" 06:42:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591728" 06:42:08 INFO [http_cmd]: http status code OK 06:42:08 DEBUG --- status code --- 06:42:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:42:08 DEBUG --- http response --- 06:42:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:42:08 DEBUG ----- output ----- 06:42:08 DEBUG { 06:42:08 DEBUG "status": "success", 06:42:08 DEBUG "data": { 06:42:08 DEBUG "resultType": "vector", 06:42:08 DEBUG "result": [ 06:42:08 DEBUG { 06:42:08 DEBUG "metric": { 06:42:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:42:08 DEBUG }, 06:42:08 DEBUG "value": [ 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG [...] 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:42:08 DEBUG }, 06:42:08 DEBUG "value": [ 06:42:08 DEBUG 1713591728, 06:42:08 DEBUG "2668.9" 06:42:08 DEBUG ] 06:42:08 DEBUG } 06:42:08 DEBUG ] 06:42:08 DEBUG } 06:42:08 DEBUG } 06:42:08 INFO [http_cmd]: http status code OK 06:42:08 DEBUG --- status code --- 06:42:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:42:08 DEBUG --- http response --- 06:42:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:42:08 DEBUG ----- output ----- 06:42:08 DEBUG { 06:42:08 DEBUG "status": "success", 06:42:08 DEBUG "data": { 06:42:08 DEBUG "resultType": "vector", 06:42:08 DEBUG "result": [ 06:42:08 DEBUG { 06:42:08 DEBUG "metric": { 06:42:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:42:08 DEBUG }, 06:42:08 DEBUG "value": [ 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG [...] 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:42:08 DEBUG }, 06:42:08 DEBUG "value": [ 06:42:08 DEBUG 1713591728, 06:42:08 DEBUG "2669.133333333333" 06:42:08 DEBUG ] 06:42:08 DEBUG } 06:42:08 DEBUG ] 06:42:08 DEBUG } 06:42:08 DEBUG } 06:42:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713591728" 06:42:08 INFO [http_cmd]: http status code OK 06:42:08 DEBUG --- status code --- 06:42:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:42:08 DEBUG --- http response --- 06:42:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:42:08 DEBUG ----- output ----- 06:42:08 DEBUG { 06:42:08 DEBUG "status": "success", 06:42:08 DEBUG "data": { 06:42:08 DEBUG "resultType": "vector", 06:42:08 DEBUG "result": [ 06:42:08 DEBUG { 06:42:08 DEBUG "metric": { 06:42:08 DEBUG "instance": "10.142.0.107:9100" 06:42:08 DEBUG }, 06:42:08 DEBUG "value": [ 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG [...] 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG 06:42:08 DEBUG "instance": "10.142.0.43:9100" 06:42:08 DEBUG }, 06:42:08 DEBUG "value": [ 06:42:08 DEBUG 1713591728, 06:42:08 DEBUG "0.0693333333333423" 06:42:08 DEBUG ] 06:42:08 DEBUG } 06:42:08 DEBUG ] 06:42:08 DEBUG } 06:42:08 DEBUG } 06:42:08 INFO 06:42:08 INFO [loop_until]: kubectl --namespace=xlou top node 06:42:08 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:42:08 INFO 06:42:08 INFO [loop_until]: kubectl --namespace=xlou top pods 06:42:08 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:42:08 INFO [loop_until]: OK (rc = 0) 06:42:08 DEBUG --- stdout --- 06:42:08 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 16898m 8647Mi am-767f6b67b5-qcqz2 17322m 8706Mi am-767f6b67b5-vbltz 19026m 8650Mi ds-cts-0 18753m 10385Mi ds-cts-1 16899m 10658Mi ds-cts-2 19210m 10034Mi ds-idrepo-0 7683m 3745Mi ds-idrepo-1 7845m 3770Mi ds-idrepo-2 7740m 3695Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 3m 1177Mi idm-b85f9d978-wnhxb 3m 1236Mi lodemon-7cb5b9456d-dskfn 2m 62Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 8257m 4652Mi 06:42:08 DEBUG --- stderr --- 06:42:08 DEBUG 06:42:08 INFO [loop_until]: OK (rc = 0) 06:42:08 DEBUG --- stdout --- 06:42:08 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 69m 0% 6262Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 8030m 26% 7418Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 17169m 57% 11368Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 83m 0% 3374Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 18917m 63% 13524Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 162m 0% 5202Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 17417m 58% 11589Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 19206m 64% 3009Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 19477m 65% 10866Mi 9% gke-xlou-cdm-ds-0f601751-j488 8152m 27% 7023Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 17296m 57% 15579Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 8047m 26% 7669Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 19409m 65% 12650Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 8160m 51% 8500Mi 14% 06:42:08 DEBUG --- stderr --- 06:42:08 DEBUG 06:43:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591788" 06:43:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713591788" 06:43:08 ERROR [http_cmd]: ERROR 06:43:08 ERROR --- status code --- 06:43:08 ERROR http status code is 400 (expected [200, 401, 403, 404]) 06:43:08 ERROR --- http response --- 06:43:08 INFO [http_cmd]: http status code OK 06:43:08 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 06:43:08 DEBUG --- status code --- 06:43:08 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 06:43:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:43:08 DEBUG --- http response --- 06:43:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (98 lines): 06:43:08 DEBUG ----- output ----- 06:43:08 DEBUG { 06:43:08 DEBUG "status": "success", 06:43:08 DEBUG "data": { 06:43:08 DEBUG "resultType": "vector", 06:43:08 DEBUG "result": [ 06:43:08 DEBUG { 06:43:08 DEBUG "metric": { 06:43:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:43:08 DEBUG }, 06:43:08 DEBUG "value": [ 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG [...] 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 06:43:08 DEBUG }, 06:43:08 DEBUG "value": [ 06:43:08 DEBUG 1713591788, 06:43:08 DEBUG "0" 06:43:08 DEBUG ] 06:43:08 DEBUG } 06:43:08 DEBUG ] 06:43:08 DEBUG } 06:43:08 DEBUG } 06:43:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591788" 06:43:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713591788" 06:43:08 INFO [http_cmd]: http status code OK 06:43:08 DEBUG --- status code --- 06:43:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:43:08 DEBUG --- http response --- 06:43:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:43:08 DEBUG ----- output ----- 06:43:08 DEBUG { 06:43:08 DEBUG "status": "success", 06:43:08 DEBUG "data": { 06:43:08 DEBUG "resultType": "vector", 06:43:08 DEBUG "result": [ 06:43:08 DEBUG { 06:43:08 DEBUG "metric": { 06:43:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:43:08 DEBUG }, 06:43:08 DEBUG "value": [ 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG [...] 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:43:08 DEBUG }, 06:43:08 DEBUG "value": [ 06:43:08 DEBUG 1713591788, 06:43:08 DEBUG "480418.5" 06:43:08 DEBUG ] 06:43:08 DEBUG } 06:43:08 DEBUG ] 06:43:08 DEBUG } 06:43:08 DEBUG } 06:43:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591788" 06:43:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591788" 06:43:08 INFO [http_cmd]: http status code OK 06:43:08 DEBUG --- status code --- 06:43:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:43:08 DEBUG --- http response --- 06:43:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:43:08 DEBUG ----- output ----- 06:43:08 DEBUG { 06:43:08 DEBUG "status": "success", 06:43:08 DEBUG "data": { 06:43:08 DEBUG "resultType": "vector", 06:43:08 DEBUG "result": [ 06:43:08 DEBUG { 06:43:08 DEBUG "metric": { 06:43:08 DEBUG "instance": "10.142.0.107:9100" 06:43:08 DEBUG }, 06:43:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713591788" 06:43:08 DEBUG "value": [ 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG [...] 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG "instance": "10.142.0.43:9100" 06:43:08 DEBUG }, 06:43:08 DEBUG "value": [ 06:43:08 DEBUG 1713591788, 06:43:08 DEBUG "3191741518848" 06:43:08 DEBUG ] 06:43:08 DEBUG } 06:43:08 DEBUG ] 06:43:08 DEBUG } 06:43:08 DEBUG } 06:43:08 INFO [http_cmd]: http status code OK 06:43:08 DEBUG --- status code --- 06:43:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:43:08 DEBUG --- http response --- 06:43:08 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 06:43:08 INFO [http_cmd]: http status code OK 06:43:08 DEBUG --- status code --- 06:43:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:43:08 DEBUG --- http response --- 06:43:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (98 lines): 06:43:08 DEBUG ----- output ----- 06:43:08 DEBUG { 06:43:08 DEBUG "status": "success", 06:43:08 DEBUG "data": { 06:43:08 DEBUG "resultType": "vector", 06:43:08 DEBUG "result": [ 06:43:08 DEBUG { 06:43:08 DEBUG "metric": { 06:43:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:43:08 DEBUG }, 06:43:08 DEBUG "value": [ 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG [...] 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 06:43:08 DEBUG }, 06:43:08 DEBUG "value": [ 06:43:08 DEBUG 1713591788, 06:43:08 DEBUG "0" 06:43:08 DEBUG ] 06:43:08 DEBUG } 06:43:08 DEBUG ] 06:43:08 DEBUG } 06:43:08 DEBUG } 06:43:08 INFO [http_cmd]: http status code OK 06:43:08 DEBUG --- status code --- 06:43:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:43:08 DEBUG --- http response --- 06:43:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:43:08 DEBUG ----- output ----- 06:43:08 DEBUG { 06:43:08 DEBUG "status": "success", 06:43:08 DEBUG "data": { 06:43:08 DEBUG "resultType": "vector", 06:43:08 DEBUG "result": [ 06:43:08 DEBUG { 06:43:08 DEBUG "metric": { 06:43:08 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 06:43:08 DEBUG }, 06:43:08 DEBUG "value": [ 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG [...] 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 06:43:08 DEBUG }, 06:43:08 DEBUG "value": [ 06:43:08 DEBUG 1713591788, 06:43:08 DEBUG "4978335744" 06:43:08 DEBUG ] 06:43:08 DEBUG } 06:43:08 DEBUG ] 06:43:08 DEBUG } 06:43:08 DEBUG } 06:43:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591788" 06:43:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591788" 06:43:08 INFO [http_cmd]: http status code OK 06:43:08 DEBUG --- status code --- 06:43:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:43:08 DEBUG --- http response --- 06:43:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:43:08 DEBUG ----- output ----- 06:43:08 DEBUG { 06:43:08 DEBUG "status": "success", 06:43:08 DEBUG "data": { 06:43:08 DEBUG "resultType": "vector", 06:43:08 DEBUG "result": [ 06:43:08 DEBUG { 06:43:08 DEBUG "metric": { 06:43:08 DEBUG "pod": "ds-idrepo-0" 06:43:08 DEBUG }, 06:43:08 DEBUG "value": [ 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG [...] 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG "pod": "ds-idrepo-2" 06:43:08 DEBUG }, 06:43:08 DEBUG "value": [ 06:43:08 DEBUG 1713591788, 06:43:08 DEBUG "0" 06:43:08 DEBUG ] 06:43:08 DEBUG } 06:43:08 DEBUG ] 06:43:08 DEBUG } 06:43:08 DEBUG } 06:43:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591788" 06:43:08 INFO [http_cmd]: http status code OK 06:43:08 DEBUG --- status code --- 06:43:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:43:08 DEBUG --- http response --- 06:43:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:43:08 DEBUG ----- output ----- 06:43:08 DEBUG { 06:43:08 DEBUG "status": "success", 06:43:08 DEBUG "data": { 06:43:08 DEBUG "resultType": "vector", 06:43:08 DEBUG "result": [ 06:43:08 DEBUG { 06:43:08 DEBUG "metric": { 06:43:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:43:08 DEBUG }, 06:43:08 DEBUG "value": [ 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG [...] 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:43:08 DEBUG }, 06:43:08 DEBUG "value": [ 06:43:08 DEBUG 1713591788, 06:43:08 DEBUG "2666.633333333333" 06:43:08 DEBUG ] 06:43:08 DEBUG } 06:43:08 DEBUG ] 06:43:08 DEBUG } 06:43:08 DEBUG } 06:43:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591788" 06:43:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591788" 06:43:08 INFO [http_cmd]: http status code OK 06:43:08 DEBUG --- status code --- 06:43:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:43:08 DEBUG --- http response --- 06:43:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:43:08 DEBUG ----- output ----- 06:43:08 DEBUG { 06:43:08 DEBUG "status": "success", 06:43:08 DEBUG "data": { 06:43:08 DEBUG "resultType": "vector", 06:43:08 DEBUG "result": [ 06:43:08 DEBUG { 06:43:08 DEBUG "metric": { 06:43:08 DEBUG "pod": "ds-cts-0" 06:43:08 DEBUG }, 06:43:08 DEBUG "value": [ 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG [...] 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG "pod": "ds-cts-2" 06:43:08 DEBUG }, 06:43:08 DEBUG "value": [ 06:43:08 DEBUG 1713591788, 06:43:08 DEBUG "0" 06:43:08 DEBUG ] 06:43:08 DEBUG } 06:43:08 DEBUG ] 06:43:08 DEBUG } 06:43:08 DEBUG } 06:43:08 INFO [http_cmd]: http status code OK 06:43:08 DEBUG --- status code --- 06:43:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:43:08 DEBUG --- http response --- 06:43:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:43:08 DEBUG ----- output ----- 06:43:08 DEBUG { 06:43:08 DEBUG "status": "success", 06:43:08 DEBUG "data": { 06:43:08 DEBUG "resultType": "vector", 06:43:08 DEBUG "result": [ 06:43:08 DEBUG { 06:43:08 DEBUG "metric": { 06:43:08 DEBUG "pod": "ds-idrepo-0" 06:43:08 DEBUG }, 06:43:08 DEBUG "value": [ 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG [...] 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG "pod": "ds-idrepo-2" 06:43:08 DEBUG }, 06:43:08 DEBUG "value": [ 06:43:08 DEBUG 1713591788, 06:43:08 DEBUG "0" 06:43:08 DEBUG ] 06:43:08 DEBUG } 06:43:08 DEBUG ] 06:43:08 DEBUG } 06:43:08 DEBUG } 06:43:08 INFO [http_cmd]: http status code OK 06:43:08 DEBUG --- status code --- 06:43:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:43:08 DEBUG --- http response --- 06:43:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:43:08 DEBUG ----- output ----- 06:43:08 DEBUG { 06:43:08 DEBUG "status": "success", 06:43:08 DEBUG "data": { 06:43:08 DEBUG "resultType": "vector", 06:43:08 DEBUG "result": [ 06:43:08 DEBUG { 06:43:08 DEBUG "metric": { 06:43:08 DEBUG "pod": "ds-cts-0" 06:43:08 DEBUG }, 06:43:08 DEBUG "value": [ 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG [...] 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG "pod": "ds-cts-2" 06:43:08 DEBUG }, 06:43:08 DEBUG "value": [ 06:43:08 DEBUG 1713591788, 06:43:08 DEBUG "0" 06:43:08 DEBUG ] 06:43:08 DEBUG } 06:43:08 DEBUG ] 06:43:08 DEBUG } 06:43:08 DEBUG } 06:43:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713591788" 06:43:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713591788" 06:43:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591788" 06:43:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713591788" 06:43:08 INFO [http_cmd]: http status code OK 06:43:08 DEBUG --- status code --- 06:43:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:43:08 DEBUG --- http response --- 06:43:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:43:08 DEBUG ----- output ----- 06:43:08 DEBUG { 06:43:08 DEBUG "status": "success", 06:43:08 DEBUG "data": { 06:43:08 DEBUG "resultType": "vector", 06:43:08 DEBUG "result": [ 06:43:08 DEBUG { 06:43:08 DEBUG "metric": { 06:43:08 DEBUG "pod": "ds-idrepo-0" 06:43:08 DEBUG }, 06:43:08 DEBUG "value": [ 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG [...] 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG "pod": "ds-cts-2" 06:43:08 DEBUG }, 06:43:08 DEBUG "value": [ 06:43:08 DEBUG 1713591788, 06:43:08 DEBUG "0" 06:43:08 DEBUG ] 06:43:08 DEBUG } 06:43:08 DEBUG ] 06:43:08 DEBUG } 06:43:08 DEBUG } 06:43:08 INFO [http_cmd]: http status code OK 06:43:08 INFO [http_cmd]: http status code OK 06:43:08 DEBUG --- status code --- 06:43:08 DEBUG --- status code --- 06:43:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:43:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:43:08 DEBUG --- http response --- 06:43:08 DEBUG --- http response --- 06:43:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:43:08 DEBUG ----- output ----- 06:43:08 DEBUG { 06:43:08 DEBUG "status": "success", 06:43:08 DEBUG "data": { 06:43:08 DEBUG "resultType": "vector", 06:43:08 DEBUG "result": [ 06:43:08 DEBUG { 06:43:08 DEBUG "metric": { 06:43:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:43:08 DEBUG }, 06:43:08 DEBUG "value": [ 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG [...] 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:43:08 DEBUG }, 06:43:08 DEBUG "value": [ 06:43:08 DEBUG 1713591788, 06:43:08 DEBUG "0" 06:43:08 DEBUG ] 06:43:08 DEBUG } 06:43:08 DEBUG ] 06:43:08 DEBUG } 06:43:08 DEBUG } 06:43:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:43:08 DEBUG ----- output ----- 06:43:08 DEBUG { 06:43:08 DEBUG "status": "success", 06:43:08 DEBUG "data": { 06:43:08 DEBUG "resultType": "vector", 06:43:08 DEBUG "result": [ 06:43:08 DEBUG { 06:43:08 DEBUG "metric": { 06:43:08 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 06:43:08 DEBUG }, 06:43:08 DEBUG "value": [ 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG [...] 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 06:43:08 DEBUG }, 06:43:08 DEBUG "value": [ 06:43:08 DEBUG 1713591788, 06:43:08 DEBUG "8.023022759227365" 06:43:08 DEBUG ] 06:43:08 DEBUG } 06:43:08 DEBUG ] 06:43:08 DEBUG } 06:43:08 DEBUG } 06:43:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591788" 06:43:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591788" 06:43:08 INFO [http_cmd]: http status code OK 06:43:08 DEBUG --- status code --- 06:43:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:43:08 DEBUG --- http response --- 06:43:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:43:08 DEBUG ----- output ----- 06:43:08 DEBUG { 06:43:08 DEBUG "status": "success", 06:43:08 DEBUG "data": { 06:43:08 DEBUG "resultType": "vector", 06:43:08 DEBUG "result": [ 06:43:08 DEBUG { 06:43:08 DEBUG "metric": { 06:43:08 DEBUG "instance": "10.142.0.107:9100" 06:43:08 DEBUG }, 06:43:08 DEBUG "value": [ 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG [...] 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG "instance": "10.142.0.43:9100" 06:43:08 DEBUG }, 06:43:08 DEBUG "value": [ 06:43:08 DEBUG 1713591788, 06:43:08 DEBUG "0.278266666666704" 06:43:08 DEBUG ] 06:43:08 DEBUG } 06:43:08 DEBUG ] 06:43:08 DEBUG } 06:43:08 DEBUG } 06:43:08 INFO [http_cmd]: http status code OK 06:43:08 DEBUG --- status code --- 06:43:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:43:08 DEBUG --- http response --- 06:43:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:43:08 DEBUG ----- output ----- 06:43:08 DEBUG { 06:43:08 DEBUG "status": "success", 06:43:08 DEBUG "data": { 06:43:08 DEBUG "resultType": "vector", 06:43:08 DEBUG "result": [ 06:43:08 DEBUG { 06:43:08 DEBUG "metric": { 06:43:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:43:08 DEBUG }, 06:43:08 DEBUG "value": [ 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG [...] 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:43:08 DEBUG }, 06:43:08 DEBUG "value": [ 06:43:08 DEBUG 1713591788, 06:43:08 DEBUG "2669.0666666666666" 06:43:08 DEBUG ] 06:43:08 DEBUG } 06:43:08 DEBUG ] 06:43:08 DEBUG } 06:43:08 DEBUG } 06:43:08 INFO [http_cmd]: http status code OK 06:43:08 DEBUG --- status code --- 06:43:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:43:08 DEBUG --- http response --- 06:43:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:43:08 DEBUG ----- output ----- 06:43:08 DEBUG { 06:43:08 DEBUG "status": "success", 06:43:08 DEBUG "data": { 06:43:08 DEBUG "resultType": "vector", 06:43:08 DEBUG "result": [ 06:43:08 DEBUG { 06:43:08 DEBUG "metric": { 06:43:08 DEBUG "pod": "ds-idrepo-0" 06:43:08 DEBUG }, 06:43:08 DEBUG "value": [ 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG [...] 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG "pod": "ds-idrepo-2" 06:43:08 DEBUG }, 06:43:08 DEBUG "value": [ 06:43:08 DEBUG 1713591788, 06:43:08 DEBUG "0" 06:43:08 DEBUG ] 06:43:08 DEBUG } 06:43:08 DEBUG ] 06:43:08 DEBUG } 06:43:08 DEBUG } 06:43:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713591788" 06:43:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713591788" 06:43:08 INFO [http_cmd]: http status code OK 06:43:08 DEBUG --- status code --- 06:43:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:43:08 DEBUG --- http response --- 06:43:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:43:08 DEBUG ----- output ----- 06:43:08 DEBUG { 06:43:08 DEBUG "status": "success", 06:43:08 DEBUG "data": { 06:43:08 DEBUG "resultType": "vector", 06:43:08 DEBUG "result": [ 06:43:08 DEBUG { 06:43:08 DEBUG "metric": { 06:43:08 DEBUG "instance": "10.142.0.107:9100" 06:43:08 DEBUG }, 06:43:08 DEBUG "value": [ 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG [...] 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG "instance": "10.142.0.43:9100" 06:43:08 DEBUG }, 06:43:08 DEBUG "value": [ 06:43:08 DEBUG 1713591788, 06:43:08 DEBUG "80765847.06666666" 06:43:08 DEBUG ] 06:43:08 DEBUG } 06:43:08 DEBUG ] 06:43:08 DEBUG } 06:43:08 DEBUG } 06:43:08 INFO [http_cmd]: http status code OK 06:43:08 DEBUG --- status code --- 06:43:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:43:08 DEBUG --- http response --- 06:43:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591788" 06:43:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:43:08 DEBUG ----- output ----- 06:43:08 DEBUG { 06:43:08 DEBUG "status": "success", 06:43:08 DEBUG "data": { 06:43:08 DEBUG "resultType": "vector", 06:43:08 DEBUG "result": [ 06:43:08 DEBUG { 06:43:08 DEBUG "metric": { 06:43:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:43:08 DEBUG }, 06:43:08 DEBUG "value": [ 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG [...] 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:43:08 DEBUG }, 06:43:08 DEBUG "value": [ 06:43:08 DEBUG 1713591788, 06:43:08 DEBUG "0.095" 06:43:08 DEBUG ] 06:43:08 DEBUG } 06:43:08 DEBUG ] 06:43:08 DEBUG } 06:43:08 DEBUG } 06:43:08 INFO [http_cmd]: http status code OK 06:43:08 DEBUG --- status code --- 06:43:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:43:08 DEBUG --- http response --- 06:43:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:43:08 DEBUG ----- output ----- 06:43:08 DEBUG { 06:43:08 DEBUG "status": "success", 06:43:08 DEBUG "data": { 06:43:08 DEBUG "resultType": "vector", 06:43:08 DEBUG "result": [ 06:43:08 DEBUG { 06:43:08 DEBUG "metric": { 06:43:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:43:08 DEBUG }, 06:43:08 DEBUG "value": [ 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG [...] 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:43:08 DEBUG }, 06:43:08 DEBUG "value": [ 06:43:08 DEBUG 1713591788, 06:43:08 DEBUG "2666.2" 06:43:08 DEBUG ] 06:43:08 DEBUG } 06:43:08 DEBUG ] 06:43:08 DEBUG } 06:43:08 DEBUG } 06:43:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713591788" 06:43:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591788" 06:43:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591788" 06:43:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591788" 06:43:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591788" 06:43:08 INFO [http_cmd]: http status code OK 06:43:08 DEBUG --- status code --- 06:43:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:43:08 DEBUG --- http response --- 06:43:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:43:08 DEBUG ----- output ----- 06:43:08 DEBUG { 06:43:08 DEBUG "status": "success", 06:43:08 DEBUG "data": { 06:43:08 DEBUG "resultType": "vector", 06:43:08 DEBUG "result": [ 06:43:08 DEBUG { 06:43:08 DEBUG "metric": { 06:43:08 DEBUG "instance": "10.142.0.107:9100" 06:43:08 DEBUG }, 06:43:08 DEBUG "value": [ 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG [...] 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG "instance": "10.142.0.43:9100" 06:43:08 DEBUG }, 06:43:08 DEBUG "value": [ 06:43:08 DEBUG 1713591788, 06:43:08 DEBUG "80771967.66666666" 06:43:08 DEBUG ] 06:43:08 DEBUG } 06:43:08 DEBUG ] 06:43:08 DEBUG } 06:43:08 DEBUG } 06:43:08 INFO [http_cmd]: http status code OK 06:43:08 DEBUG --- status code --- 06:43:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:43:08 DEBUG --- http response --- 06:43:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 06:43:08 DEBUG ----- output ----- 06:43:08 DEBUG { 06:43:08 DEBUG "status": "success", 06:43:08 DEBUG "data": { 06:43:08 DEBUG "resultType": "vector", 06:43:08 DEBUG "result": [ 06:43:08 DEBUG { 06:43:08 DEBUG "metric": { 06:43:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:43:08 DEBUG }, 06:43:08 DEBUG "value": [ 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG [...] 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG "pod": "ds-idrepo-2" 06:43:08 DEBUG }, 06:43:08 DEBUG "value": [ 06:43:08 DEBUG 1713591788, 06:43:08 DEBUG "0" 06:43:08 DEBUG ] 06:43:08 DEBUG } 06:43:08 DEBUG ] 06:43:08 DEBUG } 06:43:08 DEBUG } 06:43:08 INFO [http_cmd]: http status code OK 06:43:08 INFO [http_cmd]: http status code OK 06:43:08 DEBUG --- status code --- 06:43:08 INFO [http_cmd]: http status code OK 06:43:08 DEBUG --- status code --- 06:43:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:43:08 DEBUG --- status code --- 06:43:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:43:08 DEBUG --- http response --- 06:43:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:43:08 DEBUG --- http response --- 06:43:08 DEBUG --- http response --- 06:43:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:43:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:43:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 06:43:08 DEBUG ----- output ----- 06:43:08 DEBUG ----- output ----- 06:43:08 DEBUG ----- output ----- 06:43:08 DEBUG { 06:43:08 DEBUG { 06:43:08 DEBUG { 06:43:08 DEBUG "status": "success", 06:43:08 DEBUG "status": "success", 06:43:08 DEBUG "status": "success", 06:43:08 DEBUG "data": { 06:43:08 DEBUG "data": { 06:43:08 DEBUG "data": { 06:43:08 DEBUG "resultType": "vector", 06:43:08 DEBUG "resultType": "vector", 06:43:08 DEBUG "resultType": "vector", 06:43:08 DEBUG "result": [ 06:43:08 DEBUG "result": [ 06:43:08 DEBUG "result": [ 06:43:08 DEBUG { 06:43:08 DEBUG { 06:43:08 DEBUG { 06:43:08 DEBUG "metric": { 06:43:08 DEBUG "metric": { 06:43:08 DEBUG "metric": { 06:43:08 DEBUG "pod": "ds-cts-0" 06:43:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:43:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:43:08 DEBUG }, 06:43:08 DEBUG }, 06:43:08 DEBUG }, 06:43:08 DEBUG "value": [ 06:43:08 DEBUG "value": [ 06:43:08 DEBUG "value": [ 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG [...] 06:43:08 DEBUG [...] 06:43:08 DEBUG [...] 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG "pod": "ds-idrepo-2" 06:43:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:43:08 DEBUG "pod": "ds-cts-2" 06:43:08 DEBUG }, 06:43:08 DEBUG }, 06:43:08 DEBUG }, 06:43:08 DEBUG "value": [ 06:43:08 DEBUG "value": [ 06:43:08 DEBUG "value": [ 06:43:08 DEBUG 1713591788, 06:43:08 DEBUG 1713591788, 06:43:08 DEBUG 1713591788, 06:43:08 DEBUG "0" 06:43:08 DEBUG "2666.1" 06:43:08 DEBUG "0.0011549177777777778" 06:43:08 DEBUG ] 06:43:08 DEBUG ] 06:43:08 DEBUG ] 06:43:08 DEBUG } 06:43:08 DEBUG } 06:43:08 DEBUG } 06:43:08 DEBUG ] 06:43:08 DEBUG ] 06:43:08 DEBUG ] 06:43:08 DEBUG } 06:43:08 DEBUG } 06:43:08 DEBUG } 06:43:08 DEBUG } 06:43:08 DEBUG } 06:43:08 DEBUG } 06:43:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713591788" 06:43:08 INFO [http_cmd]: http status code OK 06:43:08 DEBUG --- status code --- 06:43:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:43:08 DEBUG --- http response --- 06:43:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:43:08 DEBUG ----- output ----- 06:43:08 DEBUG { 06:43:08 DEBUG "status": "success", 06:43:08 DEBUG "data": { 06:43:08 DEBUG "resultType": "vector", 06:43:08 DEBUG "result": [ 06:43:08 DEBUG { 06:43:08 DEBUG "metric": { 06:43:08 DEBUG "instance": "10.142.0.107:9100" 06:43:08 DEBUG }, 06:43:08 DEBUG "value": [ 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG [...] 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG 06:43:08 DEBUG "instance": "10.142.0.43:9100" 06:43:08 DEBUG }, 06:43:08 DEBUG "value": [ 06:43:08 DEBUG 1713591788, 06:43:08 DEBUG "0.1053333333333209" 06:43:08 DEBUG ] 06:43:08 DEBUG } 06:43:08 DEBUG ] 06:43:08 DEBUG } 06:43:08 DEBUG } 06:43:08 INFO 06:43:08 INFO [loop_until]: kubectl --namespace=xlou top pods 06:43:08 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:43:08 INFO [loop_until]: OK (rc = 0) 06:43:08 DEBUG --- stdout --- 06:43:08 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 17453m 8649Mi am-767f6b67b5-qcqz2 17336m 8706Mi am-767f6b67b5-vbltz 18658m 8652Mi ds-cts-0 18700m 10384Mi ds-cts-1 17153m 10667Mi ds-cts-2 19126m 10052Mi ds-idrepo-0 7931m 3744Mi ds-idrepo-1 7596m 3782Mi ds-idrepo-2 7790m 3683Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1177Mi idm-b85f9d978-wnhxb 3m 1236Mi lodemon-7cb5b9456d-dskfn 2m 62Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7935m 4655Mi 06:43:08 DEBUG --- stderr --- 06:43:08 DEBUG 06:43:08 INFO 06:43:08 INFO [loop_until]: kubectl --namespace=xlou top node 06:43:08 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:43:08 INFO [loop_until]: OK (rc = 0) 06:43:08 DEBUG --- stdout --- 06:43:08 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 70m 0% 6265Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7959m 26% 7420Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 17036m 57% 11360Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 79m 0% 3375Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 17695m 59% 13537Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 152m 0% 5198Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 17387m 58% 11602Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 19140m 64% 3024Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 19311m 64% 10866Mi 9% gke-xlou-cdm-ds-0f601751-j488 8119m 27% 7012Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 17469m 58% 15584Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 8021m 26% 7685Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 19188m 64% 12684Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 8179m 51% 8497Mi 14% 06:43:08 DEBUG --- stderr --- 06:43:08 DEBUG 06:44:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591848" 06:44:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713591848" 06:44:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591848" 06:44:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713591848" 06:44:08 INFO [http_cmd]: http status code OK 06:44:08 DEBUG --- status code --- 06:44:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:44:08 DEBUG --- http response --- 06:44:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 06:44:08 DEBUG ----- output ----- 06:44:08 DEBUG { 06:44:08 DEBUG "status": "success", 06:44:08 DEBUG "data": { 06:44:08 DEBUG "resultType": "vector", 06:44:08 DEBUG "result": [ 06:44:08 DEBUG { 06:44:08 DEBUG "metric": { 06:44:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:44:08 DEBUG }, 06:44:08 DEBUG "value": [ 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG [...] 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG "pod": "idm-b85f9d978-wnhxb" 06:44:08 DEBUG }, 06:44:08 DEBUG "value": [ 06:44:08 DEBUG 1713591848, 06:44:08 DEBUG "0" 06:44:08 DEBUG ] 06:44:08 DEBUG } 06:44:08 DEBUG ] 06:44:08 DEBUG } 06:44:08 DEBUG } 06:44:08 INFO [http_cmd]: http status code OK 06:44:08 DEBUG --- status code --- 06:44:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:44:08 DEBUG --- http response --- 06:44:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:44:08 DEBUG ----- output ----- 06:44:08 DEBUG { 06:44:08 DEBUG "status": "success", 06:44:08 DEBUG "data": { 06:44:08 DEBUG "resultType": "vector", 06:44:08 DEBUG "result": [ 06:44:08 DEBUG { 06:44:08 DEBUG "metric": { 06:44:08 DEBUG "instance": "10.142.0.107:9100" 06:44:08 DEBUG }, 06:44:08 DEBUG "value": [ 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG [...] 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG "instance": "10.142.0.43:9100" 06:44:08 DEBUG }, 06:44:08 DEBUG "value": [ 06:44:08 DEBUG 1713591848, 06:44:08 DEBUG "3196072975360" 06:44:08 DEBUG ] 06:44:08 DEBUG } 06:44:08 DEBUG ] 06:44:08 DEBUG } 06:44:08 DEBUG } 06:44:08 INFO [http_cmd]: http status code OK 06:44:08 DEBUG --- status code --- 06:44:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:44:08 DEBUG --- http response --- 06:44:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:44:08 DEBUG ----- output ----- 06:44:08 DEBUG { 06:44:08 DEBUG "status": "success", 06:44:08 DEBUG "data": { 06:44:08 DEBUG "resultType": "vector", 06:44:08 DEBUG "result": [ 06:44:08 DEBUG { 06:44:08 DEBUG "metric": { 06:44:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:44:08 DEBUG }, 06:44:08 DEBUG "value": [ 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG [...] 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:44:08 DEBUG }, 06:44:08 DEBUG "value": [ 06:44:08 DEBUG 1713591848, 06:44:08 DEBUG "480495.5" 06:44:08 DEBUG ] 06:44:08 DEBUG } 06:44:08 DEBUG ] 06:44:08 DEBUG } 06:44:08 DEBUG } 06:44:08 ERROR [http_cmd]: ERROR 06:44:08 ERROR --- status code --- 06:44:08 ERROR http status code is 400 (expected [200, 401, 403, 404]) 06:44:08 ERROR --- http response --- 06:44:08 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 06:44:08 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 06:44:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591848" 06:44:08 INFO [http_cmd]: http status code OK 06:44:08 DEBUG --- status code --- 06:44:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:44:08 DEBUG --- http response --- 06:44:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 06:44:08 DEBUG ----- output ----- 06:44:08 DEBUG { 06:44:08 DEBUG "status": "success", 06:44:08 DEBUG "data": { 06:44:08 DEBUG "resultType": "vector", 06:44:08 DEBUG "result": [ 06:44:08 DEBUG { 06:44:08 DEBUG "metric": { 06:44:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:44:08 DEBUG }, 06:44:08 DEBUG "value": [ 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG [...] 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG "pod": "idm-b85f9d978-wnhxb" 06:44:08 DEBUG }, 06:44:08 DEBUG "value": [ 06:44:08 DEBUG 1713591848, 06:44:08 DEBUG "448.52291846012116" 06:44:08 DEBUG ] 06:44:08 DEBUG } 06:44:08 DEBUG ] 06:44:08 DEBUG } 06:44:08 DEBUG } 06:44:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591848" 06:44:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591848" 06:44:08 INFO [http_cmd]: http status code OK 06:44:08 DEBUG --- status code --- 06:44:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:44:08 DEBUG --- http response --- 06:44:08 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 06:44:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713591848" 06:44:08 INFO [http_cmd]: http status code OK 06:44:08 DEBUG --- status code --- 06:44:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:44:08 DEBUG --- http response --- 06:44:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:44:08 DEBUG ----- output ----- 06:44:08 DEBUG { 06:44:08 DEBUG "status": "success", 06:44:08 DEBUG "data": { 06:44:08 DEBUG "resultType": "vector", 06:44:08 DEBUG "result": [ 06:44:08 DEBUG { 06:44:08 DEBUG "metric": { 06:44:08 DEBUG "pod": "ds-idrepo-0" 06:44:08 DEBUG }, 06:44:08 DEBUG "value": [ 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG [...] 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG "pod": "ds-idrepo-2" 06:44:08 DEBUG }, 06:44:08 DEBUG "value": [ 06:44:08 DEBUG 1713591848, 06:44:08 DEBUG "0" 06:44:08 DEBUG ] 06:44:08 DEBUG } 06:44:08 DEBUG ] 06:44:08 DEBUG } 06:44:08 DEBUG } 06:44:08 INFO [http_cmd]: http status code OK 06:44:08 DEBUG --- status code --- 06:44:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:44:08 DEBUG --- http response --- 06:44:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:44:08 DEBUG ----- output ----- 06:44:08 DEBUG { 06:44:08 DEBUG "status": "success", 06:44:08 DEBUG "data": { 06:44:08 DEBUG "resultType": "vector", 06:44:08 DEBUG "result": [ 06:44:08 DEBUG { 06:44:08 DEBUG "metric": { 06:44:08 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 06:44:08 DEBUG }, 06:44:08 DEBUG "value": [ 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG [...] 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 06:44:08 DEBUG }, 06:44:08 DEBUG "value": [ 06:44:08 DEBUG 1713591848, 06:44:08 DEBUG "4974641152" 06:44:08 DEBUG ] 06:44:08 DEBUG } 06:44:08 DEBUG ] 06:44:08 DEBUG } 06:44:08 DEBUG } 06:44:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591848" 06:44:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591848" 06:44:08 INFO [http_cmd]: http status code OK 06:44:08 DEBUG --- status code --- 06:44:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:44:08 DEBUG --- http response --- 06:44:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:44:08 DEBUG ----- output ----- 06:44:08 DEBUG { 06:44:08 DEBUG "status": "success", 06:44:08 DEBUG "data": { 06:44:08 DEBUG "resultType": "vector", 06:44:08 DEBUG "result": [ 06:44:08 DEBUG { 06:44:08 DEBUG "metric": { 06:44:08 DEBUG "pod": "ds-idrepo-0" 06:44:08 DEBUG }, 06:44:08 DEBUG "value": [ 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG [...] 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG "pod": "ds-idrepo-2" 06:44:08 DEBUG }, 06:44:08 DEBUG "value": [ 06:44:08 DEBUG 1713591848, 06:44:08 DEBUG "0" 06:44:08 DEBUG ] 06:44:08 DEBUG } 06:44:08 DEBUG ] 06:44:08 DEBUG } 06:44:08 DEBUG } 06:44:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591848" 06:44:08 INFO [http_cmd]: http status code OK 06:44:08 DEBUG --- status code --- 06:44:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:44:08 DEBUG --- http response --- 06:44:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:44:08 DEBUG ----- output ----- 06:44:08 DEBUG { 06:44:08 DEBUG "status": "success", 06:44:08 DEBUG "data": { 06:44:08 DEBUG "resultType": "vector", 06:44:08 DEBUG "result": [ 06:44:08 DEBUG { 06:44:08 DEBUG "metric": { 06:44:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:44:08 DEBUG }, 06:44:08 DEBUG "value": [ 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG [...] 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:44:08 DEBUG }, 06:44:08 DEBUG "value": [ 06:44:08 DEBUG 1713591848, 06:44:08 DEBUG "2670.9666666666667" 06:44:08 DEBUG ] 06:44:08 DEBUG } 06:44:08 DEBUG ] 06:44:08 DEBUG } 06:44:08 DEBUG } 06:44:08 INFO [http_cmd]: http status code OK 06:44:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591848" 06:44:08 DEBUG --- status code --- 06:44:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:44:08 DEBUG --- http response --- 06:44:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:44:08 DEBUG ----- output ----- 06:44:08 DEBUG { 06:44:08 DEBUG "status": "success", 06:44:08 DEBUG "data": { 06:44:08 DEBUG "resultType": "vector", 06:44:08 DEBUG "result": [ 06:44:08 DEBUG { 06:44:08 DEBUG "metric": { 06:44:08 DEBUG "pod": "ds-cts-0" 06:44:08 DEBUG }, 06:44:08 DEBUG "value": [ 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG [...] 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG "pod": "ds-cts-2" 06:44:08 DEBUG }, 06:44:08 DEBUG "value": [ 06:44:08 DEBUG 1713591848, 06:44:08 DEBUG "0" 06:44:08 DEBUG ] 06:44:08 DEBUG } 06:44:08 DEBUG ] 06:44:08 DEBUG } 06:44:08 DEBUG } 06:44:08 INFO [http_cmd]: http status code OK 06:44:08 DEBUG --- status code --- 06:44:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:44:08 DEBUG --- http response --- 06:44:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:44:08 DEBUG ----- output ----- 06:44:08 DEBUG { 06:44:08 DEBUG "status": "success", 06:44:08 DEBUG "data": { 06:44:08 DEBUG "resultType": "vector", 06:44:08 DEBUG "result": [ 06:44:08 DEBUG { 06:44:08 DEBUG "metric": { 06:44:08 DEBUG "pod": "ds-cts-0" 06:44:08 DEBUG }, 06:44:08 DEBUG "value": [ 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG [...] 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG "pod": "ds-cts-2" 06:44:08 DEBUG }, 06:44:08 DEBUG "value": [ 06:44:08 DEBUG 1713591848, 06:44:08 DEBUG "0" 06:44:08 DEBUG ] 06:44:08 DEBUG } 06:44:08 DEBUG ] 06:44:08 DEBUG } 06:44:08 DEBUG } 06:44:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591848" 06:44:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713591848" 06:44:08 INFO [http_cmd]: http status code OK 06:44:08 DEBUG --- status code --- 06:44:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:44:08 DEBUG --- http response --- 06:44:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:44:08 DEBUG ----- output ----- 06:44:08 DEBUG { 06:44:08 DEBUG "status": "success", 06:44:08 DEBUG "data": { 06:44:08 DEBUG "resultType": "vector", 06:44:08 DEBUG "result": [ 06:44:08 DEBUG { 06:44:08 DEBUG "metric": { 06:44:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:44:08 DEBUG }, 06:44:08 DEBUG "value": [ 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG [...] 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:44:08 DEBUG }, 06:44:08 DEBUG "value": [ 06:44:08 DEBUG 1713591848, 06:44:08 DEBUG "0" 06:44:08 DEBUG ] 06:44:08 DEBUG } 06:44:08 DEBUG ] 06:44:08 DEBUG } 06:44:08 DEBUG } 06:44:08 INFO [http_cmd]: http status code OK 06:44:08 DEBUG --- status code --- 06:44:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:44:08 DEBUG --- http response --- 06:44:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:44:08 DEBUG ----- output ----- 06:44:08 DEBUG { 06:44:08 DEBUG "status": "success", 06:44:08 DEBUG "data": { 06:44:08 DEBUG "resultType": "vector", 06:44:08 DEBUG "result": [ 06:44:08 DEBUG { 06:44:08 DEBUG "metric": { 06:44:08 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 06:44:08 DEBUG }, 06:44:08 DEBUG "value": [ 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG [...] 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 06:44:08 DEBUG }, 06:44:08 DEBUG "value": [ 06:44:08 DEBUG 1713591848, 06:44:08 DEBUG "8.036058173042028" 06:44:08 DEBUG ] 06:44:08 DEBUG } 06:44:08 DEBUG ] 06:44:08 DEBUG } 06:44:08 DEBUG } 06:44:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713591848" 06:44:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713591848" 06:44:08 INFO [http_cmd]: http status code OK 06:44:08 DEBUG --- status code --- 06:44:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:44:08 DEBUG --- http response --- 06:44:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:44:08 DEBUG ----- output ----- 06:44:08 DEBUG { 06:44:08 DEBUG "status": "success", 06:44:08 DEBUG "data": { 06:44:08 DEBUG "resultType": "vector", 06:44:08 DEBUG "result": [ 06:44:08 DEBUG { 06:44:08 DEBUG "metric": { 06:44:08 DEBUG "pod": "ds-idrepo-0" 06:44:08 DEBUG }, 06:44:08 DEBUG "value": [ 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG [...] 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG "pod": "ds-cts-2" 06:44:08 DEBUG }, 06:44:08 DEBUG "value": [ 06:44:08 DEBUG 1713591848, 06:44:08 DEBUG "0" 06:44:08 DEBUG ] 06:44:08 DEBUG } 06:44:08 DEBUG ] 06:44:08 DEBUG } 06:44:08 DEBUG } 06:44:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591848" 06:44:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591848" 06:44:08 INFO [http_cmd]: http status code OK 06:44:08 DEBUG --- status code --- 06:44:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:44:08 DEBUG --- http response --- 06:44:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:44:08 DEBUG ----- output ----- 06:44:08 DEBUG { 06:44:08 DEBUG "status": "success", 06:44:08 DEBUG "data": { 06:44:08 DEBUG "resultType": "vector", 06:44:08 DEBUG "result": [ 06:44:08 DEBUG { 06:44:08 DEBUG "metric": { 06:44:08 DEBUG "instance": "10.142.0.107:9100" 06:44:08 DEBUG }, 06:44:08 DEBUG "value": [ 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG [...] 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG "instance": "10.142.0.43:9100" 06:44:08 DEBUG }, 06:44:08 DEBUG "value": [ 06:44:08 DEBUG 1713591848, 06:44:08 DEBUG "0.23080000000008927" 06:44:08 DEBUG ] 06:44:08 DEBUG } 06:44:08 DEBUG ] 06:44:08 DEBUG } 06:44:08 DEBUG } 06:44:08 INFO [http_cmd]: http status code OK 06:44:08 DEBUG --- status code --- 06:44:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:44:08 DEBUG --- http response --- 06:44:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:44:08 DEBUG ----- output ----- 06:44:08 DEBUG { 06:44:08 DEBUG "status": "success", 06:44:08 DEBUG "data": { 06:44:08 DEBUG "resultType": "vector", 06:44:08 DEBUG "result": [ 06:44:08 DEBUG { 06:44:08 DEBUG "metric": { 06:44:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:44:08 DEBUG }, 06:44:08 DEBUG "value": [ 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG [...] 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:44:08 DEBUG }, 06:44:08 DEBUG "value": [ 06:44:08 DEBUG 1713591848, 06:44:08 DEBUG "2675.0333333333333" 06:44:08 DEBUG ] 06:44:08 DEBUG } 06:44:08 DEBUG ] 06:44:08 DEBUG } 06:44:08 DEBUG } 06:44:08 INFO [http_cmd]: http status code OK 06:44:08 DEBUG --- status code --- 06:44:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:44:08 DEBUG --- http response --- 06:44:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:44:08 DEBUG ----- output ----- 06:44:08 DEBUG { 06:44:08 DEBUG "status": "success", 06:44:08 DEBUG "data": { 06:44:08 DEBUG "resultType": "vector", 06:44:08 DEBUG "result": [ 06:44:08 DEBUG { 06:44:08 DEBUG "metric": { 06:44:08 DEBUG "pod": "ds-idrepo-0" 06:44:08 DEBUG }, 06:44:08 DEBUG "value": [ 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG [...] 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG "pod": "ds-idrepo-2" 06:44:08 DEBUG }, 06:44:08 DEBUG "value": [ 06:44:08 DEBUG 1713591848, 06:44:08 DEBUG "0" 06:44:08 DEBUG ] 06:44:08 DEBUG } 06:44:08 DEBUG ] 06:44:08 DEBUG } 06:44:08 DEBUG } 06:44:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713591848" 06:44:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591848" 06:44:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713591848" 06:44:08 INFO [http_cmd]: http status code OK 06:44:08 DEBUG --- status code --- 06:44:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:44:08 DEBUG --- http response --- 06:44:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:44:08 DEBUG ----- output ----- 06:44:08 DEBUG { 06:44:08 DEBUG "status": "success", 06:44:08 DEBUG "data": { 06:44:08 DEBUG "resultType": "vector", 06:44:08 DEBUG "result": [ 06:44:08 DEBUG { 06:44:08 DEBUG "metric": { 06:44:08 DEBUG "instance": "10.142.0.107:9100" 06:44:08 DEBUG }, 06:44:08 DEBUG "value": [ 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG [...] 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG "instance": "10.142.0.43:9100" 06:44:08 DEBUG }, 06:44:08 DEBUG "value": [ 06:44:08 DEBUG 1713591848, 06:44:08 DEBUG "80964373.19999999" 06:44:08 DEBUG ] 06:44:08 DEBUG } 06:44:08 DEBUG ] 06:44:08 DEBUG } 06:44:08 DEBUG } 06:44:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591848" 06:44:08 INFO [http_cmd]: http status code OK 06:44:08 DEBUG --- status code --- 06:44:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:44:08 DEBUG --- http response --- 06:44:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:44:08 DEBUG ----- output ----- 06:44:08 DEBUG { 06:44:08 DEBUG "status": "success", 06:44:08 DEBUG "data": { 06:44:08 DEBUG "resultType": "vector", 06:44:08 DEBUG "result": [ 06:44:08 DEBUG { 06:44:08 DEBUG "metric": { 06:44:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:44:08 DEBUG }, 06:44:08 DEBUG "value": [ 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG [...] 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:44:08 DEBUG }, 06:44:08 DEBUG "value": [ 06:44:08 DEBUG 1713591848, 06:44:08 DEBUG "2672" 06:44:08 DEBUG ] 06:44:08 DEBUG } 06:44:08 DEBUG ] 06:44:08 DEBUG } 06:44:08 DEBUG } 06:44:08 INFO [http_cmd]: http status code OK 06:44:08 DEBUG --- status code --- 06:44:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:44:08 DEBUG --- http response --- 06:44:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:44:08 DEBUG ----- output ----- 06:44:08 DEBUG { 06:44:08 DEBUG "status": "success", 06:44:08 DEBUG "data": { 06:44:08 DEBUG "resultType": "vector", 06:44:08 DEBUG "result": [ 06:44:08 DEBUG { 06:44:08 DEBUG "metric": { 06:44:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:44:08 DEBUG }, 06:44:08 DEBUG "value": [ 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG [...] 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:44:08 DEBUG }, 06:44:08 DEBUG "value": [ 06:44:08 DEBUG 1713591848, 06:44:08 DEBUG "0.096" 06:44:08 DEBUG ] 06:44:08 DEBUG } 06:44:08 DEBUG ] 06:44:08 DEBUG } 06:44:08 DEBUG } 06:44:08 INFO [http_cmd]: http status code OK 06:44:08 DEBUG --- status code --- 06:44:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:44:08 DEBUG --- http response --- 06:44:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:44:08 DEBUG ----- output ----- 06:44:08 DEBUG { 06:44:08 DEBUG "status": "success", 06:44:08 DEBUG "data": { 06:44:08 DEBUG "resultType": "vector", 06:44:08 DEBUG "result": [ 06:44:08 DEBUG { 06:44:08 DEBUG "metric": { 06:44:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:44:08 DEBUG }, 06:44:08 DEBUG "value": [ 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG [...] 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:44:08 DEBUG }, 06:44:08 DEBUG "value": [ 06:44:08 DEBUG 1713591848, 06:44:08 DEBUG "2672.7666666666664" 06:44:08 DEBUG ] 06:44:08 DEBUG } 06:44:08 DEBUG ] 06:44:08 DEBUG } 06:44:08 DEBUG } 06:44:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713591848" 06:44:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591848" 06:44:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591848" 06:44:08 INFO [http_cmd]: http status code OK 06:44:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591848" 06:44:08 DEBUG --- status code --- 06:44:08 INFO [http_cmd]: http status code OK 06:44:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:44:08 DEBUG --- status code --- 06:44:08 DEBUG --- http response --- 06:44:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:44:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:44:08 DEBUG --- http response --- 06:44:08 DEBUG ----- output ----- 06:44:08 DEBUG { 06:44:08 DEBUG "status": "success", 06:44:08 DEBUG "data": { 06:44:08 DEBUG "resultType": "vector", 06:44:08 DEBUG "result": [ 06:44:08 DEBUG { 06:44:08 DEBUG "metric": { 06:44:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:44:08 DEBUG }, 06:44:08 DEBUG "value": [ 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:44:08 DEBUG 06:44:08 INFO [http_cmd]: http status code OK 06:44:08 DEBUG ----- output ----- 06:44:08 DEBUG [...] 06:44:08 DEBUG --- status code --- 06:44:08 DEBUG { 06:44:08 DEBUG 06:44:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:44:08 DEBUG "status": "success", 06:44:08 DEBUG 06:44:08 DEBUG --- http response --- 06:44:08 DEBUG "data": { 06:44:08 DEBUG 06:44:08 DEBUG "resultType": "vector", 06:44:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:44:08 DEBUG "pod": "ds-idrepo-2" 06:44:08 DEBUG "result": [ 06:44:08 DEBUG ----- output ----- 06:44:08 DEBUG }, 06:44:08 DEBUG { 06:44:08 DEBUG { 06:44:08 DEBUG "value": [ 06:44:08 DEBUG "metric": { 06:44:08 DEBUG "status": "success", 06:44:08 DEBUG 1713591848, 06:44:08 DEBUG "instance": "10.142.0.107:9100" 06:44:08 DEBUG "data": { 06:44:08 DEBUG "0" 06:44:08 DEBUG }, 06:44:08 DEBUG "resultType": "vector", 06:44:08 DEBUG ] 06:44:08 DEBUG "value": [ 06:44:08 DEBUG "result": [ 06:44:08 DEBUG } 06:44:08 DEBUG 06:44:08 DEBUG { 06:44:08 DEBUG ] 06:44:08 DEBUG 06:44:08 DEBUG "metric": { 06:44:08 DEBUG } 06:44:08 DEBUG 06:44:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:44:08 DEBUG } 06:44:08 DEBUG [...] 06:44:08 DEBUG }, 06:44:08 DEBUG 06:44:08 DEBUG "value": [ 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG "instance": "10.142.0.43:9100" 06:44:08 DEBUG 06:44:08 DEBUG }, 06:44:08 DEBUG [...] 06:44:08 DEBUG "value": [ 06:44:08 DEBUG 06:44:08 DEBUG 1713591848, 06:44:08 DEBUG 06:44:08 DEBUG "80969315.93333334" 06:44:08 DEBUG 06:44:08 DEBUG ] 06:44:08 DEBUG "pod": "ds-idrepo-2" 06:44:08 DEBUG } 06:44:08 DEBUG }, 06:44:08 DEBUG ] 06:44:08 DEBUG "value": [ 06:44:08 DEBUG } 06:44:08 DEBUG 1713591848, 06:44:08 DEBUG } 06:44:08 DEBUG "0" 06:44:08 DEBUG ] 06:44:08 DEBUG } 06:44:08 DEBUG ] 06:44:08 DEBUG } 06:44:08 DEBUG } 06:44:08 INFO [http_cmd]: http status code OK 06:44:08 DEBUG --- status code --- 06:44:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:44:08 DEBUG --- http response --- 06:44:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:44:08 DEBUG ----- output ----- 06:44:08 DEBUG { 06:44:08 DEBUG "status": "success", 06:44:08 DEBUG "data": { 06:44:08 DEBUG "resultType": "vector", 06:44:08 DEBUG "result": [ 06:44:08 DEBUG { 06:44:08 DEBUG "metric": { 06:44:08 DEBUG "pod": "ds-cts-0" 06:44:08 DEBUG }, 06:44:08 DEBUG "value": [ 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG [...] 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG "pod": "ds-cts-2" 06:44:08 DEBUG }, 06:44:08 DEBUG "value": [ 06:44:08 DEBUG 1713591848, 06:44:08 DEBUG "0.0013666666666666666" 06:44:08 DEBUG ] 06:44:08 DEBUG } 06:44:08 DEBUG ] 06:44:08 DEBUG } 06:44:08 DEBUG } 06:44:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713591848" 06:44:08 INFO [http_cmd]: http status code OK 06:44:08 DEBUG --- status code --- 06:44:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:44:08 DEBUG --- http response --- 06:44:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:44:08 DEBUG ----- output ----- 06:44:08 DEBUG { 06:44:08 DEBUG "status": "success", 06:44:08 DEBUG "data": { 06:44:08 DEBUG "resultType": "vector", 06:44:08 DEBUG "result": [ 06:44:08 DEBUG { 06:44:08 DEBUG "metric": { 06:44:08 DEBUG "instance": "10.142.0.107:9100" 06:44:08 DEBUG }, 06:44:08 DEBUG "value": [ 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG [...] 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG 06:44:08 DEBUG "instance": "10.142.0.43:9100" 06:44:08 DEBUG }, 06:44:08 DEBUG "value": [ 06:44:08 DEBUG 1713591848, 06:44:08 DEBUG "0.08966666666667035" 06:44:08 DEBUG ] 06:44:08 DEBUG } 06:44:08 DEBUG ] 06:44:08 DEBUG } 06:44:08 DEBUG } 06:44:08 INFO 06:44:08 INFO [loop_until]: kubectl --namespace=xlou top pods 06:44:08 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:44:08 INFO [loop_until]: OK (rc = 0) 06:44:08 DEBUG --- stdout --- 06:44:08 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 17309m 8646Mi am-767f6b67b5-qcqz2 17335m 8704Mi am-767f6b67b5-vbltz 19061m 8651Mi ds-cts-0 18957m 10384Mi ds-cts-1 17093m 10667Mi ds-cts-2 19376m 10093Mi ds-idrepo-0 7511m 3745Mi ds-idrepo-1 7604m 3798Mi ds-idrepo-2 7956m 3683Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1177Mi idm-b85f9d978-wnhxb 4m 1236Mi lodemon-7cb5b9456d-dskfn 2m 62Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7825m 4651Mi 06:44:08 DEBUG --- stderr --- 06:44:08 DEBUG 06:44:08 INFO 06:44:08 INFO [loop_until]: kubectl --namespace=xlou top node 06:44:08 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:44:08 INFO [loop_until]: OK (rc = 0) 06:44:08 DEBUG --- stdout --- 06:44:08 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 70m 0% 6264Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 8067m 27% 7419Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 17126m 57% 11351Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 81m 0% 3376Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 18619m 62% 13537Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 150m 0% 5192Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 17347m 58% 11588Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 19119m 64% 3024Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 19526m 65% 10860Mi 9% gke-xlou-cdm-ds-0f601751-j488 8265m 27% 7004Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 17007m 56% 15590Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 7980m 26% 7697Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 19258m 64% 12693Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 8179m 51% 8491Mi 14% 06:44:08 DEBUG --- stderr --- 06:44:08 DEBUG 06:45:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591908" 06:45:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591908" 06:45:08 INFO [http_cmd]: http status code OK 06:45:08 INFO [http_cmd]: http status code OK 06:45:08 DEBUG --- status code --- 06:45:08 DEBUG --- status code --- 06:45:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:45:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:45:08 DEBUG --- http response --- 06:45:08 DEBUG --- http response --- 06:45:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:45:08 DEBUG ----- output ----- 06:45:08 DEBUG { 06:45:08 DEBUG "status": "success", 06:45:08 DEBUG "data": { 06:45:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 06:45:08 DEBUG "resultType": "vector", 06:45:08 DEBUG ----- output ----- 06:45:08 DEBUG "result": [ 06:45:08 DEBUG { 06:45:08 DEBUG { 06:45:08 DEBUG "status": "success", 06:45:08 DEBUG "metric": { 06:45:08 DEBUG "data": { 06:45:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:45:08 DEBUG "resultType": "vector", 06:45:08 DEBUG }, 06:45:08 DEBUG "result": [ 06:45:08 DEBUG "value": [ 06:45:08 DEBUG { 06:45:08 DEBUG 06:45:08 DEBUG "metric": { 06:45:08 DEBUG 06:45:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:45:08 DEBUG 06:45:08 DEBUG }, 06:45:08 DEBUG [...] 06:45:08 DEBUG "value": [ 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:45:08 DEBUG [...] 06:45:08 DEBUG }, 06:45:08 DEBUG 06:45:08 DEBUG "value": [ 06:45:08 DEBUG 06:45:08 DEBUG 1713591908, 06:45:08 DEBUG 06:45:08 DEBUG "480586.5" 06:45:08 DEBUG "pod": "lodemon-7cb5b9456d-dskfn" 06:45:08 DEBUG ] 06:45:08 DEBUG }, 06:45:08 DEBUG } 06:45:08 DEBUG "value": [ 06:45:08 DEBUG ] 06:45:08 DEBUG 1713591908, 06:45:08 DEBUG } 06:45:08 DEBUG "0" 06:45:08 DEBUG } 06:45:08 DEBUG ] 06:45:08 DEBUG } 06:45:08 DEBUG ] 06:45:08 DEBUG } 06:45:08 DEBUG } 06:45:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591908" 06:45:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591908" 06:45:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713591908" 06:45:08 INFO [http_cmd]: http status code OK 06:45:08 DEBUG --- status code --- 06:45:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:45:08 DEBUG --- http response --- 06:45:08 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 06:45:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591908" 06:45:08 INFO [http_cmd]: http status code OK 06:45:08 DEBUG --- status code --- 06:45:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:45:08 DEBUG --- http response --- 06:45:08 INFO [http_cmd]: http status code OK 06:45:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:45:08 DEBUG --- status code --- 06:45:08 DEBUG ----- output ----- 06:45:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:45:08 DEBUG { 06:45:08 DEBUG --- http response --- 06:45:08 DEBUG "status": "success", 06:45:08 DEBUG "data": { 06:45:08 DEBUG "resultType": "vector", 06:45:08 DEBUG "result": [ 06:45:08 DEBUG { 06:45:08 DEBUG "metric": { 06:45:08 DEBUG "pod": "ds-idrepo-0" 06:45:08 DEBUG }, 06:45:08 DEBUG "value": [ 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG [...] 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG "pod": "ds-idrepo-2" 06:45:08 DEBUG }, 06:45:08 DEBUG "value": [ 06:45:08 DEBUG 1713591908, 06:45:08 DEBUG "0" 06:45:08 DEBUG ] 06:45:08 DEBUG } 06:45:08 DEBUG ] 06:45:08 DEBUG } 06:45:08 DEBUG } 06:45:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:45:08 DEBUG ----- output ----- 06:45:08 DEBUG { 06:45:08 DEBUG "status": "success", 06:45:08 DEBUG "data": { 06:45:08 DEBUG "resultType": "vector", 06:45:08 DEBUG "result": [ 06:45:08 DEBUG { 06:45:08 DEBUG "metric": { 06:45:08 DEBUG "instance": "10.142.0.107:9100" 06:45:08 DEBUG }, 06:45:08 DEBUG "value": [ 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG [...] 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG "instance": "10.142.0.43:9100" 06:45:08 DEBUG }, 06:45:08 DEBUG "value": [ 06:45:08 DEBUG 1713591908, 06:45:08 DEBUG "3200136469504" 06:45:08 DEBUG ] 06:45:08 DEBUG } 06:45:08 DEBUG ] 06:45:08 DEBUG } 06:45:08 DEBUG } 06:45:08 INFO [http_cmd]: http status code OK 06:45:08 DEBUG --- status code --- 06:45:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:45:08 DEBUG --- http response --- 06:45:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 06:45:08 DEBUG ----- output ----- 06:45:08 DEBUG { 06:45:08 DEBUG "status": "success", 06:45:08 DEBUG "data": { 06:45:08 DEBUG "resultType": "vector", 06:45:08 DEBUG "result": [ 06:45:08 DEBUG { 06:45:08 DEBUG "metric": { 06:45:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:45:08 DEBUG }, 06:45:08 DEBUG "value": [ 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG [...] 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG "pod": "lodemon-7cb5b9456d-dskfn" 06:45:08 DEBUG }, 06:45:08 DEBUG "value": [ 06:45:08 DEBUG 1713591908, 06:45:08 DEBUG "0" 06:45:08 DEBUG ] 06:45:08 DEBUG } 06:45:08 DEBUG ] 06:45:08 DEBUG } 06:45:08 DEBUG } 06:45:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713591908" 06:45:08 ERROR [http_cmd]: ERROR 06:45:08 ERROR --- status code --- 06:45:08 ERROR http status code is 400 (expected [200, 401, 403, 404]) 06:45:08 ERROR --- http response --- 06:45:08 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 06:45:08 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 06:45:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713591908" 06:45:08 INFO [http_cmd]: http status code OK 06:45:08 DEBUG --- status code --- 06:45:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:45:08 DEBUG --- http response --- 06:45:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:45:08 DEBUG ----- output ----- 06:45:08 DEBUG { 06:45:08 DEBUG "status": "success", 06:45:08 DEBUG "data": { 06:45:08 DEBUG "resultType": "vector", 06:45:08 DEBUG "result": [ 06:45:08 DEBUG { 06:45:08 DEBUG "metric": { 06:45:08 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 06:45:08 DEBUG }, 06:45:08 DEBUG "value": [ 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG [...] 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 06:45:08 DEBUG }, 06:45:08 DEBUG "value": [ 06:45:08 DEBUG 1713591908, 06:45:08 DEBUG "4973060096" 06:45:08 DEBUG ] 06:45:08 DEBUG } 06:45:08 DEBUG ] 06:45:08 DEBUG } 06:45:08 DEBUG } 06:45:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591908" 06:45:08 INFO [http_cmd]: http status code OK 06:45:08 DEBUG --- status code --- 06:45:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:45:08 DEBUG --- http response --- 06:45:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:45:08 DEBUG ----- output ----- 06:45:08 DEBUG { 06:45:08 DEBUG "status": "success", 06:45:08 DEBUG "data": { 06:45:08 DEBUG "resultType": "vector", 06:45:08 DEBUG "result": [ 06:45:08 DEBUG { 06:45:08 DEBUG "metric": { 06:45:08 DEBUG "pod": "ds-idrepo-0" 06:45:08 DEBUG }, 06:45:08 DEBUG "value": [ 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG [...] 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG "pod": "ds-idrepo-2" 06:45:08 DEBUG }, 06:45:08 DEBUG "value": [ 06:45:08 DEBUG 1713591908, 06:45:08 DEBUG "0" 06:45:08 DEBUG ] 06:45:08 DEBUG } 06:45:08 DEBUG ] 06:45:08 DEBUG } 06:45:08 DEBUG } 06:45:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591908" 06:45:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591908" 06:45:08 INFO [http_cmd]: http status code OK 06:45:08 DEBUG --- status code --- 06:45:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:45:08 DEBUG --- http response --- 06:45:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:45:08 DEBUG ----- output ----- 06:45:08 DEBUG { 06:45:08 DEBUG "status": "success", 06:45:08 DEBUG "data": { 06:45:08 DEBUG "resultType": "vector", 06:45:08 DEBUG "result": [ 06:45:08 DEBUG { 06:45:08 DEBUG "metric": { 06:45:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:45:08 DEBUG }, 06:45:08 DEBUG "value": [ 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG [...] 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:45:08 DEBUG }, 06:45:08 DEBUG "value": [ 06:45:08 DEBUG 1713591908, 06:45:08 DEBUG "2667.733333333333" 06:45:08 DEBUG ] 06:45:08 DEBUG } 06:45:08 DEBUG ] 06:45:08 DEBUG } 06:45:08 DEBUG } 06:45:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591908" 06:45:08 INFO [http_cmd]: http status code OK 06:45:08 DEBUG --- status code --- 06:45:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:45:08 DEBUG --- http response --- 06:45:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:45:08 DEBUG ----- output ----- 06:45:08 DEBUG { 06:45:08 DEBUG "status": "success", 06:45:08 DEBUG "data": { 06:45:08 DEBUG "resultType": "vector", 06:45:08 DEBUG "result": [ 06:45:08 DEBUG { 06:45:08 DEBUG "metric": { 06:45:08 DEBUG "pod": "ds-cts-0" 06:45:08 DEBUG }, 06:45:08 DEBUG "value": [ 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG [...] 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG "pod": "ds-cts-2" 06:45:08 DEBUG }, 06:45:08 DEBUG "value": [ 06:45:08 DEBUG 1713591908, 06:45:08 DEBUG "0" 06:45:08 DEBUG ] 06:45:08 DEBUG } 06:45:08 DEBUG ] 06:45:08 DEBUG } 06:45:08 DEBUG } 06:45:08 INFO [http_cmd]: http status code OK 06:45:08 DEBUG --- status code --- 06:45:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:45:08 DEBUG --- http response --- 06:45:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:45:08 DEBUG ----- output ----- 06:45:08 DEBUG { 06:45:08 DEBUG "status": "success", 06:45:08 DEBUG "data": { 06:45:08 DEBUG "resultType": "vector", 06:45:08 DEBUG "result": [ 06:45:08 DEBUG { 06:45:08 DEBUG "metric": { 06:45:08 DEBUG "pod": "ds-cts-0" 06:45:08 DEBUG }, 06:45:08 DEBUG "value": [ 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG [...] 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG "pod": "ds-cts-2" 06:45:08 DEBUG }, 06:45:08 DEBUG "value": [ 06:45:08 DEBUG 1713591908, 06:45:08 DEBUG "0" 06:45:08 DEBUG ] 06:45:08 DEBUG } 06:45:08 DEBUG ] 06:45:08 DEBUG } 06:45:08 DEBUG } 06:45:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591908" 06:45:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713591908" 06:45:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713591908" 06:45:08 INFO [http_cmd]: http status code OK 06:45:08 DEBUG --- status code --- 06:45:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:45:08 DEBUG --- http response --- 06:45:08 INFO [http_cmd]: http status code OK 06:45:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:45:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591908" 06:45:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591908" 06:45:08 DEBUG --- status code --- 06:45:08 DEBUG ----- output ----- 06:45:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:45:08 DEBUG { 06:45:08 DEBUG --- http response --- 06:45:08 DEBUG "status": "success", 06:45:08 DEBUG "data": { 06:45:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:45:08 DEBUG "resultType": "vector", 06:45:08 DEBUG ----- output ----- 06:45:08 DEBUG "result": [ 06:45:08 DEBUG { 06:45:08 DEBUG { 06:45:08 DEBUG "status": "success", 06:45:08 DEBUG "metric": { 06:45:08 DEBUG "data": { 06:45:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:45:08 DEBUG "resultType": "vector", 06:45:08 DEBUG }, 06:45:08 DEBUG "result": [ 06:45:08 DEBUG "value": [ 06:45:08 DEBUG { 06:45:08 DEBUG 06:45:08 DEBUG "metric": { 06:45:08 DEBUG 06:45:08 DEBUG "instance": "10.142.0.107:9100" 06:45:08 DEBUG 06:45:08 DEBUG }, 06:45:08 DEBUG [...] 06:45:08 DEBUG "value": [ 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:45:08 DEBUG [...] 06:45:08 DEBUG }, 06:45:08 DEBUG 06:45:08 DEBUG "value": [ 06:45:08 DEBUG 06:45:08 DEBUG 1713591908, 06:45:08 DEBUG 06:45:08 DEBUG "0" 06:45:08 DEBUG "instance": "10.142.0.43:9100" 06:45:08 DEBUG ] 06:45:08 DEBUG }, 06:45:08 DEBUG } 06:45:08 DEBUG "value": [ 06:45:08 DEBUG ] 06:45:08 DEBUG 1713591908, 06:45:08 DEBUG } 06:45:08 DEBUG "0.2220666666666214" 06:45:08 DEBUG } 06:45:08 DEBUG ] 06:45:08 DEBUG } 06:45:08 DEBUG ] 06:45:08 DEBUG } 06:45:08 DEBUG } 06:45:08 INFO [http_cmd]: http status code OK 06:45:08 DEBUG --- status code --- 06:45:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:45:08 DEBUG --- http response --- 06:45:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591908" 06:45:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:45:08 DEBUG ----- output ----- 06:45:08 DEBUG { 06:45:08 DEBUG "status": "success", 06:45:08 DEBUG "data": { 06:45:08 DEBUG "resultType": "vector", 06:45:08 DEBUG "result": [ 06:45:08 DEBUG { 06:45:08 DEBUG "metric": { 06:45:08 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 06:45:08 DEBUG }, 06:45:08 DEBUG "value": [ 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG [...] 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 06:45:08 DEBUG }, 06:45:08 DEBUG "value": [ 06:45:08 DEBUG 1713591908, 06:45:08 DEBUG "8.047861623669554" 06:45:08 DEBUG ] 06:45:08 DEBUG } 06:45:08 DEBUG ] 06:45:08 DEBUG } 06:45:08 DEBUG } 06:45:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713591908" 06:45:08 INFO [http_cmd]: http status code OK 06:45:08 INFO [http_cmd]: http status code OK 06:45:08 DEBUG --- status code --- 06:45:08 DEBUG --- status code --- 06:45:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:45:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:45:08 DEBUG --- http response --- 06:45:08 DEBUG --- http response --- 06:45:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:45:08 DEBUG ----- output ----- 06:45:08 DEBUG { 06:45:08 DEBUG "status": "success", 06:45:08 DEBUG "data": { 06:45:08 DEBUG "resultType": "vector", 06:45:08 DEBUG "result": [ 06:45:08 DEBUG { 06:45:08 DEBUG "metric": { 06:45:08 DEBUG "pod": "ds-idrepo-0" 06:45:08 DEBUG }, 06:45:08 DEBUG "value": [ 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG [...] 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG "pod": "ds-idrepo-2" 06:45:08 DEBUG }, 06:45:08 DEBUG "value": [ 06:45:08 DEBUG 1713591908, 06:45:08 DEBUG "0" 06:45:08 DEBUG ] 06:45:08 DEBUG } 06:45:08 DEBUG ] 06:45:08 DEBUG } 06:45:08 DEBUG } 06:45:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:45:08 DEBUG ----- output ----- 06:45:08 DEBUG { 06:45:08 DEBUG "status": "success", 06:45:08 DEBUG "data": { 06:45:08 DEBUG "resultType": "vector", 06:45:08 DEBUG "result": [ 06:45:08 DEBUG { 06:45:08 DEBUG "metric": { 06:45:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:45:08 DEBUG }, 06:45:08 DEBUG "value": [ 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG [...] 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:45:08 DEBUG }, 06:45:08 DEBUG "value": [ 06:45:08 DEBUG 1713591908, 06:45:08 DEBUG "2667.4333333333334" 06:45:08 DEBUG ] 06:45:08 DEBUG } 06:45:08 DEBUG ] 06:45:08 DEBUG } 06:45:08 DEBUG } 06:45:08 INFO [http_cmd]: http status code OK 06:45:08 INFO [http_cmd]: http status code OK 06:45:08 DEBUG --- status code --- 06:45:08 DEBUG --- status code --- 06:45:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:45:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:45:08 DEBUG --- http response --- 06:45:08 DEBUG --- http response --- 06:45:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:45:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591908" 06:45:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:45:08 DEBUG ----- output ----- 06:45:08 DEBUG ----- output ----- 06:45:08 DEBUG { 06:45:08 DEBUG { 06:45:08 DEBUG "status": "success", 06:45:08 DEBUG "status": "success", 06:45:08 DEBUG "data": { 06:45:08 DEBUG "data": { 06:45:08 DEBUG "resultType": "vector", 06:45:08 DEBUG "resultType": "vector", 06:45:08 DEBUG "result": [ 06:45:08 DEBUG "result": [ 06:45:08 DEBUG { 06:45:08 DEBUG { 06:45:08 DEBUG "metric": { 06:45:08 DEBUG "metric": { 06:45:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:45:08 DEBUG "pod": "ds-idrepo-0" 06:45:08 DEBUG }, 06:45:08 DEBUG }, 06:45:08 DEBUG "value": [ 06:45:08 DEBUG "value": [ 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG [...] 06:45:08 DEBUG [...] 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG "pod": "ds-cts-2" 06:45:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:45:08 DEBUG }, 06:45:08 DEBUG }, 06:45:08 DEBUG "value": [ 06:45:08 DEBUG "value": [ 06:45:08 DEBUG 1713591908, 06:45:08 DEBUG 1713591908, 06:45:08 DEBUG "0" 06:45:08 DEBUG "2667.4" 06:45:08 DEBUG ] 06:45:08 DEBUG ] 06:45:08 DEBUG } 06:45:08 DEBUG } 06:45:08 DEBUG ] 06:45:08 DEBUG ] 06:45:08 DEBUG } 06:45:08 DEBUG } 06:45:08 DEBUG } 06:45:08 DEBUG } 06:45:08 INFO [http_cmd]: http status code OK 06:45:08 DEBUG --- status code --- 06:45:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:45:08 DEBUG --- http response --- 06:45:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:45:08 DEBUG ----- output ----- 06:45:08 DEBUG { 06:45:08 DEBUG "status": "success", 06:45:08 DEBUG "data": { 06:45:08 DEBUG "resultType": "vector", 06:45:08 DEBUG "result": [ 06:45:08 DEBUG { 06:45:08 DEBUG "metric": { 06:45:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:45:08 DEBUG }, 06:45:08 DEBUG "value": [ 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG [...] 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:45:08 DEBUG }, 06:45:08 DEBUG "value": [ 06:45:08 DEBUG 1713591908, 06:45:08 DEBUG "2667.4" 06:45:08 DEBUG ] 06:45:08 DEBUG } 06:45:08 DEBUG ] 06:45:08 DEBUG } 06:45:08 DEBUG } 06:45:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713591908" 06:45:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591908" 06:45:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591908" 06:45:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713591908" 06:45:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713591908" 06:45:08 INFO [http_cmd]: http status code OK 06:45:08 DEBUG --- status code --- 06:45:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:45:08 DEBUG --- http response --- 06:45:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:45:08 DEBUG ----- output ----- 06:45:08 DEBUG { 06:45:08 DEBUG "status": "success", 06:45:08 DEBUG "data": { 06:45:08 DEBUG "resultType": "vector", 06:45:08 DEBUG "result": [ 06:45:08 DEBUG { 06:45:08 DEBUG "metric": { 06:45:08 DEBUG "pod": "ds-cts-0" 06:45:08 DEBUG }, 06:45:08 DEBUG "value": [ 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG [...] 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG "pod": "ds-cts-2" 06:45:08 DEBUG }, 06:45:08 INFO [http_cmd]: http status code OK 06:45:08 DEBUG "value": [ 06:45:08 INFO [http_cmd]: http status code OK 06:45:08 DEBUG --- status code --- 06:45:08 DEBUG 1713591908, 06:45:08 DEBUG --- status code --- 06:45:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:45:08 DEBUG "0.00046478222222222213" 06:45:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:45:08 DEBUG --- http response --- 06:45:08 DEBUG ] 06:45:08 DEBUG --- http response --- 06:45:08 DEBUG } 06:45:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 06:45:08 DEBUG ] 06:45:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:45:08 DEBUG ----- output ----- 06:45:08 DEBUG } 06:45:08 DEBUG ----- output ----- 06:45:08 DEBUG { 06:45:08 DEBUG } 06:45:08 DEBUG { 06:45:08 DEBUG "status": "success", 06:45:08 DEBUG "status": "success", 06:45:08 DEBUG "data": { 06:45:08 DEBUG "data": { 06:45:08 DEBUG "resultType": "vector", 06:45:08 DEBUG "resultType": "vector", 06:45:08 DEBUG "result": [ 06:45:08 DEBUG "result": [ 06:45:08 DEBUG { 06:45:08 DEBUG { 06:45:08 DEBUG "metric": { 06:45:08 DEBUG "metric": { 06:45:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:45:08 DEBUG "instance": "10.142.0.107:9100" 06:45:08 DEBUG }, 06:45:08 DEBUG }, 06:45:08 DEBUG "value": [ 06:45:08 DEBUG "value": [ 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG [...] 06:45:08 DEBUG [...] 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG "pod": "idm-b85f9d978-wnhxb" 06:45:08 DEBUG "instance": "10.142.0.43:9100" 06:45:08 DEBUG }, 06:45:08 DEBUG }, 06:45:08 DEBUG "value": [ 06:45:08 DEBUG "value": [ 06:45:08 DEBUG 1713591908, 06:45:08 DEBUG 1713591908, 06:45:08 DEBUG "0" 06:45:08 DEBUG "80732486.83333333" 06:45:08 DEBUG ] 06:45:08 DEBUG ] 06:45:08 DEBUG } 06:45:08 DEBUG } 06:45:08 DEBUG ] 06:45:08 DEBUG ] 06:45:08 DEBUG } 06:45:08 DEBUG } 06:45:08 DEBUG } 06:45:08 DEBUG } 06:45:08 INFO [http_cmd]: http status code OK 06:45:08 DEBUG --- status code --- 06:45:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:45:08 DEBUG --- http response --- 06:45:08 INFO [http_cmd]: http status code OK 06:45:08 DEBUG --- status code --- 06:45:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:45:08 DEBUG --- http response --- 06:45:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:45:08 DEBUG ----- output ----- 06:45:08 DEBUG { 06:45:08 DEBUG "status": "success", 06:45:08 DEBUG "data": { 06:45:08 DEBUG "resultType": "vector", 06:45:08 DEBUG "result": [ 06:45:08 DEBUG { 06:45:08 DEBUG "metric": { 06:45:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:45:08 DEBUG }, 06:45:08 DEBUG "value": [ 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG [...] 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:45:08 DEBUG }, 06:45:08 DEBUG "value": [ 06:45:08 DEBUG 1713591908, 06:45:08 DEBUG "0.096" 06:45:08 DEBUG ] 06:45:08 DEBUG } 06:45:08 DEBUG ] 06:45:08 DEBUG } 06:45:08 DEBUG } 06:45:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:45:08 DEBUG ----- output ----- 06:45:08 DEBUG { 06:45:08 DEBUG "status": "success", 06:45:08 DEBUG "data": { 06:45:08 DEBUG "resultType": "vector", 06:45:08 DEBUG "result": [ 06:45:08 DEBUG { 06:45:08 DEBUG "metric": { 06:45:08 DEBUG "instance": "10.142.0.107:9100" 06:45:08 DEBUG }, 06:45:08 DEBUG "value": [ 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG [...] 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG "instance": "10.142.0.43:9100" 06:45:08 DEBUG }, 06:45:08 DEBUG "value": [ 06:45:08 DEBUG 1713591908, 06:45:08 DEBUG "80736657.33333333" 06:45:08 DEBUG ] 06:45:08 DEBUG } 06:45:08 DEBUG ] 06:45:08 DEBUG } 06:45:08 DEBUG } 06:45:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591908" 06:45:08 INFO [http_cmd]: http status code OK 06:45:08 DEBUG --- status code --- 06:45:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:45:08 DEBUG --- http response --- 06:45:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 06:45:08 DEBUG ----- output ----- 06:45:08 DEBUG { 06:45:08 DEBUG "status": "success", 06:45:08 DEBUG "data": { 06:45:08 DEBUG "resultType": "vector", 06:45:08 DEBUG "result": [ 06:45:08 DEBUG { 06:45:08 DEBUG "metric": { 06:45:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:45:08 DEBUG }, 06:45:08 DEBUG "value": [ 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG [...] 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG "pod": "idm-b85f9d978-wnhxb" 06:45:08 DEBUG }, 06:45:08 DEBUG "value": [ 06:45:08 DEBUG 1713591908, 06:45:08 DEBUG "0" 06:45:08 DEBUG ] 06:45:08 DEBUG } 06:45:08 DEBUG ] 06:45:08 DEBUG } 06:45:08 DEBUG } 06:45:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713591908" 06:45:08 INFO [http_cmd]: http status code OK 06:45:08 DEBUG --- status code --- 06:45:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:45:08 DEBUG --- http response --- 06:45:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:45:08 DEBUG ----- output ----- 06:45:08 DEBUG { 06:45:08 DEBUG "status": "success", 06:45:08 DEBUG "data": { 06:45:08 DEBUG "resultType": "vector", 06:45:08 DEBUG "result": [ 06:45:08 DEBUG { 06:45:08 DEBUG "metric": { 06:45:08 DEBUG "instance": "10.142.0.107:9100" 06:45:08 DEBUG }, 06:45:08 DEBUG "value": [ 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG [...] 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG 06:45:08 DEBUG "instance": "10.142.0.43:9100" 06:45:08 DEBUG }, 06:45:08 DEBUG "value": [ 06:45:08 DEBUG 1713591908, 06:45:08 DEBUG "0.08933333333332409" 06:45:08 DEBUG ] 06:45:08 DEBUG } 06:45:08 DEBUG ] 06:45:08 DEBUG } 06:45:08 DEBUG } 06:45:08 INFO 06:45:08 INFO [loop_until]: kubectl --namespace=xlou top pods 06:45:08 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:45:08 INFO [loop_until]: OK (rc = 0) 06:45:08 DEBUG --- stdout --- 06:45:08 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 17546m 8647Mi am-767f6b67b5-qcqz2 17049m 8705Mi am-767f6b67b5-vbltz 18827m 8652Mi ds-cts-0 19229m 10384Mi ds-cts-1 16987m 10661Mi ds-cts-2 19127m 10119Mi ds-idrepo-0 7803m 3745Mi ds-idrepo-1 7462m 3801Mi ds-idrepo-2 8165m 3684Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 3m 1177Mi idm-b85f9d978-wnhxb 3m 1236Mi lodemon-7cb5b9456d-dskfn 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7905m 4652Mi 06:45:08 DEBUG --- stderr --- 06:45:08 DEBUG 06:45:08 INFO 06:45:08 INFO [loop_until]: kubectl --namespace=xlou top node 06:45:08 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:45:08 INFO [loop_until]: OK (rc = 0) 06:45:08 DEBUG --- stdout --- 06:45:08 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 72m 0% 6277Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 8005m 26% 7421Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 17243m 57% 11367Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 80m 0% 3377Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 18952m 63% 13516Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 146m 0% 5203Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 17587m 58% 11589Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 18509m 61% 3029Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 19531m 65% 10867Mi 9% gke-xlou-cdm-ds-0f601751-j488 8149m 27% 7005Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 17128m 57% 15587Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 8082m 27% 7703Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 19525m 65% 12736Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 8162m 51% 8492Mi 14% 06:45:08 DEBUG --- stderr --- 06:45:08 DEBUG 06:46:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713591968" 06:46:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591968" 06:46:08 INFO [http_cmd]: http status code OK 06:46:08 DEBUG --- status code --- 06:46:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:46:08 DEBUG --- http response --- 06:46:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:46:08 DEBUG ----- output ----- 06:46:08 DEBUG { 06:46:08 DEBUG "status": "success", 06:46:08 DEBUG "data": { 06:46:08 DEBUG "resultType": "vector", 06:46:08 DEBUG "result": [ 06:46:08 DEBUG { 06:46:08 DEBUG "metric": { 06:46:08 DEBUG "instance": "10.142.0.107:9100" 06:46:08 DEBUG }, 06:46:08 DEBUG "value": [ 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG [...] 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG "instance": "10.142.0.43:9100" 06:46:08 DEBUG }, 06:46:08 DEBUG "value": [ 06:46:08 DEBUG 1713591968, 06:46:08 DEBUG "3204748491776" 06:46:08 DEBUG ] 06:46:08 DEBUG } 06:46:08 DEBUG ] 06:46:08 DEBUG } 06:46:08 DEBUG } 06:46:08 INFO [http_cmd]: http status code OK 06:46:08 DEBUG --- status code --- 06:46:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:46:08 DEBUG --- http response --- 06:46:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (107 lines): 06:46:08 DEBUG ----- output ----- 06:46:08 DEBUG { 06:46:08 DEBUG "status": "success", 06:46:08 DEBUG "data": { 06:46:08 DEBUG "resultType": "vector", 06:46:08 DEBUG "result": [ 06:46:08 DEBUG { 06:46:08 DEBUG "metric": { 06:46:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:46:08 DEBUG }, 06:46:08 DEBUG "value": [ 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG [...] 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG "pod": "lodemon-7cb5b9456d-dskfn" 06:46:08 DEBUG }, 06:46:08 DEBUG "value": [ 06:46:08 DEBUG 1713591968, 06:46:08 DEBUG "4494.349145971251" 06:46:08 DEBUG ] 06:46:08 DEBUG } 06:46:08 DEBUG ] 06:46:08 DEBUG } 06:46:08 DEBUG } 06:46:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591968" 06:46:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713591968" 06:46:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591968" 06:46:08 INFO [http_cmd]: http status code OK 06:46:08 DEBUG --- status code --- 06:46:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:46:08 DEBUG --- http response --- 06:46:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:46:08 DEBUG ----- output ----- 06:46:08 DEBUG { 06:46:08 DEBUG "status": "success", 06:46:08 DEBUG "data": { 06:46:08 DEBUG "resultType": "vector", 06:46:08 DEBUG "result": [ 06:46:08 DEBUG { 06:46:08 DEBUG "metric": { 06:46:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:46:08 DEBUG }, 06:46:08 DEBUG "value": [ 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG [...] 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:46:08 DEBUG }, 06:46:08 DEBUG "value": [ 06:46:08 DEBUG 1713591968, 06:46:08 DEBUG "480188.5" 06:46:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591968" 06:46:08 DEBUG ] 06:46:08 DEBUG } 06:46:08 DEBUG ] 06:46:08 DEBUG } 06:46:08 DEBUG } 06:46:08 ERROR [http_cmd]: ERROR 06:46:08 ERROR --- status code --- 06:46:08 ERROR http status code is 400 (expected [200, 401, 403, 404]) 06:46:08 ERROR --- http response --- 06:46:08 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 06:46:08 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 06:46:08 INFO [http_cmd]: http status code OK 06:46:08 DEBUG --- status code --- 06:46:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:46:08 DEBUG --- http response --- 06:46:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (107 lines): 06:46:08 DEBUG ----- output ----- 06:46:08 DEBUG { 06:46:08 DEBUG "status": "success", 06:46:08 DEBUG "data": { 06:46:08 DEBUG "resultType": "vector", 06:46:08 DEBUG "result": [ 06:46:08 DEBUG { 06:46:08 DEBUG "metric": { 06:46:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:46:08 DEBUG }, 06:46:08 DEBUG "value": [ 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG [...] 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG "pod": "lodemon-7cb5b9456d-dskfn" 06:46:08 DEBUG }, 06:46:08 DEBUG "value": [ 06:46:08 DEBUG 1713591968, 06:46:08 DEBUG "0" 06:46:08 DEBUG ] 06:46:08 DEBUG } 06:46:08 DEBUG ] 06:46:08 DEBUG } 06:46:08 DEBUG } 06:46:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591968" 06:46:08 INFO [http_cmd]: http status code OK 06:46:08 DEBUG --- status code --- 06:46:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:46:08 DEBUG --- http response --- 06:46:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:46:08 DEBUG ----- output ----- 06:46:08 DEBUG { 06:46:08 DEBUG "status": "success", 06:46:08 DEBUG "data": { 06:46:08 DEBUG "resultType": "vector", 06:46:08 DEBUG "result": [ 06:46:08 DEBUG { 06:46:08 DEBUG "metric": { 06:46:08 DEBUG "pod": "ds-idrepo-0" 06:46:08 DEBUG }, 06:46:08 DEBUG "value": [ 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG [...] 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG "pod": "ds-idrepo-2" 06:46:08 DEBUG }, 06:46:08 DEBUG "value": [ 06:46:08 DEBUG 1713591968, 06:46:08 DEBUG "0" 06:46:08 DEBUG ] 06:46:08 DEBUG } 06:46:08 DEBUG ] 06:46:08 DEBUG } 06:46:08 DEBUG } 06:46:08 INFO [http_cmd]: http status code OK 06:46:08 DEBUG --- status code --- 06:46:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:46:08 DEBUG --- http response --- 06:46:08 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 06:46:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713591968" 06:46:08 INFO [http_cmd]: http status code OK 06:46:08 DEBUG --- status code --- 06:46:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:46:08 DEBUG --- http response --- 06:46:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:46:08 DEBUG ----- output ----- 06:46:08 DEBUG { 06:46:08 DEBUG "status": "success", 06:46:08 DEBUG "data": { 06:46:08 DEBUG "resultType": "vector", 06:46:08 DEBUG "result": [ 06:46:08 DEBUG { 06:46:08 DEBUG "metric": { 06:46:08 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 06:46:08 DEBUG }, 06:46:08 DEBUG "value": [ 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG [...] 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 06:46:08 DEBUG }, 06:46:08 DEBUG "value": [ 06:46:08 DEBUG 1713591968, 06:46:08 DEBUG "4976467968" 06:46:08 DEBUG ] 06:46:08 DEBUG } 06:46:08 DEBUG ] 06:46:08 DEBUG } 06:46:08 DEBUG } 06:46:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591968" 06:46:08 INFO [http_cmd]: http status code OK 06:46:08 DEBUG --- status code --- 06:46:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:46:08 DEBUG --- http response --- 06:46:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:46:08 DEBUG ----- output ----- 06:46:08 DEBUG { 06:46:08 DEBUG "status": "success", 06:46:08 DEBUG "data": { 06:46:08 DEBUG "resultType": "vector", 06:46:08 DEBUG "result": [ 06:46:08 DEBUG { 06:46:08 DEBUG "metric": { 06:46:08 DEBUG "pod": "ds-cts-0" 06:46:08 DEBUG }, 06:46:08 DEBUG "value": [ 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG [...] 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG "pod": "ds-cts-2" 06:46:08 DEBUG }, 06:46:08 DEBUG "value": [ 06:46:08 DEBUG 1713591968, 06:46:08 DEBUG "0" 06:46:08 DEBUG ] 06:46:08 DEBUG } 06:46:08 DEBUG ] 06:46:08 DEBUG } 06:46:08 DEBUG } 06:46:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591968" 06:46:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591968" 06:46:08 INFO [http_cmd]: http status code OK 06:46:08 DEBUG --- status code --- 06:46:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:46:08 DEBUG --- http response --- 06:46:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:46:08 DEBUG ----- output ----- 06:46:08 DEBUG { 06:46:08 DEBUG "status": "success", 06:46:08 DEBUG "data": { 06:46:08 DEBUG "resultType": "vector", 06:46:08 DEBUG "result": [ 06:46:08 DEBUG { 06:46:08 DEBUG "metric": { 06:46:08 DEBUG "pod": "ds-idrepo-0" 06:46:08 DEBUG }, 06:46:08 DEBUG "value": [ 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG [...] 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG "pod": "ds-idrepo-2" 06:46:08 DEBUG }, 06:46:08 DEBUG "value": [ 06:46:08 DEBUG 1713591968, 06:46:08 DEBUG "0" 06:46:08 DEBUG ] 06:46:08 DEBUG } 06:46:08 DEBUG ] 06:46:08 DEBUG } 06:46:08 DEBUG } 06:46:08 INFO [http_cmd]: http status code OK 06:46:08 DEBUG --- status code --- 06:46:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:46:08 DEBUG --- http response --- 06:46:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:46:08 DEBUG ----- output ----- 06:46:08 DEBUG { 06:46:08 DEBUG "status": "success", 06:46:08 DEBUG "data": { 06:46:08 DEBUG "resultType": "vector", 06:46:08 DEBUG "result": [ 06:46:08 DEBUG { 06:46:08 DEBUG "metric": { 06:46:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:46:08 DEBUG }, 06:46:08 DEBUG "value": [ 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG [...] 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:46:08 DEBUG }, 06:46:08 DEBUG "value": [ 06:46:08 DEBUG 1713591968, 06:46:08 DEBUG "2657.2" 06:46:08 DEBUG ] 06:46:08 DEBUG } 06:46:08 DEBUG ] 06:46:08 DEBUG } 06:46:08 DEBUG } 06:46:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591968" 06:46:08 INFO [http_cmd]: http status code OK 06:46:08 DEBUG --- status code --- 06:46:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:46:08 DEBUG --- http response --- 06:46:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:46:08 DEBUG ----- output ----- 06:46:08 DEBUG { 06:46:08 DEBUG "status": "success", 06:46:08 DEBUG "data": { 06:46:08 DEBUG "resultType": "vector", 06:46:08 DEBUG "result": [ 06:46:08 DEBUG { 06:46:08 DEBUG "metric": { 06:46:08 DEBUG "pod": "ds-cts-0" 06:46:08 DEBUG }, 06:46:08 DEBUG "value": [ 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG [...] 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG "pod": "ds-cts-2" 06:46:08 DEBUG }, 06:46:08 DEBUG "value": [ 06:46:08 DEBUG 1713591968, 06:46:08 DEBUG "0" 06:46:08 DEBUG ] 06:46:08 DEBUG } 06:46:08 DEBUG ] 06:46:08 DEBUG } 06:46:08 DEBUG } 06:46:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591968" 06:46:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591968" 06:46:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591968" 06:46:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713591968" 06:46:08 INFO [http_cmd]: http status code OK 06:46:08 DEBUG --- status code --- 06:46:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:46:08 DEBUG --- http response --- 06:46:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:46:08 DEBUG ----- output ----- 06:46:08 DEBUG { 06:46:08 DEBUG "status": "success", 06:46:08 DEBUG "data": { 06:46:08 DEBUG "resultType": "vector", 06:46:08 DEBUG "result": [ 06:46:08 DEBUG { 06:46:08 DEBUG "metric": { 06:46:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:46:08 DEBUG }, 06:46:08 DEBUG "value": [ 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG [...] 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:46:08 DEBUG }, 06:46:08 DEBUG "value": [ 06:46:08 DEBUG 1713591968, 06:46:08 DEBUG "0" 06:46:08 DEBUG ] 06:46:08 DEBUG } 06:46:08 DEBUG ] 06:46:08 DEBUG } 06:46:08 DEBUG } 06:46:08 INFO [http_cmd]: http status code OK 06:46:08 DEBUG --- status code --- 06:46:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:46:08 DEBUG --- http response --- 06:46:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:46:08 DEBUG ----- output ----- 06:46:08 DEBUG { 06:46:08 DEBUG "status": "success", 06:46:08 DEBUG "data": { 06:46:08 DEBUG "resultType": "vector", 06:46:08 DEBUG "result": [ 06:46:08 DEBUG { 06:46:08 DEBUG "metric": { 06:46:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:46:08 DEBUG }, 06:46:08 DEBUG "value": [ 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG [...] 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:46:08 DEBUG }, 06:46:08 DEBUG "value": [ 06:46:08 DEBUG 1713591968, 06:46:08 DEBUG "2664.0666666666666" 06:46:08 DEBUG ] 06:46:08 DEBUG } 06:46:08 DEBUG ] 06:46:08 DEBUG } 06:46:08 DEBUG } 06:46:08 INFO [http_cmd]: http status code OK 06:46:08 DEBUG --- status code --- 06:46:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591968" 06:46:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:46:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591968" 06:46:08 INFO [http_cmd]: http status code OK 06:46:08 DEBUG --- http response --- 06:46:08 DEBUG --- status code --- 06:46:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:46:08 DEBUG --- http response --- 06:46:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:46:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:46:08 DEBUG ----- output ----- 06:46:08 DEBUG ----- output ----- 06:46:08 DEBUG { 06:46:08 DEBUG { 06:46:08 DEBUG "status": "success", 06:46:08 DEBUG "status": "success", 06:46:08 DEBUG "data": { 06:46:08 DEBUG "data": { 06:46:08 DEBUG "resultType": "vector", 06:46:08 DEBUG "resultType": "vector", 06:46:08 DEBUG "result": [ 06:46:08 DEBUG "result": [ 06:46:08 DEBUG { 06:46:08 DEBUG { 06:46:08 DEBUG "metric": { 06:46:08 DEBUG "metric": { 06:46:08 DEBUG "pod": "ds-idrepo-0" 06:46:08 DEBUG "pod": "ds-idrepo-0" 06:46:08 DEBUG }, 06:46:08 DEBUG }, 06:46:08 DEBUG "value": [ 06:46:08 DEBUG "value": [ 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG [...] 06:46:08 DEBUG [...] 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG "pod": "ds-cts-2" 06:46:08 DEBUG "pod": "ds-idrepo-2" 06:46:08 DEBUG }, 06:46:08 DEBUG }, 06:46:08 DEBUG "value": [ 06:46:08 DEBUG "value": [ 06:46:08 DEBUG 1713591968, 06:46:08 DEBUG 1713591968, 06:46:08 DEBUG "0" 06:46:08 DEBUG "0" 06:46:08 DEBUG ] 06:46:08 DEBUG ] 06:46:08 DEBUG } 06:46:08 DEBUG } 06:46:08 DEBUG ] 06:46:08 DEBUG ] 06:46:08 DEBUG } 06:46:08 DEBUG } 06:46:08 DEBUG } 06:46:08 DEBUG } 06:46:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713591968" 06:46:08 INFO [http_cmd]: http status code OK 06:46:08 DEBUG --- status code --- 06:46:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:46:08 DEBUG --- http response --- 06:46:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:46:08 DEBUG ----- output ----- 06:46:08 DEBUG { 06:46:08 DEBUG "status": "success", 06:46:08 DEBUG "data": { 06:46:08 INFO [http_cmd]: http status code OK 06:46:08 DEBUG "resultType": "vector", 06:46:08 DEBUG --- status code --- 06:46:08 DEBUG "result": [ 06:46:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:46:08 DEBUG { 06:46:08 DEBUG --- http response --- 06:46:08 DEBUG "metric": { 06:46:08 DEBUG "pod": "ds-cts-0" 06:46:08 DEBUG }, 06:46:08 DEBUG "value": [ 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG [...] 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG "pod": "ds-cts-2" 06:46:08 DEBUG }, 06:46:08 DEBUG "value": [ 06:46:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:46:08 DEBUG 1713591968, 06:46:08 DEBUG ----- output ----- 06:46:08 DEBUG "0.0009884866666666668" 06:46:08 DEBUG { 06:46:08 DEBUG ] 06:46:08 DEBUG "status": "success", 06:46:08 DEBUG } 06:46:08 DEBUG "data": { 06:46:08 DEBUG ] 06:46:08 DEBUG "resultType": "vector", 06:46:08 DEBUG } 06:46:08 DEBUG "result": [ 06:46:08 DEBUG } 06:46:08 DEBUG { 06:46:08 DEBUG "metric": { 06:46:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:46:08 DEBUG }, 06:46:08 DEBUG "value": [ 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG [...] 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:46:08 DEBUG }, 06:46:08 DEBUG "value": [ 06:46:08 DEBUG 1713591968, 06:46:08 DEBUG "2667.2" 06:46:08 DEBUG ] 06:46:08 DEBUG } 06:46:08 DEBUG ] 06:46:08 DEBUG } 06:46:08 DEBUG } 06:46:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713591968" 06:46:08 INFO [http_cmd]: http status code OK 06:46:08 DEBUG --- status code --- 06:46:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:46:08 DEBUG --- http response --- 06:46:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:46:08 DEBUG ----- output ----- 06:46:08 DEBUG { 06:46:08 DEBUG "status": "success", 06:46:08 DEBUG "data": { 06:46:08 DEBUG "resultType": "vector", 06:46:08 DEBUG "result": [ 06:46:08 DEBUG { 06:46:08 DEBUG "metric": { 06:46:08 DEBUG "instance": "10.142.0.107:9100" 06:46:08 DEBUG }, 06:46:08 DEBUG "value": [ 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG [...] 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG "instance": "10.142.0.43:9100" 06:46:08 DEBUG }, 06:46:08 DEBUG "value": [ 06:46:08 DEBUG 1713591968, 06:46:08 DEBUG "0.28123333333334316" 06:46:08 DEBUG ] 06:46:08 DEBUG } 06:46:08 DEBUG ] 06:46:08 DEBUG } 06:46:08 DEBUG } 06:46:08 INFO [http_cmd]: http status code OK 06:46:08 DEBUG --- status code --- 06:46:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:46:08 DEBUG --- http response --- 06:46:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:46:08 DEBUG ----- output ----- 06:46:08 DEBUG { 06:46:08 DEBUG "status": "success", 06:46:08 DEBUG "data": { 06:46:08 DEBUG "resultType": "vector", 06:46:08 DEBUG "result": [ 06:46:08 DEBUG { 06:46:08 DEBUG "metric": { 06:46:08 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 06:46:08 DEBUG }, 06:46:08 DEBUG "value": [ 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG [...] 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 06:46:08 DEBUG }, 06:46:08 DEBUG "value": [ 06:46:08 DEBUG 1713591968, 06:46:08 DEBUG "8.034426815898577" 06:46:08 DEBUG ] 06:46:08 DEBUG } 06:46:08 DEBUG ] 06:46:08 DEBUG } 06:46:08 DEBUG } 06:46:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591968" 06:46:08 INFO [http_cmd]: http status code OK 06:46:08 DEBUG --- status code --- 06:46:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:46:08 DEBUG --- http response --- 06:46:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:46:08 DEBUG ----- output ----- 06:46:08 DEBUG { 06:46:08 DEBUG "status": "success", 06:46:08 DEBUG "data": { 06:46:08 DEBUG "resultType": "vector", 06:46:08 DEBUG "result": [ 06:46:08 DEBUG { 06:46:08 DEBUG "metric": { 06:46:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:46:08 DEBUG }, 06:46:08 DEBUG "value": [ 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG [...] 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:46:08 DEBUG }, 06:46:08 DEBUG "value": [ 06:46:08 DEBUG 1713591968, 06:46:08 DEBUG "2663.866666666667" 06:46:08 DEBUG ] 06:46:08 DEBUG } 06:46:08 DEBUG ] 06:46:08 DEBUG } 06:46:08 DEBUG } 06:46:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591968" 06:46:08 INFO [http_cmd]: http status code OK 06:46:08 DEBUG --- status code --- 06:46:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:46:08 DEBUG --- http response --- 06:46:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 06:46:08 DEBUG ----- output ----- 06:46:08 DEBUG { 06:46:08 DEBUG "status": "success", 06:46:08 DEBUG "data": { 06:46:08 DEBUG "resultType": "vector", 06:46:08 DEBUG "result": [ 06:46:08 DEBUG { 06:46:08 DEBUG "metric": { 06:46:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:46:08 DEBUG }, 06:46:08 DEBUG "value": [ 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG [...] 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG "pod": "ds-cts-2" 06:46:08 DEBUG }, 06:46:08 DEBUG "value": [ 06:46:08 DEBUG 1713591968, 06:46:08 DEBUG "0" 06:46:08 DEBUG ] 06:46:08 DEBUG } 06:46:08 DEBUG ] 06:46:08 DEBUG } 06:46:08 DEBUG } 06:46:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713591968" 06:46:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713591968" 06:46:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713591968" 06:46:08 INFO [http_cmd]: http status code OK 06:46:08 INFO [http_cmd]: http status code OK 06:46:08 DEBUG --- status code --- 06:46:08 DEBUG --- status code --- 06:46:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:46:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:46:08 DEBUG --- http response --- 06:46:08 DEBUG --- http response --- 06:46:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:46:08 DEBUG ----- output ----- 06:46:08 DEBUG { 06:46:08 DEBUG "status": "success", 06:46:08 DEBUG "data": { 06:46:08 DEBUG "resultType": "vector", 06:46:08 DEBUG "result": [ 06:46:08 DEBUG { 06:46:08 DEBUG "metric": { 06:46:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:46:08 DEBUG }, 06:46:08 DEBUG "value": [ 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG [...] 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:46:08 DEBUG }, 06:46:08 DEBUG "value": [ 06:46:08 DEBUG 1713591968, 06:46:08 DEBUG "0.096" 06:46:08 DEBUG ] 06:46:08 DEBUG } 06:46:08 DEBUG ] 06:46:08 DEBUG } 06:46:08 DEBUG } 06:46:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:46:08 DEBUG ----- output ----- 06:46:08 DEBUG { 06:46:08 DEBUG "status": "success", 06:46:08 DEBUG "data": { 06:46:08 DEBUG "resultType": "vector", 06:46:08 DEBUG "result": [ 06:46:08 DEBUG { 06:46:08 DEBUG "metric": { 06:46:08 DEBUG "instance": "10.142.0.107:9100" 06:46:08 DEBUG }, 06:46:08 DEBUG "value": [ 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG [...] 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG "instance": "10.142.0.43:9100" 06:46:08 DEBUG }, 06:46:08 DEBUG "value": [ 06:46:08 DEBUG 1713591968, 06:46:08 DEBUG "80707240.36666667" 06:46:08 DEBUG ] 06:46:08 DEBUG } 06:46:08 DEBUG ] 06:46:08 DEBUG } 06:46:08 DEBUG } 06:46:08 INFO [http_cmd]: http status code OK 06:46:08 DEBUG --- status code --- 06:46:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:46:08 DEBUG --- http response --- 06:46:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:46:08 DEBUG ----- output ----- 06:46:08 DEBUG { 06:46:08 DEBUG "status": "success", 06:46:08 DEBUG "data": { 06:46:08 DEBUG "resultType": "vector", 06:46:08 DEBUG "result": [ 06:46:08 DEBUG { 06:46:08 DEBUG "metric": { 06:46:08 DEBUG "instance": "10.142.0.107:9100" 06:46:08 DEBUG }, 06:46:08 DEBUG "value": [ 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG [...] 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG "instance": "10.142.0.43:9100" 06:46:08 DEBUG }, 06:46:08 DEBUG "value": [ 06:46:08 DEBUG 1713591968, 06:46:08 DEBUG "80703478.96666667" 06:46:08 DEBUG ] 06:46:08 DEBUG } 06:46:08 DEBUG ] 06:46:08 DEBUG } 06:46:08 DEBUG } 06:46:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713591968" 06:46:08 INFO [http_cmd]: http status code OK 06:46:08 DEBUG --- status code --- 06:46:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:46:08 DEBUG --- http response --- 06:46:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 06:46:08 DEBUG ----- output ----- 06:46:08 DEBUG { 06:46:08 DEBUG "status": "success", 06:46:08 DEBUG "data": { 06:46:08 DEBUG "resultType": "vector", 06:46:08 DEBUG "result": [ 06:46:08 DEBUG { 06:46:08 DEBUG "metric": { 06:46:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:46:08 DEBUG }, 06:46:08 DEBUG "value": [ 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG [...] 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG "pod": "ds-cts-2" 06:46:08 DEBUG }, 06:46:08 DEBUG "value": [ 06:46:08 DEBUG 1713591968, 06:46:08 DEBUG "0" 06:46:08 DEBUG ] 06:46:08 DEBUG } 06:46:08 DEBUG ] 06:46:08 DEBUG } 06:46:08 DEBUG } 06:46:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713591968" 06:46:08 INFO [http_cmd]: http status code OK 06:46:08 DEBUG --- status code --- 06:46:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:46:08 DEBUG --- http response --- 06:46:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:46:08 DEBUG ----- output ----- 06:46:08 DEBUG { 06:46:08 DEBUG "status": "success", 06:46:08 DEBUG "data": { 06:46:08 DEBUG "resultType": "vector", 06:46:08 DEBUG "result": [ 06:46:08 DEBUG { 06:46:08 DEBUG "metric": { 06:46:08 DEBUG "instance": "10.142.0.107:9100" 06:46:08 DEBUG }, 06:46:08 DEBUG "value": [ 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG [...] 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG 06:46:08 DEBUG "instance": "10.142.0.43:9100" 06:46:08 DEBUG }, 06:46:08 DEBUG "value": [ 06:46:08 DEBUG 1713591968, 06:46:08 DEBUG "0.11499999999999962" 06:46:08 DEBUG ] 06:46:08 DEBUG } 06:46:08 DEBUG ] 06:46:08 DEBUG } 06:46:08 DEBUG } 06:46:08 INFO 06:46:08 INFO [loop_until]: kubectl --namespace=xlou top pods 06:46:08 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:46:08 INFO [loop_until]: OK (rc = 0) 06:46:08 DEBUG --- stdout --- 06:46:08 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 16940m 8650Mi am-767f6b67b5-qcqz2 17708m 8711Mi am-767f6b67b5-vbltz 18568m 8652Mi ds-cts-0 18491m 10376Mi ds-cts-1 17472m 10662Mi ds-cts-2 19164m 10143Mi ds-idrepo-0 7662m 3745Mi ds-idrepo-1 7350m 3801Mi ds-idrepo-2 7886m 3687Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 3m 1177Mi idm-b85f9d978-wnhxb 4m 1236Mi lodemon-7cb5b9456d-dskfn 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 8224m 4654Mi 06:46:08 DEBUG --- stderr --- 06:46:08 DEBUG 06:46:09 INFO 06:46:09 INFO [loop_until]: kubectl --namespace=xlou top node 06:46:09 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:46:09 INFO [loop_until]: OK (rc = 0) 06:46:09 DEBUG --- stdout --- 06:46:09 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 72m 0% 6263Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 8024m 26% 7420Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 17130m 57% 11356Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 81m 0% 3380Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 18789m 62% 13518Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 150m 0% 5195Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 17420m 58% 11592Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 18998m 63% 3015Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 18697m 62% 10871Mi 9% gke-xlou-cdm-ds-0f601751-j488 8144m 27% 6986Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 17545m 58% 15592Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 8031m 26% 7701Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 19209m 64% 12763Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 8164m 51% 8493Mi 14% 06:46:09 DEBUG --- stderr --- 06:46:09 DEBUG 06:47:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592028" 06:47:08 INFO [http_cmd]: http status code OK 06:47:08 DEBUG --- status code --- 06:47:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:47:08 DEBUG --- http response --- 06:47:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:47:08 DEBUG ----- output ----- 06:47:08 DEBUG { 06:47:08 DEBUG "status": "success", 06:47:08 DEBUG "data": { 06:47:08 DEBUG "resultType": "vector", 06:47:08 DEBUG "result": [ 06:47:08 DEBUG { 06:47:08 DEBUG "metric": { 06:47:08 DEBUG "instance": "10.142.0.107:9100" 06:47:08 DEBUG }, 06:47:08 DEBUG "value": [ 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG [...] 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG "instance": "10.142.0.43:9100" 06:47:08 DEBUG }, 06:47:08 DEBUG "value": [ 06:47:08 DEBUG 1713592028, 06:47:08 DEBUG "3209297542144" 06:47:08 DEBUG ] 06:47:08 DEBUG } 06:47:08 DEBUG ] 06:47:08 DEBUG } 06:47:08 DEBUG } 06:47:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592028" 06:47:08 INFO [http_cmd]: http status code OK 06:47:08 DEBUG --- status code --- 06:47:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:47:08 DEBUG --- http response --- 06:47:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 06:47:08 DEBUG ----- output ----- 06:47:08 DEBUG { 06:47:08 DEBUG "status": "success", 06:47:08 DEBUG "data": { 06:47:08 DEBUG "resultType": "vector", 06:47:08 DEBUG "result": [ 06:47:08 DEBUG { 06:47:08 DEBUG "metric": { 06:47:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:47:08 DEBUG }, 06:47:08 DEBUG "value": [ 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG [...] 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG "pod": "idm-b85f9d978-txpwv" 06:47:08 DEBUG }, 06:47:08 DEBUG "value": [ 06:47:08 DEBUG 1713592028, 06:47:08 DEBUG "0" 06:47:08 DEBUG ] 06:47:08 DEBUG } 06:47:08 DEBUG ] 06:47:08 DEBUG } 06:47:08 DEBUG } 06:47:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592028" 06:47:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592028" 06:47:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592028" 06:47:08 INFO [http_cmd]: http status code OK 06:47:08 DEBUG --- status code --- 06:47:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:47:08 DEBUG --- http response --- 06:47:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 06:47:08 DEBUG ----- output ----- 06:47:08 DEBUG { 06:47:08 DEBUG "status": "success", 06:47:08 DEBUG "data": { 06:47:08 DEBUG "resultType": "vector", 06:47:08 DEBUG "result": [ 06:47:08 DEBUG { 06:47:08 DEBUG "metric": { 06:47:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:47:08 DEBUG }, 06:47:08 DEBUG "value": [ 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG [...] 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG "pod": "idm-b85f9d978-txpwv" 06:47:08 DEBUG }, 06:47:08 DEBUG "value": [ 06:47:08 DEBUG 1713592028, 06:47:08 DEBUG "0" 06:47:08 DEBUG ] 06:47:08 DEBUG } 06:47:08 DEBUG ] 06:47:08 DEBUG } 06:47:08 DEBUG } 06:47:08 INFO [http_cmd]: http status code OK 06:47:08 DEBUG --- status code --- 06:47:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:47:08 DEBUG --- http response --- 06:47:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:47:08 DEBUG ----- output ----- 06:47:08 DEBUG { 06:47:08 DEBUG "status": "success", 06:47:08 DEBUG "data": { 06:47:08 DEBUG "resultType": "vector", 06:47:08 DEBUG "result": [ 06:47:08 DEBUG { 06:47:08 DEBUG "metric": { 06:47:08 DEBUG "pod": "ds-idrepo-0" 06:47:08 DEBUG }, 06:47:08 DEBUG "value": [ 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG [...] 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG "pod": "ds-idrepo-2" 06:47:08 DEBUG }, 06:47:08 DEBUG "value": [ 06:47:08 DEBUG 1713592028, 06:47:08 DEBUG "0" 06:47:08 DEBUG ] 06:47:08 DEBUG } 06:47:08 DEBUG ] 06:47:08 DEBUG } 06:47:08 DEBUG } 06:47:08 INFO [http_cmd]: http status code OK 06:47:08 DEBUG --- status code --- 06:47:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:47:08 DEBUG --- http response --- 06:47:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:47:08 DEBUG ----- output ----- 06:47:08 DEBUG { 06:47:08 DEBUG "status": "success", 06:47:08 DEBUG "data": { 06:47:08 DEBUG "resultType": "vector", 06:47:08 DEBUG "result": [ 06:47:08 DEBUG { 06:47:08 DEBUG "metric": { 06:47:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:47:08 DEBUG }, 06:47:08 DEBUG "value": [ 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG [...] 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:47:08 DEBUG }, 06:47:08 DEBUG "value": [ 06:47:08 DEBUG 1713592028, 06:47:08 DEBUG "479956.5" 06:47:08 DEBUG ] 06:47:08 DEBUG } 06:47:08 DEBUG ] 06:47:08 DEBUG } 06:47:08 DEBUG } 06:47:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713592028" 06:47:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592028" 06:47:08 ERROR [http_cmd]: ERROR 06:47:08 ERROR --- status code --- 06:47:08 ERROR http status code is 400 (expected [200, 401, 403, 404]) 06:47:08 ERROR --- http response --- 06:47:08 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 06:47:08 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 06:47:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592028" 06:47:08 INFO [http_cmd]: http status code OK 06:47:08 DEBUG --- status code --- 06:47:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:47:08 DEBUG --- http response --- 06:47:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:47:08 DEBUG ----- output ----- 06:47:08 DEBUG { 06:47:08 DEBUG "status": "success", 06:47:08 DEBUG "data": { 06:47:08 DEBUG "resultType": "vector", 06:47:08 DEBUG "result": [ 06:47:08 DEBUG { 06:47:08 DEBUG "metric": { 06:47:08 DEBUG "pod": "ds-cts-0" 06:47:08 DEBUG }, 06:47:08 DEBUG "value": [ 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG [...] 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG "pod": "ds-cts-2" 06:47:08 DEBUG }, 06:47:08 DEBUG "value": [ 06:47:08 DEBUG 1713592028, 06:47:08 DEBUG "12.233333333333333" 06:47:08 DEBUG ] 06:47:08 DEBUG } 06:47:08 DEBUG ] 06:47:08 DEBUG } 06:47:08 DEBUG } 06:47:08 INFO [http_cmd]: http status code OK 06:47:08 DEBUG --- status code --- 06:47:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:47:08 DEBUG --- http response --- 06:47:08 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 06:47:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713592028" 06:47:08 INFO [http_cmd]: http status code OK 06:47:08 DEBUG --- status code --- 06:47:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:47:08 DEBUG --- http response --- 06:47:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:47:08 DEBUG ----- output ----- 06:47:08 DEBUG { 06:47:08 DEBUG "status": "success", 06:47:08 DEBUG "data": { 06:47:08 DEBUG "resultType": "vector", 06:47:08 DEBUG "result": [ 06:47:08 DEBUG { 06:47:08 DEBUG "metric": { 06:47:08 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 06:47:08 DEBUG }, 06:47:08 DEBUG "value": [ 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG [...] 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 06:47:08 DEBUG }, 06:47:08 DEBUG "value": [ 06:47:08 DEBUG 1713592028, 06:47:08 DEBUG "4978913280" 06:47:08 DEBUG ] 06:47:08 DEBUG } 06:47:08 DEBUG ] 06:47:08 DEBUG } 06:47:08 DEBUG } 06:47:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592028" 06:47:08 INFO [http_cmd]: http status code OK 06:47:08 DEBUG --- status code --- 06:47:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:47:08 DEBUG --- http response --- 06:47:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:47:08 DEBUG ----- output ----- 06:47:08 DEBUG { 06:47:08 DEBUG "status": "success", 06:47:08 DEBUG "data": { 06:47:08 DEBUG "resultType": "vector", 06:47:08 DEBUG "result": [ 06:47:08 DEBUG { 06:47:08 DEBUG "metric": { 06:47:08 DEBUG "pod": "ds-idrepo-0" 06:47:08 DEBUG }, 06:47:08 DEBUG "value": [ 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG [...] 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG "pod": "ds-idrepo-2" 06:47:08 DEBUG }, 06:47:08 DEBUG "value": [ 06:47:08 DEBUG 1713592028, 06:47:08 DEBUG "0" 06:47:08 DEBUG ] 06:47:08 DEBUG } 06:47:08 DEBUG ] 06:47:08 DEBUG } 06:47:08 DEBUG } 06:47:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592028" 06:47:08 INFO [http_cmd]: http status code OK 06:47:08 DEBUG --- status code --- 06:47:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:47:08 DEBUG --- http response --- 06:47:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:47:08 DEBUG ----- output ----- 06:47:08 DEBUG { 06:47:08 DEBUG "status": "success", 06:47:08 DEBUG "data": { 06:47:08 DEBUG "resultType": "vector", 06:47:08 DEBUG "result": [ 06:47:08 DEBUG { 06:47:08 DEBUG "metric": { 06:47:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:47:08 DEBUG }, 06:47:08 DEBUG "value": [ 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG [...] 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:47:08 DEBUG }, 06:47:08 DEBUG "value": [ 06:47:08 DEBUG 1713592028, 06:47:08 DEBUG "2659" 06:47:08 DEBUG ] 06:47:08 DEBUG } 06:47:08 DEBUG ] 06:47:08 DEBUG } 06:47:08 DEBUG } 06:47:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592028" 06:47:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592028" 06:47:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592028" 06:47:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713592028" 06:47:08 INFO [http_cmd]: http status code OK 06:47:08 INFO [http_cmd]: http status code OK 06:47:08 INFO [http_cmd]: http status code OK 06:47:08 DEBUG --- status code --- 06:47:08 DEBUG --- status code --- 06:47:08 DEBUG --- status code --- 06:47:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:47:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:47:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:47:08 DEBUG --- http response --- 06:47:08 DEBUG --- http response --- 06:47:08 DEBUG --- http response --- 06:47:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:47:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:47:08 DEBUG ----- output ----- 06:47:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:47:08 DEBUG ----- output ----- 06:47:08 DEBUG { 06:47:08 DEBUG ----- output ----- 06:47:08 DEBUG { 06:47:08 DEBUG "status": "success", 06:47:08 DEBUG { 06:47:08 DEBUG "status": "success", 06:47:08 DEBUG "data": { 06:47:08 DEBUG "status": "success", 06:47:08 DEBUG "data": { 06:47:08 DEBUG "resultType": "vector", 06:47:08 DEBUG "data": { 06:47:08 DEBUG "resultType": "vector", 06:47:08 DEBUG "result": [ 06:47:08 DEBUG "resultType": "vector", 06:47:08 DEBUG "result": [ 06:47:08 DEBUG { 06:47:08 DEBUG "result": [ 06:47:08 DEBUG { 06:47:08 DEBUG "metric": { 06:47:08 DEBUG { 06:47:08 DEBUG "metric": { 06:47:08 DEBUG "pod": "ds-cts-0" 06:47:08 DEBUG "metric": { 06:47:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:47:08 DEBUG }, 06:47:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:47:08 DEBUG }, 06:47:08 DEBUG "value": [ 06:47:08 DEBUG }, 06:47:08 DEBUG "value": [ 06:47:08 DEBUG 06:47:08 DEBUG "value": [ 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG [...] 06:47:08 DEBUG 06:47:08 DEBUG [...] 06:47:08 DEBUG 06:47:08 DEBUG [...] 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG "pod": "ds-cts-2" 06:47:08 DEBUG 06:47:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:47:08 DEBUG }, 06:47:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:47:08 DEBUG }, 06:47:08 DEBUG "value": [ 06:47:08 DEBUG }, 06:47:08 DEBUG "value": [ 06:47:08 DEBUG 1713592028, 06:47:08 DEBUG "value": [ 06:47:08 DEBUG 1713592028, 06:47:08 DEBUG "0" 06:47:08 DEBUG 1713592028, 06:47:08 DEBUG "2659.3333333333335" 06:47:08 DEBUG ] 06:47:08 DEBUG "2672.6666666666665" 06:47:08 DEBUG ] 06:47:08 DEBUG } 06:47:08 DEBUG ] 06:47:08 DEBUG } 06:47:08 DEBUG ] 06:47:08 DEBUG } 06:47:08 DEBUG ] 06:47:08 DEBUG } 06:47:08 DEBUG ] 06:47:08 DEBUG } 06:47:08 DEBUG } 06:47:08 DEBUG } 06:47:08 INFO [http_cmd]: http status code OK 06:47:08 DEBUG } 06:47:08 DEBUG } 06:47:08 DEBUG --- status code --- 06:47:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:47:08 DEBUG --- http response --- 06:47:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:47:08 DEBUG ----- output ----- 06:47:08 DEBUG { 06:47:08 DEBUG "status": "success", 06:47:08 DEBUG "data": { 06:47:08 DEBUG "resultType": "vector", 06:47:08 DEBUG "result": [ 06:47:08 DEBUG { 06:47:08 DEBUG "metric": { 06:47:08 DEBUG "pod": "ds-idrepo-0" 06:47:08 DEBUG }, 06:47:08 DEBUG "value": [ 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG [...] 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG "pod": "ds-cts-2" 06:47:08 DEBUG }, 06:47:08 DEBUG "value": [ 06:47:08 DEBUG 1713592028, 06:47:08 DEBUG "0" 06:47:08 DEBUG ] 06:47:08 DEBUG } 06:47:08 DEBUG ] 06:47:08 DEBUG } 06:47:08 DEBUG } 06:47:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592028" 06:47:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592028" 06:47:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592028" 06:47:08 INFO [http_cmd]: http status code OK 06:47:08 DEBUG --- status code --- 06:47:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:47:08 DEBUG --- http response --- 06:47:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:47:08 DEBUG ----- output ----- 06:47:08 DEBUG { 06:47:08 DEBUG "status": "success", 06:47:08 DEBUG "data": { 06:47:08 DEBUG "resultType": "vector", 06:47:08 DEBUG "result": [ 06:47:08 DEBUG { 06:47:08 DEBUG "metric": { 06:47:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:47:08 DEBUG }, 06:47:08 DEBUG "value": [ 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG [...] 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:47:08 DEBUG }, 06:47:08 DEBUG "value": [ 06:47:08 DEBUG 1713592028, 06:47:08 DEBUG "0" 06:47:08 DEBUG ] 06:47:08 DEBUG } 06:47:08 DEBUG ] 06:47:08 DEBUG } 06:47:08 DEBUG } 06:47:08 INFO [http_cmd]: http status code OK 06:47:08 DEBUG --- status code --- 06:47:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:47:08 DEBUG --- http response --- 06:47:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:47:08 DEBUG ----- output ----- 06:47:08 DEBUG { 06:47:08 DEBUG "status": "success", 06:47:08 DEBUG "data": { 06:47:08 DEBUG "resultType": "vector", 06:47:08 DEBUG "result": [ 06:47:08 DEBUG { 06:47:08 DEBUG "metric": { 06:47:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:47:08 DEBUG }, 06:47:08 DEBUG "value": [ 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG [...] 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:47:08 DEBUG }, 06:47:08 DEBUG "value": [ 06:47:08 DEBUG 1713592028, 06:47:08 DEBUG "2658.5666666666666" 06:47:08 DEBUG ] 06:47:08 DEBUG } 06:47:08 DEBUG ] 06:47:08 DEBUG } 06:47:08 DEBUG } 06:47:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592028" 06:47:08 INFO [http_cmd]: http status code OK 06:47:08 DEBUG --- status code --- 06:47:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:47:08 DEBUG --- http response --- 06:47:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:47:08 DEBUG ----- output ----- 06:47:08 DEBUG { 06:47:08 DEBUG "status": "success", 06:47:08 DEBUG "data": { 06:47:08 DEBUG "resultType": "vector", 06:47:08 DEBUG "result": [ 06:47:08 DEBUG { 06:47:08 DEBUG "metric": { 06:47:08 DEBUG "pod": "ds-idrepo-0" 06:47:08 DEBUG }, 06:47:08 DEBUG "value": [ 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG [...] 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG "pod": "ds-idrepo-2" 06:47:08 DEBUG }, 06:47:08 DEBUG "value": [ 06:47:08 DEBUG 1713592028, 06:47:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592028" 06:47:08 DEBUG "0" 06:47:08 DEBUG ] 06:47:08 DEBUG } 06:47:08 DEBUG ] 06:47:08 DEBUG } 06:47:08 DEBUG } 06:47:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713592028" 06:47:08 INFO [http_cmd]: http status code OK 06:47:08 DEBUG --- status code --- 06:47:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:47:08 DEBUG --- http response --- 06:47:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:47:08 DEBUG ----- output ----- 06:47:08 DEBUG { 06:47:08 DEBUG "status": "success", 06:47:08 DEBUG "data": { 06:47:08 DEBUG "resultType": "vector", 06:47:08 DEBUG "result": [ 06:47:08 DEBUG { 06:47:08 DEBUG "metric": { 06:47:08 DEBUG "pod": "ds-cts-0" 06:47:08 DEBUG }, 06:47:08 DEBUG "value": [ 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG [...] 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG "pod": "ds-cts-2" 06:47:08 DEBUG }, 06:47:08 DEBUG "value": [ 06:47:08 DEBUG 1713592028, 06:47:08 DEBUG "0.00040656666666666666" 06:47:08 DEBUG ] 06:47:08 DEBUG } 06:47:08 DEBUG ] 06:47:08 DEBUG } 06:47:08 DEBUG } 06:47:08 INFO [http_cmd]: http status code OK 06:47:08 DEBUG --- status code --- 06:47:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:47:08 DEBUG --- http response --- 06:47:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:47:08 DEBUG ----- output ----- 06:47:08 DEBUG { 06:47:08 DEBUG "status": "success", 06:47:08 DEBUG "data": { 06:47:08 DEBUG "resultType": "vector", 06:47:08 DEBUG "result": [ 06:47:08 DEBUG { 06:47:08 DEBUG "metric": { 06:47:08 DEBUG "instance": "10.142.0.107:9100" 06:47:08 DEBUG }, 06:47:08 DEBUG "value": [ 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG [...] 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG "instance": "10.142.0.43:9100" 06:47:08 DEBUG }, 06:47:08 DEBUG "value": [ 06:47:08 DEBUG 1713592028, 06:47:08 DEBUG "0.26056666666672185" 06:47:08 DEBUG ] 06:47:08 DEBUG } 06:47:08 DEBUG ] 06:47:08 DEBUG } 06:47:08 DEBUG } 06:47:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713592028" 06:47:08 INFO [http_cmd]: http status code OK 06:47:08 DEBUG --- status code --- 06:47:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:47:08 DEBUG --- http response --- 06:47:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:47:08 DEBUG ----- output ----- 06:47:08 DEBUG { 06:47:08 DEBUG "status": "success", 06:47:08 DEBUG "data": { 06:47:08 DEBUG "resultType": "vector", 06:47:08 DEBUG "result": [ 06:47:08 DEBUG { 06:47:08 DEBUG "metric": { 06:47:08 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 06:47:08 DEBUG }, 06:47:08 DEBUG "value": [ 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG [...] 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 06:47:08 DEBUG }, 06:47:08 DEBUG "value": [ 06:47:08 DEBUG 1713592028, 06:47:08 DEBUG "8.00171813450889" 06:47:08 DEBUG ] 06:47:08 DEBUG } 06:47:08 DEBUG ] 06:47:08 DEBUG } 06:47:08 DEBUG } 06:47:08 INFO [http_cmd]: http status code OK 06:47:08 DEBUG --- status code --- 06:47:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:47:08 DEBUG --- http response --- 06:47:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:47:08 DEBUG ----- output ----- 06:47:08 DEBUG { 06:47:08 DEBUG "status": "success", 06:47:08 DEBUG "data": { 06:47:08 DEBUG "resultType": "vector", 06:47:08 DEBUG "result": [ 06:47:08 DEBUG { 06:47:08 DEBUG "metric": { 06:47:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:47:08 DEBUG }, 06:47:08 DEBUG "value": [ 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG [...] 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:47:08 DEBUG }, 06:47:08 DEBUG "value": [ 06:47:08 DEBUG 1713592028, 06:47:08 DEBUG "0.096" 06:47:08 DEBUG ] 06:47:08 DEBUG } 06:47:08 DEBUG ] 06:47:08 DEBUG } 06:47:08 DEBUG } 06:47:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592028" 06:47:08 INFO [http_cmd]: http status code OK 06:47:08 DEBUG --- status code --- 06:47:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:47:08 DEBUG --- http response --- 06:47:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 06:47:08 DEBUG ----- output ----- 06:47:08 DEBUG { 06:47:08 DEBUG "status": "success", 06:47:08 DEBUG "data": { 06:47:08 DEBUG "resultType": "vector", 06:47:08 DEBUG "result": [ 06:47:08 DEBUG { 06:47:08 DEBUG "metric": { 06:47:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:47:08 DEBUG }, 06:47:08 DEBUG "value": [ 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG [...] 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG "pod": "idm-b85f9d978-txpwv" 06:47:08 DEBUG }, 06:47:08 DEBUG "value": [ 06:47:08 DEBUG 1713592028, 06:47:08 DEBUG "207.49651403806294" 06:47:08 DEBUG ] 06:47:08 DEBUG } 06:47:08 DEBUG ] 06:47:08 DEBUG } 06:47:08 DEBUG } 06:47:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592028" 06:47:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592028" 06:47:08 INFO [http_cmd]: http status code OK 06:47:08 DEBUG --- status code --- 06:47:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:47:08 DEBUG --- http response --- 06:47:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:47:08 DEBUG ----- output ----- 06:47:08 DEBUG { 06:47:08 DEBUG "status": "success", 06:47:08 DEBUG "data": { 06:47:08 DEBUG "resultType": "vector", 06:47:08 DEBUG "result": [ 06:47:08 DEBUG { 06:47:08 DEBUG "metric": { 06:47:08 DEBUG "instance": "10.142.0.107:9100" 06:47:08 DEBUG }, 06:47:08 DEBUG "value": [ 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG [...] 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG "instance": "10.142.0.43:9100" 06:47:08 DEBUG }, 06:47:08 DEBUG "value": [ 06:47:08 DEBUG 1713592028, 06:47:08 DEBUG "80666068.33333334" 06:47:08 DEBUG ] 06:47:08 DEBUG } 06:47:08 DEBUG ] 06:47:08 DEBUG } 06:47:08 DEBUG } 06:47:08 INFO [http_cmd]: http status code OK 06:47:08 DEBUG --- status code --- 06:47:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:47:08 DEBUG --- http response --- 06:47:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:47:08 DEBUG ----- output ----- 06:47:08 DEBUG { 06:47:08 DEBUG "status": "success", 06:47:08 DEBUG "data": { 06:47:08 DEBUG "resultType": "vector", 06:47:08 DEBUG "result": [ 06:47:08 DEBUG { 06:47:08 DEBUG "metric": { 06:47:08 DEBUG "instance": "10.142.0.107:9100" 06:47:08 DEBUG }, 06:47:08 DEBUG "value": [ 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG [...] 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG "instance": "10.142.0.43:9100" 06:47:08 DEBUG }, 06:47:08 DEBUG "value": [ 06:47:08 DEBUG 1713592028, 06:47:08 DEBUG "80660736.66666666" 06:47:08 DEBUG ] 06:47:08 DEBUG } 06:47:08 DEBUG ] 06:47:08 DEBUG } 06:47:08 DEBUG } 06:47:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592028" 06:47:08 INFO [http_cmd]: http status code OK 06:47:08 DEBUG --- status code --- 06:47:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:47:08 DEBUG --- http response --- 06:47:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 06:47:08 DEBUG ----- output ----- 06:47:08 DEBUG { 06:47:08 DEBUG "status": "success", 06:47:08 DEBUG "data": { 06:47:08 DEBUG "resultType": "vector", 06:47:08 DEBUG "result": [ 06:47:08 DEBUG { 06:47:08 DEBUG "metric": { 06:47:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:47:08 DEBUG }, 06:47:08 DEBUG "value": [ 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG [...] 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG "pod": "idm-b85f9d978-txpwv" 06:47:08 DEBUG }, 06:47:08 DEBUG "value": [ 06:47:08 DEBUG 1713592028, 06:47:08 DEBUG "0.0023814226993829654" 06:47:08 DEBUG ] 06:47:08 DEBUG } 06:47:08 DEBUG ] 06:47:08 DEBUG } 06:47:08 DEBUG } 06:47:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592028" 06:47:08 INFO [http_cmd]: http status code OK 06:47:08 DEBUG --- status code --- 06:47:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:47:08 DEBUG --- http response --- 06:47:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:47:08 DEBUG ----- output ----- 06:47:08 DEBUG { 06:47:08 DEBUG "status": "success", 06:47:08 DEBUG "data": { 06:47:08 DEBUG "resultType": "vector", 06:47:08 DEBUG "result": [ 06:47:08 DEBUG { 06:47:08 DEBUG "metric": { 06:47:08 DEBUG "instance": "10.142.0.107:9100" 06:47:08 DEBUG }, 06:47:08 DEBUG "value": [ 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG [...] 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG 06:47:08 DEBUG "instance": "10.142.0.43:9100" 06:47:08 DEBUG }, 06:47:08 DEBUG "value": [ 06:47:08 DEBUG 1713592028, 06:47:08 DEBUG "0.09900000000001227" 06:47:08 DEBUG ] 06:47:08 DEBUG } 06:47:08 DEBUG ] 06:47:08 DEBUG } 06:47:08 DEBUG } 06:47:09 INFO 06:47:09 INFO [loop_until]: kubectl --namespace=xlou top pods 06:47:09 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:47:09 INFO [loop_until]: OK (rc = 0) 06:47:09 DEBUG --- stdout --- 06:47:09 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 17342m 8650Mi am-767f6b67b5-qcqz2 17503m 8718Mi am-767f6b67b5-vbltz 18773m 8649Mi ds-cts-0 19358m 10386Mi ds-cts-1 17703m 10661Mi ds-cts-2 19092m 10146Mi ds-idrepo-0 7733m 3747Mi ds-idrepo-1 7459m 3802Mi ds-idrepo-2 7817m 3671Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1177Mi idm-b85f9d978-wnhxb 4m 1236Mi lodemon-7cb5b9456d-dskfn 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 8247m 4656Mi 06:47:09 DEBUG --- stderr --- 06:47:09 DEBUG 06:47:09 INFO 06:47:09 INFO [loop_until]: kubectl --namespace=xlou top node 06:47:09 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:47:09 INFO [loop_until]: OK (rc = 0) 06:47:09 DEBUG --- stdout --- 06:47:09 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 69m 0% 6262Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7992m 26% 7428Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 17111m 57% 11363Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 81m 0% 3365Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 18517m 62% 13529Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 156m 0% 5181Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 17380m 58% 11594Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 19064m 63% 3010Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 19289m 64% 10874Mi 9% gke-xlou-cdm-ds-0f601751-j488 8053m 26% 6994Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 17385m 58% 15591Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 8023m 26% 7690Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 18706m 62% 12745Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 8164m 51% 8492Mi 14% 06:47:09 DEBUG --- stderr --- 06:47:09 DEBUG 06:48:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592088" 06:48:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592088" 06:48:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592088" 06:48:08 INFO [http_cmd]: http status code OK 06:48:08 DEBUG --- status code --- 06:48:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:48:08 DEBUG --- http response --- 06:48:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:48:08 DEBUG ----- output ----- 06:48:08 DEBUG { 06:48:08 DEBUG "status": "success", 06:48:08 DEBUG "data": { 06:48:08 DEBUG "resultType": "vector", 06:48:08 DEBUG "result": [ 06:48:08 DEBUG { 06:48:08 DEBUG "metric": { 06:48:08 DEBUG "instance": "10.142.0.107:9100" 06:48:08 DEBUG }, 06:48:08 DEBUG "value": [ 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG [...] 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG "instance": "10.142.0.43:9100" 06:48:08 DEBUG }, 06:48:08 DEBUG "value": [ 06:48:08 DEBUG 1713592088, 06:48:08 DEBUG "3213417272320" 06:48:08 DEBUG ] 06:48:08 DEBUG } 06:48:08 DEBUG ] 06:48:08 DEBUG } 06:48:08 DEBUG } 06:48:08 INFO [http_cmd]: http status code OK 06:48:08 DEBUG --- status code --- 06:48:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:48:08 DEBUG --- http response --- 06:48:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:48:08 DEBUG ----- output ----- 06:48:08 DEBUG { 06:48:08 DEBUG "status": "success", 06:48:08 DEBUG "data": { 06:48:08 DEBUG "resultType": "vector", 06:48:08 DEBUG "result": [ 06:48:08 DEBUG { 06:48:08 DEBUG "metric": { 06:48:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:48:08 DEBUG }, 06:48:08 DEBUG "value": [ 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG [...] 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG "pod": "am-767f6b67b5-qcqz2" 06:48:08 DEBUG }, 06:48:08 DEBUG "value": [ 06:48:08 DEBUG 1713592088, 06:48:08 DEBUG "825.310066576563" 06:48:08 DEBUG ] 06:48:08 DEBUG } 06:48:08 DEBUG ] 06:48:08 DEBUG } 06:48:08 DEBUG } 06:48:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592088" 06:48:08 INFO [http_cmd]: http status code OK 06:48:08 DEBUG --- status code --- 06:48:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:48:08 DEBUG --- http response --- 06:48:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:48:08 DEBUG ----- output ----- 06:48:08 DEBUG { 06:48:08 DEBUG "status": "success", 06:48:08 DEBUG "data": { 06:48:08 DEBUG "resultType": "vector", 06:48:08 DEBUG "result": [ 06:48:08 DEBUG { 06:48:08 DEBUG "metric": { 06:48:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:48:08 DEBUG }, 06:48:08 DEBUG "value": [ 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG [...] 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG "pod": "am-767f6b67b5-qcqz2" 06:48:08 DEBUG }, 06:48:08 DEBUG "value": [ 06:48:08 DEBUG 1713592088, 06:48:08 DEBUG "0" 06:48:08 DEBUG ] 06:48:08 DEBUG } 06:48:08 DEBUG ] 06:48:08 DEBUG } 06:48:08 DEBUG } 06:48:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592088" 06:48:08 INFO [http_cmd]: http status code OK 06:48:08 DEBUG --- status code --- 06:48:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:48:08 DEBUG --- http response --- 06:48:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:48:08 DEBUG ----- output ----- 06:48:08 DEBUG { 06:48:08 DEBUG "status": "success", 06:48:08 DEBUG "data": { 06:48:08 DEBUG "resultType": "vector", 06:48:08 DEBUG "result": [ 06:48:08 DEBUG { 06:48:08 DEBUG "metric": { 06:48:08 DEBUG "pod": "ds-cts-0" 06:48:08 DEBUG }, 06:48:08 DEBUG "value": [ 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG [...] 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG "pod": "ds-cts-2" 06:48:08 DEBUG }, 06:48:08 DEBUG "value": [ 06:48:08 DEBUG 1713592088, 06:48:08 DEBUG "30" 06:48:08 DEBUG ] 06:48:08 DEBUG } 06:48:08 DEBUG ] 06:48:08 DEBUG } 06:48:08 DEBUG } 06:48:08 INFO [http_cmd]: http status code OK 06:48:08 DEBUG --- status code --- 06:48:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:48:08 DEBUG --- http response --- 06:48:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:48:08 DEBUG ----- output ----- 06:48:08 DEBUG { 06:48:08 DEBUG "status": "success", 06:48:08 DEBUG "data": { 06:48:08 DEBUG "resultType": "vector", 06:48:08 DEBUG "result": [ 06:48:08 DEBUG { 06:48:08 DEBUG "metric": { 06:48:08 DEBUG "pod": "ds-idrepo-0" 06:48:08 DEBUG }, 06:48:08 DEBUG "value": [ 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG [...] 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG "pod": "ds-idrepo-2" 06:48:08 DEBUG }, 06:48:08 DEBUG "value": [ 06:48:08 DEBUG 1713592088, 06:48:08 DEBUG "0" 06:48:08 DEBUG ] 06:48:08 DEBUG } 06:48:08 DEBUG ] 06:48:08 DEBUG } 06:48:08 DEBUG } 06:48:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592088" 06:48:08 INFO [http_cmd]: http status code OK 06:48:08 DEBUG --- status code --- 06:48:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:48:08 DEBUG --- http response --- 06:48:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:48:08 DEBUG ----- output ----- 06:48:08 DEBUG { 06:48:08 DEBUG "status": "success", 06:48:08 DEBUG "data": { 06:48:08 DEBUG "resultType": "vector", 06:48:08 DEBUG "result": [ 06:48:08 DEBUG { 06:48:08 DEBUG "metric": { 06:48:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:48:08 DEBUG }, 06:48:08 DEBUG "value": [ 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG [...] 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:48:08 DEBUG }, 06:48:08 DEBUG "value": [ 06:48:08 DEBUG 1713592088, 06:48:08 DEBUG "479255" 06:48:08 DEBUG ] 06:48:08 DEBUG } 06:48:08 DEBUG ] 06:48:08 DEBUG } 06:48:08 DEBUG } 06:48:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713592088" 06:48:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592088" 06:48:08 ERROR [http_cmd]: ERROR 06:48:08 ERROR --- status code --- 06:48:08 ERROR http status code is 400 (expected [200, 401, 403, 404]) 06:48:08 ERROR --- http response --- 06:48:08 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 06:48:08 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 06:48:08 INFO [http_cmd]: http status code OK 06:48:08 DEBUG --- status code --- 06:48:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:48:08 DEBUG --- http response --- 06:48:08 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 06:48:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713592088" 06:48:08 INFO [http_cmd]: http status code OK 06:48:08 DEBUG --- status code --- 06:48:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:48:08 DEBUG --- http response --- 06:48:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:48:08 DEBUG ----- output ----- 06:48:08 DEBUG { 06:48:08 DEBUG "status": "success", 06:48:08 DEBUG "data": { 06:48:08 DEBUG "resultType": "vector", 06:48:08 DEBUG "result": [ 06:48:08 DEBUG { 06:48:08 DEBUG "metric": { 06:48:08 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 06:48:08 DEBUG }, 06:48:08 DEBUG "value": [ 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG [...] 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 06:48:08 DEBUG }, 06:48:08 DEBUG "value": [ 06:48:08 DEBUG 1713592088, 06:48:08 DEBUG "4981764096" 06:48:08 DEBUG ] 06:48:08 DEBUG } 06:48:08 DEBUG ] 06:48:08 DEBUG } 06:48:08 DEBUG } 06:48:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592088" 06:48:08 INFO [http_cmd]: http status code OK 06:48:08 DEBUG --- status code --- 06:48:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:48:08 DEBUG --- http response --- 06:48:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:48:08 DEBUG ----- output ----- 06:48:08 DEBUG { 06:48:08 DEBUG "status": "success", 06:48:08 DEBUG "data": { 06:48:08 DEBUG "resultType": "vector", 06:48:08 DEBUG "result": [ 06:48:08 DEBUG { 06:48:08 DEBUG "metric": { 06:48:08 DEBUG "pod": "ds-idrepo-0" 06:48:08 DEBUG }, 06:48:08 DEBUG "value": [ 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG [...] 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG "pod": "ds-idrepo-2" 06:48:08 DEBUG }, 06:48:08 DEBUG "value": [ 06:48:08 DEBUG 1713592088, 06:48:08 DEBUG "0" 06:48:08 DEBUG ] 06:48:08 DEBUG } 06:48:08 DEBUG ] 06:48:08 DEBUG } 06:48:08 DEBUG } 06:48:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592088" 06:48:08 INFO [http_cmd]: http status code OK 06:48:08 DEBUG --- status code --- 06:48:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:48:08 DEBUG --- http response --- 06:48:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:48:08 DEBUG ----- output ----- 06:48:08 DEBUG { 06:48:08 DEBUG "status": "success", 06:48:08 DEBUG "data": { 06:48:08 DEBUG "resultType": "vector", 06:48:08 DEBUG "result": [ 06:48:08 DEBUG { 06:48:08 DEBUG "metric": { 06:48:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:48:08 DEBUG }, 06:48:08 DEBUG "value": [ 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG [...] 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:48:08 DEBUG }, 06:48:08 DEBUG "value": [ 06:48:08 DEBUG 1713592088, 06:48:08 DEBUG "2662.5" 06:48:08 DEBUG ] 06:48:08 DEBUG } 06:48:08 DEBUG ] 06:48:08 DEBUG } 06:48:08 DEBUG } 06:48:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592088" 06:48:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592088" 06:48:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592088" 06:48:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592088" 06:48:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713592088" 06:48:08 INFO [http_cmd]: http status code OK 06:48:08 INFO [http_cmd]: http status code OK 06:48:08 DEBUG --- status code --- 06:48:08 INFO [http_cmd]: http status code OK 06:48:08 DEBUG --- status code --- 06:48:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:48:08 DEBUG --- status code --- 06:48:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:48:08 DEBUG --- http response --- 06:48:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:48:08 DEBUG --- http response --- 06:48:08 DEBUG --- http response --- 06:48:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:48:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:48:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:48:08 DEBUG ----- output ----- 06:48:08 DEBUG ----- output ----- 06:48:08 DEBUG ----- output ----- 06:48:08 DEBUG { 06:48:08 DEBUG { 06:48:08 DEBUG { 06:48:08 DEBUG "status": "success", 06:48:08 DEBUG "status": "success", 06:48:08 DEBUG "status": "success", 06:48:08 DEBUG "data": { 06:48:08 DEBUG "data": { 06:48:08 DEBUG "data": { 06:48:08 DEBUG "resultType": "vector", 06:48:08 DEBUG "resultType": "vector", 06:48:08 DEBUG "resultType": "vector", 06:48:08 DEBUG "result": [ 06:48:08 DEBUG "result": [ 06:48:08 DEBUG "result": [ 06:48:08 DEBUG { 06:48:08 DEBUG { 06:48:08 DEBUG { 06:48:08 DEBUG "metric": { 06:48:08 DEBUG "metric": { 06:48:08 DEBUG "metric": { 06:48:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:48:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:48:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:48:08 DEBUG }, 06:48:08 DEBUG }, 06:48:08 DEBUG }, 06:48:08 DEBUG "value": [ 06:48:08 DEBUG "value": [ 06:48:08 DEBUG "value": [ 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592088" 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG [...] 06:48:08 DEBUG [...] 06:48:08 DEBUG [...] 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 INFO [http_cmd]: http status code OK 06:48:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:48:08 INFO [http_cmd]: http status code OK 06:48:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:48:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:48:08 DEBUG --- status code --- 06:48:08 DEBUG }, 06:48:08 DEBUG --- status code --- 06:48:08 DEBUG }, 06:48:08 DEBUG }, 06:48:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:48:08 DEBUG "value": [ 06:48:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:48:08 DEBUG "value": [ 06:48:08 DEBUG "value": [ 06:48:08 DEBUG --- http response --- 06:48:08 DEBUG 1713592088, 06:48:08 DEBUG --- http response --- 06:48:08 DEBUG 1713592088, 06:48:08 DEBUG 1713592088, 06:48:08 DEBUG "2664.1666666666665" 06:48:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:48:08 DEBUG "2663.233333333333" 06:48:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:48:08 DEBUG "2665.2666666666664" 06:48:08 DEBUG ] 06:48:08 DEBUG ----- output ----- 06:48:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592088" 06:48:08 DEBUG ] 06:48:08 DEBUG ----- output ----- 06:48:08 DEBUG ] 06:48:08 DEBUG } 06:48:08 DEBUG { 06:48:08 DEBUG } 06:48:08 DEBUG { 06:48:08 DEBUG } 06:48:08 DEBUG ] 06:48:08 DEBUG "status": "success", 06:48:08 DEBUG ] 06:48:08 DEBUG "status": "success", 06:48:08 DEBUG ] 06:48:08 DEBUG } 06:48:08 DEBUG "data": { 06:48:08 DEBUG } 06:48:08 DEBUG "data": { 06:48:08 DEBUG } 06:48:08 DEBUG } 06:48:08 DEBUG "resultType": "vector", 06:48:08 DEBUG } 06:48:08 DEBUG "resultType": "vector", 06:48:08 DEBUG } 06:48:08 DEBUG "result": [ 06:48:08 DEBUG "result": [ 06:48:08 DEBUG { 06:48:08 DEBUG { 06:48:08 DEBUG "metric": { 06:48:08 DEBUG "metric": { 06:48:08 DEBUG "pod": "ds-cts-0" 06:48:08 DEBUG "pod": "ds-idrepo-0" 06:48:08 DEBUG }, 06:48:08 DEBUG }, 06:48:08 DEBUG "value": [ 06:48:08 DEBUG "value": [ 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG [...] 06:48:08 DEBUG [...] 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG "pod": "ds-cts-2" 06:48:08 DEBUG "pod": "ds-cts-2" 06:48:08 INFO [http_cmd]: http status code OK 06:48:08 DEBUG }, 06:48:08 DEBUG }, 06:48:08 DEBUG --- status code --- 06:48:08 DEBUG "value": [ 06:48:08 DEBUG "value": [ 06:48:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:48:08 DEBUG 1713592088, 06:48:08 DEBUG 1713592088, 06:48:08 DEBUG --- http response --- 06:48:08 DEBUG "0" 06:48:08 DEBUG "0" 06:48:08 DEBUG ] 06:48:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:48:08 DEBUG ] 06:48:08 DEBUG } 06:48:08 DEBUG ----- output ----- 06:48:08 DEBUG } 06:48:08 DEBUG ] 06:48:08 DEBUG { 06:48:08 DEBUG ] 06:48:08 DEBUG } 06:48:08 DEBUG "status": "success", 06:48:08 DEBUG } 06:48:08 DEBUG } 06:48:08 DEBUG "data": { 06:48:08 DEBUG } 06:48:08 DEBUG "resultType": "vector", 06:48:08 DEBUG "result": [ 06:48:08 DEBUG { 06:48:08 DEBUG "metric": { 06:48:08 DEBUG "pod": "ds-cts-0" 06:48:08 DEBUG }, 06:48:08 DEBUG "value": [ 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG [...] 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG "pod": "ds-cts-2" 06:48:08 DEBUG }, 06:48:08 DEBUG "value": [ 06:48:08 DEBUG 1713592088, 06:48:08 DEBUG "0.00013333333333333329" 06:48:08 DEBUG ] 06:48:08 DEBUG } 06:48:08 DEBUG ] 06:48:08 DEBUG } 06:48:08 DEBUG } 06:48:08 INFO [http_cmd]: http status code OK 06:48:08 DEBUG --- status code --- 06:48:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:48:08 DEBUG --- http response --- 06:48:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592088" 06:48:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:48:08 DEBUG ----- output ----- 06:48:08 DEBUG { 06:48:08 DEBUG "status": "success", 06:48:08 DEBUG "data": { 06:48:08 DEBUG "resultType": "vector", 06:48:08 DEBUG "result": [ 06:48:08 DEBUG { 06:48:08 DEBUG "metric": { 06:48:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:48:08 DEBUG }, 06:48:08 DEBUG "value": [ 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG [...] 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:48:08 DEBUG }, 06:48:08 DEBUG "value": [ 06:48:08 DEBUG 1713592088, 06:48:08 DEBUG "0" 06:48:08 DEBUG ] 06:48:08 DEBUG } 06:48:08 DEBUG ] 06:48:08 DEBUG } 06:48:08 DEBUG } 06:48:08 INFO [http_cmd]: http status code OK 06:48:08 DEBUG --- status code --- 06:48:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:48:08 DEBUG --- http response --- 06:48:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:48:08 DEBUG ----- output ----- 06:48:08 DEBUG { 06:48:08 DEBUG "status": "success", 06:48:08 DEBUG "data": { 06:48:08 DEBUG "resultType": "vector", 06:48:08 DEBUG "result": [ 06:48:08 DEBUG { 06:48:08 DEBUG "metric": { 06:48:08 DEBUG "pod": "ds-idrepo-0" 06:48:08 DEBUG }, 06:48:08 DEBUG "value": [ 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG [...] 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG "pod": "ds-idrepo-2" 06:48:08 DEBUG }, 06:48:08 DEBUG "value": [ 06:48:08 DEBUG 1713592088, 06:48:08 DEBUG "0" 06:48:08 DEBUG ] 06:48:08 DEBUG } 06:48:08 DEBUG ] 06:48:08 DEBUG } 06:48:08 DEBUG } 06:48:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592088" 06:48:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592088" 06:48:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592088" 06:48:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713592088" 06:48:08 INFO [http_cmd]: http status code OK 06:48:08 DEBUG --- status code --- 06:48:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:48:08 DEBUG --- http response --- 06:48:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:48:08 DEBUG ----- output ----- 06:48:08 DEBUG { 06:48:08 DEBUG "status": "success", 06:48:08 DEBUG "data": { 06:48:08 DEBUG "resultType": "vector", 06:48:08 DEBUG "result": [ 06:48:08 DEBUG { 06:48:08 DEBUG "metric": { 06:48:08 DEBUG "instance": "10.142.0.107:9100" 06:48:08 DEBUG }, 06:48:08 DEBUG "value": [ 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG [...] 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG "instance": "10.142.0.43:9100" 06:48:08 DEBUG }, 06:48:08 DEBUG "value": [ 06:48:08 DEBUG 1713592088, 06:48:08 DEBUG "0.27176666666668814" 06:48:08 DEBUG ] 06:48:08 DEBUG } 06:48:08 DEBUG ] 06:48:08 DEBUG } 06:48:08 DEBUG } 06:48:08 INFO [http_cmd]: http status code OK 06:48:08 INFO [http_cmd]: http status code OK 06:48:08 DEBUG --- status code --- 06:48:08 DEBUG --- status code --- 06:48:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:48:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:48:08 DEBUG --- http response --- 06:48:08 DEBUG --- http response --- 06:48:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:48:08 DEBUG ----- output ----- 06:48:08 DEBUG { 06:48:08 DEBUG "status": "success", 06:48:08 DEBUG "data": { 06:48:08 DEBUG "resultType": "vector", 06:48:08 DEBUG "result": [ 06:48:08 DEBUG { 06:48:08 DEBUG "metric": { 06:48:08 DEBUG "instance": "10.142.0.107:9100" 06:48:08 DEBUG }, 06:48:08 DEBUG "value": [ 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG [...] 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG "instance": "10.142.0.43:9100" 06:48:08 DEBUG }, 06:48:08 DEBUG "value": [ 06:48:08 DEBUG 1713592088, 06:48:08 DEBUG "80694876.76666667" 06:48:08 DEBUG ] 06:48:08 DEBUG } 06:48:08 DEBUG ] 06:48:08 DEBUG } 06:48:08 DEBUG } 06:48:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:48:08 DEBUG ----- output ----- 06:48:08 DEBUG { 06:48:08 DEBUG "status": "success", 06:48:08 DEBUG "data": { 06:48:08 DEBUG "resultType": "vector", 06:48:08 DEBUG "result": [ 06:48:08 DEBUG { 06:48:08 DEBUG "metric": { 06:48:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:48:08 DEBUG }, 06:48:08 DEBUG "value": [ 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG [...] 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG "pod": "am-767f6b67b5-qcqz2" 06:48:08 DEBUG }, 06:48:08 DEBUG "value": [ 06:48:08 DEBUG 1713592088, 06:48:08 DEBUG "230266254.1077439" 06:48:08 DEBUG ] 06:48:08 DEBUG } 06:48:08 DEBUG ] 06:48:08 DEBUG } 06:48:08 DEBUG } 06:48:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713592088" 06:48:08 INFO [http_cmd]: http status code OK 06:48:08 DEBUG --- status code --- 06:48:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:48:08 DEBUG --- http response --- 06:48:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:48:08 DEBUG ----- output ----- 06:48:08 DEBUG { 06:48:08 DEBUG "status": "success", 06:48:08 DEBUG "data": { 06:48:08 DEBUG "resultType": "vector", 06:48:08 DEBUG "result": [ 06:48:08 DEBUG { 06:48:08 DEBUG "metric": { 06:48:08 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 06:48:08 DEBUG }, 06:48:08 DEBUG "value": [ 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG [...] 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 06:48:08 DEBUG }, 06:48:08 DEBUG "value": [ 06:48:08 DEBUG 1713592088, 06:48:08 DEBUG "8.026978407426212" 06:48:08 DEBUG ] 06:48:08 DEBUG } 06:48:08 DEBUG ] 06:48:08 DEBUG } 06:48:08 DEBUG } 06:48:08 INFO [http_cmd]: http status code OK 06:48:08 DEBUG --- status code --- 06:48:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:48:08 DEBUG --- http response --- 06:48:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:48:08 DEBUG ----- output ----- 06:48:08 DEBUG { 06:48:08 DEBUG "status": "success", 06:48:08 DEBUG "data": { 06:48:08 DEBUG "resultType": "vector", 06:48:08 DEBUG "result": [ 06:48:08 DEBUG { 06:48:08 DEBUG "metric": { 06:48:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:48:08 DEBUG }, 06:48:08 DEBUG "value": [ 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG [...] 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:48:08 DEBUG }, 06:48:08 DEBUG "value": [ 06:48:08 DEBUG 1713592088, 06:48:08 DEBUG "0.096" 06:48:08 DEBUG ] 06:48:08 DEBUG } 06:48:08 DEBUG ] 06:48:08 DEBUG } 06:48:08 DEBUG } 06:48:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592088" 06:48:08 INFO [http_cmd]: http status code OK 06:48:08 DEBUG --- status code --- 06:48:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:48:08 DEBUG --- http response --- 06:48:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:48:08 DEBUG ----- output ----- 06:48:08 DEBUG { 06:48:08 DEBUG "status": "success", 06:48:08 DEBUG "data": { 06:48:08 DEBUG "resultType": "vector", 06:48:08 DEBUG "result": [ 06:48:08 DEBUG { 06:48:08 DEBUG "metric": { 06:48:08 DEBUG "instance": "10.142.0.107:9100" 06:48:08 DEBUG }, 06:48:08 DEBUG "value": [ 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG [...] 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG "instance": "10.142.0.43:9100" 06:48:08 DEBUG }, 06:48:08 DEBUG "value": [ 06:48:08 DEBUG 1713592088, 06:48:08 DEBUG "80689284.73333333" 06:48:08 DEBUG ] 06:48:08 DEBUG } 06:48:08 DEBUG ] 06:48:08 DEBUG } 06:48:08 DEBUG } 06:48:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592088" 06:48:08 INFO [http_cmd]: http status code OK 06:48:08 DEBUG --- status code --- 06:48:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:48:08 DEBUG --- http response --- 06:48:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:48:08 DEBUG ----- output ----- 06:48:08 DEBUG { 06:48:08 DEBUG "status": "success", 06:48:08 DEBUG "data": { 06:48:08 DEBUG "resultType": "vector", 06:48:08 DEBUG "result": [ 06:48:08 DEBUG { 06:48:08 DEBUG "metric": { 06:48:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:48:08 DEBUG }, 06:48:08 DEBUG "value": [ 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG [...] 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG "pod": "am-767f6b67b5-qcqz2" 06:48:08 DEBUG }, 06:48:08 DEBUG "value": [ 06:48:08 DEBUG 1713592088, 06:48:08 DEBUG "11.630464300685588" 06:48:08 DEBUG ] 06:48:08 DEBUG } 06:48:08 DEBUG ] 06:48:08 DEBUG } 06:48:08 DEBUG } 06:48:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592088" 06:48:08 INFO [http_cmd]: http status code OK 06:48:08 DEBUG --- status code --- 06:48:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:48:08 DEBUG --- http response --- 06:48:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:48:08 DEBUG ----- output ----- 06:48:08 DEBUG { 06:48:08 DEBUG "status": "success", 06:48:08 DEBUG "data": { 06:48:08 DEBUG "resultType": "vector", 06:48:08 DEBUG "result": [ 06:48:08 DEBUG { 06:48:08 DEBUG "metric": { 06:48:08 DEBUG "instance": "10.142.0.107:9100" 06:48:08 DEBUG }, 06:48:08 DEBUG "value": [ 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG [...] 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG 06:48:08 DEBUG "instance": "10.142.0.43:9100" 06:48:08 DEBUG }, 06:48:08 DEBUG "value": [ 06:48:08 DEBUG 1713592088, 06:48:08 DEBUG "0.10566666666667855" 06:48:08 DEBUG ] 06:48:08 DEBUG } 06:48:08 DEBUG ] 06:48:08 DEBUG } 06:48:08 DEBUG } 06:48:09 INFO 06:48:09 INFO [loop_until]: kubectl --namespace=xlou top pods 06:48:09 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:48:09 INFO [loop_until]: OK (rc = 0) 06:48:09 DEBUG --- stdout --- 06:48:09 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 17092m 8650Mi am-767f6b67b5-qcqz2 17001m 8718Mi am-767f6b67b5-vbltz 18455m 8651Mi ds-cts-0 18945m 10385Mi ds-cts-1 16765m 10677Mi ds-cts-2 19209m 10225Mi ds-idrepo-0 7691m 3748Mi ds-idrepo-1 7640m 3802Mi ds-idrepo-2 7788m 3680Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1177Mi idm-b85f9d978-wnhxb 4m 1237Mi lodemon-7cb5b9456d-dskfn 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7926m 4658Mi 06:48:09 DEBUG --- stderr --- 06:48:09 DEBUG 06:48:09 INFO 06:48:09 INFO [loop_until]: kubectl --namespace=xlou top node 06:48:09 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:48:09 INFO [loop_until]: OK (rc = 0) 06:48:09 DEBUG --- stdout --- 06:48:09 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 71m 0% 6273Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7947m 26% 7426Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 16628m 55% 11361Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 81m 0% 3364Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 19056m 63% 13512Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 166m 0% 5205Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 17363m 58% 11594Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 19119m 64% 3013Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 19476m 65% 10870Mi 9% gke-xlou-cdm-ds-0f601751-j488 8140m 27% 7017Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 16619m 55% 15598Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 8064m 27% 7699Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 18764m 62% 12825Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 8164m 51% 8497Mi 14% 06:48:09 DEBUG --- stderr --- 06:48:09 DEBUG 06:49:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592148" 06:49:08 INFO [http_cmd]: http status code OK 06:49:08 DEBUG --- status code --- 06:49:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:49:08 DEBUG --- http response --- 06:49:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 06:49:08 DEBUG ----- output ----- 06:49:08 DEBUG { 06:49:08 DEBUG "status": "success", 06:49:08 DEBUG "data": { 06:49:08 DEBUG "resultType": "vector", 06:49:08 DEBUG "result": [ 06:49:08 DEBUG { 06:49:08 DEBUG "metric": { 06:49:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:49:08 DEBUG }, 06:49:08 DEBUG "value": [ 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG [...] 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:49:08 DEBUG }, 06:49:08 DEBUG "value": [ 06:49:08 DEBUG 1713592148, 06:49:08 DEBUG "0" 06:49:08 DEBUG ] 06:49:08 DEBUG } 06:49:08 DEBUG ] 06:49:08 DEBUG } 06:49:08 DEBUG } 06:49:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592148" 06:49:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592148" 06:49:08 INFO [http_cmd]: http status code OK 06:49:08 DEBUG --- status code --- 06:49:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:49:08 DEBUG --- http response --- 06:49:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:49:08 DEBUG ----- output ----- 06:49:08 DEBUG { 06:49:08 DEBUG "status": "success", 06:49:08 DEBUG "data": { 06:49:08 DEBUG "resultType": "vector", 06:49:08 DEBUG "result": [ 06:49:08 DEBUG { 06:49:08 DEBUG "metric": { 06:49:08 DEBUG "instance": "10.142.0.107:9100" 06:49:08 DEBUG }, 06:49:08 DEBUG "value": [ 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG [...] 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG "instance": "10.142.0.43:9100" 06:49:08 DEBUG }, 06:49:08 DEBUG "value": [ 06:49:08 DEBUG 1713592148, 06:49:08 DEBUG "3217691640832" 06:49:08 DEBUG ] 06:49:08 DEBUG } 06:49:08 DEBUG ] 06:49:08 DEBUG } 06:49:08 DEBUG } 06:49:08 INFO [http_cmd]: http status code OK 06:49:08 DEBUG --- status code --- 06:49:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:49:08 DEBUG --- http response --- 06:49:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 06:49:08 DEBUG ----- output ----- 06:49:08 DEBUG { 06:49:08 DEBUG "status": "success", 06:49:08 DEBUG "data": { 06:49:08 DEBUG "resultType": "vector", 06:49:08 DEBUG "result": [ 06:49:08 DEBUG { 06:49:08 DEBUG "metric": { 06:49:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:49:08 DEBUG }, 06:49:08 DEBUG "value": [ 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG [...] 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:49:08 DEBUG }, 06:49:08 DEBUG "value": [ 06:49:08 DEBUG 1713592148, 06:49:08 DEBUG "596.657345747199" 06:49:08 DEBUG ] 06:49:08 DEBUG } 06:49:08 DEBUG ] 06:49:08 DEBUG } 06:49:08 DEBUG } 06:49:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592148" 06:49:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592148" 06:49:08 INFO [http_cmd]: http status code OK 06:49:08 DEBUG --- status code --- 06:49:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:49:08 DEBUG --- http response --- 06:49:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:49:08 DEBUG ----- output ----- 06:49:08 DEBUG { 06:49:08 DEBUG "status": "success", 06:49:08 DEBUG "data": { 06:49:08 DEBUG "resultType": "vector", 06:49:08 DEBUG "result": [ 06:49:08 DEBUG { 06:49:08 DEBUG "metric": { 06:49:08 DEBUG "pod": "ds-cts-0" 06:49:08 DEBUG }, 06:49:08 DEBUG "value": [ 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG [...] 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG "pod": "ds-cts-2" 06:49:08 DEBUG }, 06:49:08 DEBUG "value": [ 06:49:08 DEBUG 1713592148, 06:49:08 DEBUG "29.733333333333334" 06:49:08 DEBUG ] 06:49:08 DEBUG } 06:49:08 DEBUG ] 06:49:08 DEBUG } 06:49:08 DEBUG } 06:49:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713592148" 06:49:08 INFO [http_cmd]: http status code OK 06:49:08 DEBUG --- status code --- 06:49:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:49:08 DEBUG --- http response --- 06:49:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:49:08 DEBUG ----- output ----- 06:49:08 DEBUG { 06:49:08 DEBUG "status": "success", 06:49:08 DEBUG "data": { 06:49:08 DEBUG "resultType": "vector", 06:49:08 DEBUG "result": [ 06:49:08 DEBUG { 06:49:08 DEBUG "metric": { 06:49:08 DEBUG "pod": "ds-idrepo-0" 06:49:08 DEBUG }, 06:49:08 DEBUG "value": [ 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG [...] 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG "pod": "ds-idrepo-2" 06:49:08 DEBUG }, 06:49:08 DEBUG "value": [ 06:49:08 DEBUG 1713592148, 06:49:08 DEBUG "0" 06:49:08 DEBUG ] 06:49:08 DEBUG } 06:49:08 DEBUG ] 06:49:08 DEBUG } 06:49:08 DEBUG } 06:49:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592148" 06:49:08 INFO [http_cmd]: http status code OK 06:49:08 DEBUG --- status code --- 06:49:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:49:08 DEBUG --- http response --- 06:49:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:49:08 DEBUG ----- output ----- 06:49:08 DEBUG { 06:49:08 DEBUG "status": "success", 06:49:08 DEBUG "data": { 06:49:08 DEBUG "resultType": "vector", 06:49:08 DEBUG "result": [ 06:49:08 DEBUG { 06:49:08 DEBUG "metric": { 06:49:08 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 06:49:08 DEBUG }, 06:49:08 DEBUG "value": [ 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG [...] 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 06:49:08 DEBUG }, 06:49:08 DEBUG "value": [ 06:49:08 DEBUG 1713592148, 06:49:08 DEBUG "4985966592" 06:49:08 DEBUG ] 06:49:08 DEBUG } 06:49:08 DEBUG ] 06:49:08 DEBUG } 06:49:08 DEBUG } 06:49:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592148" 06:49:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713592148" 06:49:08 INFO [http_cmd]: http status code OK 06:49:08 DEBUG --- status code --- 06:49:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:49:08 DEBUG --- http response --- 06:49:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:49:08 DEBUG ----- output ----- 06:49:08 DEBUG { 06:49:08 DEBUG "status": "success", 06:49:08 DEBUG "data": { 06:49:08 DEBUG "resultType": "vector", 06:49:08 DEBUG "result": [ 06:49:08 DEBUG { 06:49:08 DEBUG "metric": { 06:49:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:49:08 DEBUG }, 06:49:08 DEBUG "value": [ 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG [...] 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:49:08 DEBUG }, 06:49:08 DEBUG "value": [ 06:49:08 DEBUG 1713592148, 06:49:08 DEBUG "479563" 06:49:08 DEBUG ] 06:49:08 DEBUG } 06:49:08 DEBUG ] 06:49:08 DEBUG } 06:49:08 DEBUG } 06:49:08 INFO [http_cmd]: http status code OK 06:49:08 DEBUG --- status code --- 06:49:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:49:08 DEBUG --- http response --- 06:49:08 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 06:49:08 ERROR [http_cmd]: ERROR 06:49:08 ERROR --- status code --- 06:49:08 ERROR http status code is 400 (expected [200, 401, 403, 404]) 06:49:08 ERROR --- http response --- 06:49:08 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 06:49:08 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 06:49:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592148" 06:49:08 INFO [http_cmd]: http status code OK 06:49:08 DEBUG --- status code --- 06:49:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:49:08 DEBUG --- http response --- 06:49:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:49:08 DEBUG ----- output ----- 06:49:08 DEBUG { 06:49:08 DEBUG "status": "success", 06:49:08 DEBUG "data": { 06:49:08 DEBUG "resultType": "vector", 06:49:08 DEBUG "result": [ 06:49:08 DEBUG { 06:49:08 DEBUG "metric": { 06:49:08 DEBUG "pod": "ds-idrepo-0" 06:49:08 DEBUG }, 06:49:08 DEBUG "value": [ 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG [...] 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG "pod": "ds-idrepo-2" 06:49:08 DEBUG }, 06:49:08 DEBUG "value": [ 06:49:08 DEBUG 1713592148, 06:49:08 DEBUG "0" 06:49:08 DEBUG ] 06:49:08 DEBUG } 06:49:08 DEBUG ] 06:49:08 DEBUG } 06:49:08 DEBUG } 06:49:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592148" 06:49:08 INFO [http_cmd]: http status code OK 06:49:08 DEBUG --- status code --- 06:49:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:49:08 DEBUG --- http response --- 06:49:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:49:08 DEBUG ----- output ----- 06:49:08 DEBUG { 06:49:08 DEBUG "status": "success", 06:49:08 DEBUG "data": { 06:49:08 DEBUG "resultType": "vector", 06:49:08 DEBUG "result": [ 06:49:08 DEBUG { 06:49:08 DEBUG "metric": { 06:49:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:49:08 DEBUG }, 06:49:08 DEBUG "value": [ 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG [...] 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:49:08 DEBUG }, 06:49:08 DEBUG "value": [ 06:49:08 DEBUG 1713592148, 06:49:08 DEBUG "2675.1666666666665" 06:49:08 DEBUG ] 06:49:08 DEBUG } 06:49:08 DEBUG ] 06:49:08 DEBUG } 06:49:08 DEBUG } 06:49:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592148" 06:49:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592148" 06:49:08 INFO [http_cmd]: http status code OK 06:49:08 DEBUG --- status code --- 06:49:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:49:08 DEBUG --- http response --- 06:49:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:49:08 DEBUG ----- output ----- 06:49:08 DEBUG { 06:49:08 DEBUG "status": "success", 06:49:08 DEBUG "data": { 06:49:08 DEBUG "resultType": "vector", 06:49:08 DEBUG "result": [ 06:49:08 DEBUG { 06:49:08 DEBUG "metric": { 06:49:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:49:08 DEBUG }, 06:49:08 DEBUG "value": [ 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG [...] 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:49:08 DEBUG }, 06:49:08 DEBUG "value": [ 06:49:08 DEBUG 1713592148, 06:49:08 DEBUG "2680.7" 06:49:08 DEBUG ] 06:49:08 DEBUG } 06:49:08 DEBUG ] 06:49:08 DEBUG } 06:49:08 DEBUG } 06:49:08 INFO [http_cmd]: http status code OK 06:49:08 DEBUG --- status code --- 06:49:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:49:08 DEBUG --- http response --- 06:49:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:49:08 DEBUG ----- output ----- 06:49:08 DEBUG { 06:49:08 DEBUG "status": "success", 06:49:08 DEBUG "data": { 06:49:08 DEBUG "resultType": "vector", 06:49:08 DEBUG "result": [ 06:49:08 DEBUG { 06:49:08 DEBUG "metric": { 06:49:08 DEBUG "pod": "ds-cts-0" 06:49:08 DEBUG }, 06:49:08 DEBUG "value": [ 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG [...] 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG "pod": "ds-cts-2" 06:49:08 DEBUG }, 06:49:08 DEBUG "value": [ 06:49:08 DEBUG 1713592148, 06:49:08 DEBUG "0" 06:49:08 DEBUG ] 06:49:08 DEBUG } 06:49:08 DEBUG ] 06:49:08 DEBUG } 06:49:08 DEBUG } 06:49:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592148" 06:49:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592148" 06:49:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592148" 06:49:08 INFO [http_cmd]: http status code OK 06:49:08 DEBUG --- status code --- 06:49:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:49:08 DEBUG --- http response --- 06:49:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:49:08 DEBUG ----- output ----- 06:49:08 DEBUG { 06:49:08 DEBUG "status": "success", 06:49:08 DEBUG "data": { 06:49:08 DEBUG "resultType": "vector", 06:49:08 DEBUG "result": [ 06:49:08 DEBUG { 06:49:08 DEBUG "metric": { 06:49:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:49:08 DEBUG }, 06:49:08 DEBUG "value": [ 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG [...] 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:49:08 DEBUG }, 06:49:08 DEBUG "value": [ 06:49:08 DEBUG 1713592148, 06:49:08 DEBUG "0" 06:49:08 DEBUG ] 06:49:08 DEBUG } 06:49:08 DEBUG ] 06:49:08 DEBUG } 06:49:08 DEBUG } 06:49:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592148" 06:49:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713592148" 06:49:08 INFO [http_cmd]: http status code OK 06:49:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592148" 06:49:08 DEBUG --- status code --- 06:49:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:49:08 DEBUG --- http response --- 06:49:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:49:08 DEBUG ----- output ----- 06:49:08 DEBUG { 06:49:08 DEBUG "status": "success", 06:49:08 DEBUG "data": { 06:49:08 DEBUG "resultType": "vector", 06:49:08 DEBUG "result": [ 06:49:08 DEBUG { 06:49:08 DEBUG "metric": { 06:49:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:49:08 DEBUG }, 06:49:08 DEBUG "value": [ 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG [...] 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:49:08 DEBUG }, 06:49:08 DEBUG "value": [ 06:49:08 DEBUG 1713592148, 06:49:08 DEBUG "2666.5666666666666" 06:49:08 DEBUG ] 06:49:08 DEBUG } 06:49:08 DEBUG ] 06:49:08 DEBUG } 06:49:08 DEBUG } 06:49:08 INFO [http_cmd]: http status code OK 06:49:08 DEBUG --- status code --- 06:49:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:49:08 DEBUG --- http response --- 06:49:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:49:08 DEBUG ----- output ----- 06:49:08 DEBUG { 06:49:08 DEBUG "status": "success", 06:49:08 DEBUG "data": { 06:49:08 DEBUG "resultType": "vector", 06:49:08 DEBUG "result": [ 06:49:08 DEBUG { 06:49:08 DEBUG "metric": { 06:49:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:49:08 DEBUG }, 06:49:08 DEBUG "value": [ 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG [...] 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:49:08 DEBUG }, 06:49:08 DEBUG "value": [ 06:49:08 DEBUG 1713592148, 06:49:08 DEBUG "2680.2" 06:49:08 DEBUG ] 06:49:08 DEBUG } 06:49:08 DEBUG ] 06:49:08 DEBUG } 06:49:08 DEBUG } 06:49:08 INFO [http_cmd]: http status code OK 06:49:08 DEBUG --- status code --- 06:49:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:49:08 DEBUG --- http response --- 06:49:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:49:08 DEBUG ----- output ----- 06:49:08 DEBUG { 06:49:08 DEBUG "status": "success", 06:49:08 DEBUG "data": { 06:49:08 DEBUG "resultType": "vector", 06:49:08 DEBUG "result": [ 06:49:08 DEBUG { 06:49:08 DEBUG "metric": { 06:49:08 DEBUG "pod": "ds-cts-0" 06:49:08 DEBUG }, 06:49:08 DEBUG "value": [ 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG [...] 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG "pod": "ds-cts-2" 06:49:08 DEBUG }, 06:49:08 DEBUG "value": [ 06:49:08 DEBUG 1713592148, 06:49:08 DEBUG "0.0004333333333333333" 06:49:08 DEBUG ] 06:49:08 DEBUG } 06:49:08 DEBUG ] 06:49:08 DEBUG } 06:49:08 DEBUG } 06:49:08 INFO [http_cmd]: http status code OK 06:49:08 DEBUG --- status code --- 06:49:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:49:08 DEBUG --- http response --- 06:49:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:49:08 DEBUG ----- output ----- 06:49:08 DEBUG { 06:49:08 DEBUG "status": "success", 06:49:08 DEBUG "data": { 06:49:08 DEBUG "resultType": "vector", 06:49:08 DEBUG "result": [ 06:49:08 DEBUG { 06:49:08 DEBUG "metric": { 06:49:08 DEBUG "pod": "ds-idrepo-0" 06:49:08 DEBUG }, 06:49:08 DEBUG "value": [ 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG [...] 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG "pod": "ds-cts-2" 06:49:08 DEBUG }, 06:49:08 DEBUG "value": [ 06:49:08 DEBUG 1713592148, 06:49:08 DEBUG "0" 06:49:08 DEBUG ] 06:49:08 DEBUG } 06:49:08 DEBUG ] 06:49:08 DEBUG } 06:49:08 DEBUG } 06:49:08 INFO [http_cmd]: http status code OK 06:49:08 DEBUG --- status code --- 06:49:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:49:08 DEBUG --- http response --- 06:49:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:49:08 DEBUG ----- output ----- 06:49:08 DEBUG { 06:49:08 DEBUG "status": "success", 06:49:08 DEBUG "data": { 06:49:08 DEBUG "resultType": "vector", 06:49:08 DEBUG "result": [ 06:49:08 DEBUG { 06:49:08 DEBUG "metric": { 06:49:08 DEBUG "pod": "ds-idrepo-0" 06:49:08 DEBUG }, 06:49:08 DEBUG "value": [ 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG [...] 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG "pod": "ds-idrepo-2" 06:49:08 DEBUG }, 06:49:08 DEBUG "value": [ 06:49:08 DEBUG 1713592148, 06:49:08 DEBUG "0" 06:49:08 DEBUG ] 06:49:08 DEBUG } 06:49:08 DEBUG ] 06:49:08 DEBUG } 06:49:08 DEBUG } 06:49:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592148" 06:49:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592148" 06:49:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592148" 06:49:08 INFO [http_cmd]: http status code OK 06:49:08 DEBUG --- status code --- 06:49:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:49:08 DEBUG --- http response --- 06:49:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:49:08 DEBUG ----- output ----- 06:49:08 DEBUG { 06:49:08 DEBUG "status": "success", 06:49:08 DEBUG "data": { 06:49:08 DEBUG "resultType": "vector", 06:49:08 DEBUG "result": [ 06:49:08 DEBUG { 06:49:08 DEBUG "metric": { 06:49:08 DEBUG "instance": "10.142.0.107:9100" 06:49:08 DEBUG }, 06:49:08 DEBUG "value": [ 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG [...] 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG "instance": "10.142.0.43:9100" 06:49:08 DEBUG }, 06:49:08 DEBUG "value": [ 06:49:08 DEBUG 1713592148, 06:49:08 DEBUG "0.2808666666665876" 06:49:08 DEBUG ] 06:49:08 DEBUG } 06:49:08 DEBUG ] 06:49:08 DEBUG } 06:49:08 DEBUG } 06:49:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713592148" 06:49:08 INFO [http_cmd]: http status code OK 06:49:08 INFO [http_cmd]: http status code OK 06:49:08 DEBUG --- status code --- 06:49:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713592148" 06:49:08 DEBUG --- status code --- 06:49:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:49:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:49:08 DEBUG --- http response --- 06:49:08 DEBUG --- http response --- 06:49:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:49:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 06:49:08 DEBUG ----- output ----- 06:49:08 DEBUG ----- output ----- 06:49:08 DEBUG { 06:49:08 DEBUG { 06:49:08 DEBUG "status": "success", 06:49:08 DEBUG "status": "success", 06:49:08 DEBUG "data": { 06:49:08 DEBUG "data": { 06:49:08 DEBUG "resultType": "vector", 06:49:08 DEBUG "resultType": "vector", 06:49:08 DEBUG "result": [ 06:49:08 DEBUG "result": [ 06:49:08 DEBUG { 06:49:08 DEBUG { 06:49:08 DEBUG "metric": { 06:49:08 DEBUG "metric": { 06:49:08 DEBUG "instance": "10.142.0.107:9100" 06:49:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:49:08 DEBUG }, 06:49:08 DEBUG }, 06:49:08 DEBUG "value": [ 06:49:08 DEBUG "value": [ 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG [...] 06:49:08 DEBUG [...] 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG "instance": "10.142.0.43:9100" 06:49:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:49:08 DEBUG }, 06:49:08 DEBUG }, 06:49:08 DEBUG "value": [ 06:49:08 DEBUG "value": [ 06:49:08 DEBUG 1713592148, 06:49:08 DEBUG 1713592148, 06:49:08 DEBUG "80932745.43333334" 06:49:08 DEBUG "24164.62250276156" 06:49:08 DEBUG ] 06:49:08 DEBUG ] 06:49:08 DEBUG } 06:49:08 DEBUG } 06:49:08 DEBUG ] 06:49:08 DEBUG ] 06:49:08 INFO [http_cmd]: http status code OK 06:49:08 DEBUG } 06:49:08 DEBUG } 06:49:08 DEBUG --- status code --- 06:49:08 DEBUG } 06:49:08 DEBUG } 06:49:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:49:08 DEBUG --- http response --- 06:49:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:49:08 DEBUG ----- output ----- 06:49:08 DEBUG { 06:49:08 DEBUG "status": "success", 06:49:08 DEBUG "data": { 06:49:08 DEBUG "resultType": "vector", 06:49:08 DEBUG "result": [ 06:49:08 DEBUG { 06:49:08 DEBUG "metric": { 06:49:08 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 06:49:08 DEBUG }, 06:49:08 DEBUG "value": [ 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG [...] 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 06:49:08 DEBUG }, 06:49:08 DEBUG "value": [ 06:49:08 DEBUG 1713592148, 06:49:08 DEBUG "8.040432218214766" 06:49:08 DEBUG ] 06:49:08 DEBUG } 06:49:08 DEBUG ] 06:49:08 DEBUG } 06:49:08 DEBUG } 06:49:08 INFO [http_cmd]: http status code OK 06:49:08 DEBUG --- status code --- 06:49:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:49:08 DEBUG --- http response --- 06:49:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:49:08 DEBUG ----- output ----- 06:49:08 DEBUG { 06:49:08 DEBUG "status": "success", 06:49:08 DEBUG "data": { 06:49:08 DEBUG "resultType": "vector", 06:49:08 DEBUG "result": [ 06:49:08 DEBUG { 06:49:08 DEBUG "metric": { 06:49:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:49:08 DEBUG }, 06:49:08 DEBUG "value": [ 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG [...] 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:49:08 DEBUG }, 06:49:08 DEBUG "value": [ 06:49:08 DEBUG 1713592148, 06:49:08 DEBUG "0.096" 06:49:08 DEBUG ] 06:49:08 DEBUG } 06:49:08 DEBUG ] 06:49:08 DEBUG } 06:49:08 DEBUG } 06:49:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592148" 06:49:08 INFO [http_cmd]: http status code OK 06:49:08 DEBUG --- status code --- 06:49:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:49:08 DEBUG --- http response --- 06:49:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:49:08 DEBUG ----- output ----- 06:49:08 DEBUG { 06:49:08 DEBUG "status": "success", 06:49:08 DEBUG "data": { 06:49:08 DEBUG "resultType": "vector", 06:49:08 DEBUG "result": [ 06:49:08 DEBUG { 06:49:08 DEBUG "metric": { 06:49:08 DEBUG "instance": "10.142.0.107:9100" 06:49:08 DEBUG }, 06:49:08 DEBUG "value": [ 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG [...] 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG "instance": "10.142.0.43:9100" 06:49:08 DEBUG }, 06:49:08 DEBUG "value": [ 06:49:08 DEBUG 1713592148, 06:49:08 DEBUG "80927473.4" 06:49:08 DEBUG ] 06:49:08 DEBUG } 06:49:08 DEBUG ] 06:49:08 DEBUG } 06:49:08 DEBUG } 06:49:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592148" 06:49:08 INFO [http_cmd]: http status code OK 06:49:08 DEBUG --- status code --- 06:49:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:49:08 DEBUG --- http response --- 06:49:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 06:49:08 DEBUG ----- output ----- 06:49:08 DEBUG { 06:49:08 DEBUG "status": "success", 06:49:08 DEBUG "data": { 06:49:08 DEBUG "resultType": "vector", 06:49:08 DEBUG "result": [ 06:49:08 DEBUG { 06:49:08 DEBUG "metric": { 06:49:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:49:08 DEBUG }, 06:49:08 DEBUG "value": [ 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG [...] 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:49:08 DEBUG }, 06:49:08 DEBUG "value": [ 06:49:08 DEBUG 1713592148, 06:49:08 DEBUG "11.943160190117942" 06:49:08 DEBUG ] 06:49:08 DEBUG } 06:49:08 DEBUG ] 06:49:08 DEBUG } 06:49:08 DEBUG } 06:49:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592148" 06:49:08 INFO [http_cmd]: http status code OK 06:49:08 DEBUG --- status code --- 06:49:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:49:08 DEBUG --- http response --- 06:49:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:49:08 DEBUG ----- output ----- 06:49:08 DEBUG { 06:49:08 DEBUG "status": "success", 06:49:08 DEBUG "data": { 06:49:08 DEBUG "resultType": "vector", 06:49:08 DEBUG "result": [ 06:49:08 DEBUG { 06:49:08 DEBUG "metric": { 06:49:08 DEBUG "instance": "10.142.0.107:9100" 06:49:08 DEBUG }, 06:49:08 DEBUG "value": [ 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG [...] 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG 06:49:08 DEBUG "instance": "10.142.0.43:9100" 06:49:08 DEBUG }, 06:49:08 DEBUG "value": [ 06:49:08 DEBUG 1713592148, 06:49:08 DEBUG "0.11566666666666757" 06:49:08 DEBUG ] 06:49:08 DEBUG } 06:49:08 DEBUG ] 06:49:08 DEBUG } 06:49:08 DEBUG } 06:49:09 INFO 06:49:09 INFO [loop_until]: kubectl --namespace=xlou top pods 06:49:09 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:49:09 INFO [loop_until]: OK (rc = 0) 06:49:09 DEBUG --- stdout --- 06:49:09 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 17374m 8649Mi am-767f6b67b5-qcqz2 17768m 8715Mi am-767f6b67b5-vbltz 18731m 8655Mi ds-cts-0 19023m 10375Mi ds-cts-1 17130m 10663Mi ds-cts-2 19127m 10249Mi ds-idrepo-0 7850m 3747Mi ds-idrepo-1 7475m 3802Mi ds-idrepo-2 7567m 3691Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1177Mi idm-b85f9d978-wnhxb 3m 1236Mi lodemon-7cb5b9456d-dskfn 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7956m 4661Mi 06:49:09 DEBUG --- stderr --- 06:49:09 DEBUG 06:49:09 INFO 06:49:09 INFO [loop_until]: kubectl --namespace=xlou top node 06:49:09 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:49:09 INFO [loop_until]: OK (rc = 0) 06:49:09 DEBUG --- stdout --- 06:49:09 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 67m 0% 6274Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7956m 26% 7423Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 17181m 57% 11360Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 83m 0% 3362Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 19240m 64% 13519Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 154m 0% 5195Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 17472m 58% 11607Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 19000m 63% 3007Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 19659m 65% 10862Mi 9% gke-xlou-cdm-ds-0f601751-j488 8159m 27% 7017Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 17305m 57% 15598Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 8060m 26% 7703Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 19453m 65% 12828Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 8163m 51% 8493Mi 14% 06:49:09 DEBUG --- stderr --- 06:49:09 DEBUG 06:50:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592208" 06:50:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713592208" 06:50:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592208" 06:50:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592208" 06:50:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592208" 06:50:08 INFO [http_cmd]: http status code OK 06:50:08 INFO [http_cmd]: http status code OK 06:50:08 DEBUG --- status code --- 06:50:08 DEBUG --- status code --- 06:50:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:50:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:50:08 DEBUG --- http response --- 06:50:08 DEBUG --- http response --- 06:50:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 06:50:08 DEBUG ----- output ----- 06:50:08 DEBUG { 06:50:08 DEBUG "status": "success", 06:50:08 DEBUG "data": { 06:50:08 DEBUG "resultType": "vector", 06:50:08 DEBUG "result": [ 06:50:08 DEBUG { 06:50:08 DEBUG "metric": { 06:50:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:50:08 DEBUG }, 06:50:08 DEBUG "value": [ 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG [...] 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG "pod": "ds-idrepo-2" 06:50:08 DEBUG }, 06:50:08 DEBUG "value": [ 06:50:08 DEBUG 1713592208, 06:50:08 DEBUG "0" 06:50:08 DEBUG ] 06:50:08 DEBUG } 06:50:08 DEBUG ] 06:50:08 DEBUG } 06:50:08 DEBUG } 06:50:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:50:08 DEBUG ----- output ----- 06:50:08 DEBUG { 06:50:08 DEBUG "status": "success", 06:50:08 DEBUG "data": { 06:50:08 DEBUG "resultType": "vector", 06:50:08 DEBUG "result": [ 06:50:08 DEBUG { 06:50:08 DEBUG "metric": { 06:50:08 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 06:50:08 DEBUG }, 06:50:08 DEBUG "value": [ 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG [...] 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 06:50:08 DEBUG }, 06:50:08 DEBUG "value": [ 06:50:08 DEBUG 1713592208, 06:50:08 DEBUG "4988559360" 06:50:08 DEBUG ] 06:50:08 DEBUG } 06:50:08 DEBUG ] 06:50:08 DEBUG } 06:50:08 DEBUG } 06:50:08 INFO [http_cmd]: http status code OK 06:50:08 INFO [http_cmd]: http status code OK 06:50:08 DEBUG --- status code --- 06:50:08 DEBUG --- status code --- 06:50:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:50:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:50:08 DEBUG --- http response --- 06:50:08 DEBUG --- http response --- 06:50:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:50:08 DEBUG ----- output ----- 06:50:08 DEBUG { 06:50:08 DEBUG "status": "success", 06:50:08 DEBUG "data": { 06:50:08 DEBUG "resultType": "vector", 06:50:08 DEBUG "result": [ 06:50:08 DEBUG { 06:50:08 DEBUG "metric": { 06:50:08 DEBUG "pod": "ds-idrepo-0" 06:50:08 DEBUG }, 06:50:08 DEBUG "value": [ 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG [...] 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG "pod": "ds-idrepo-2" 06:50:08 DEBUG }, 06:50:08 DEBUG "value": [ 06:50:08 DEBUG 1713592208, 06:50:08 DEBUG "0" 06:50:08 DEBUG ] 06:50:08 DEBUG } 06:50:08 DEBUG ] 06:50:08 DEBUG } 06:50:08 DEBUG } 06:50:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:50:08 DEBUG ----- output ----- 06:50:08 DEBUG { 06:50:08 DEBUG "status": "success", 06:50:08 DEBUG "data": { 06:50:08 DEBUG "resultType": "vector", 06:50:08 DEBUG "result": [ 06:50:08 DEBUG { 06:50:08 DEBUG "metric": { 06:50:08 DEBUG "instance": "10.142.0.107:9100" 06:50:08 DEBUG }, 06:50:08 DEBUG "value": [ 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG [...] 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG "instance": "10.142.0.43:9100" 06:50:08 DEBUG }, 06:50:08 DEBUG "value": [ 06:50:08 DEBUG 1713592208, 06:50:08 DEBUG "3222200015872" 06:50:08 DEBUG ] 06:50:08 DEBUG } 06:50:08 DEBUG ] 06:50:08 DEBUG } 06:50:08 DEBUG } 06:50:08 INFO [http_cmd]: http status code OK 06:50:08 DEBUG --- status code --- 06:50:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:50:08 DEBUG --- http response --- 06:50:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:50:08 DEBUG ----- output ----- 06:50:08 DEBUG { 06:50:08 DEBUG "status": "success", 06:50:08 DEBUG "data": { 06:50:08 DEBUG "resultType": "vector", 06:50:08 DEBUG "result": [ 06:50:08 DEBUG { 06:50:08 DEBUG "metric": { 06:50:08 DEBUG "pod": "ds-cts-0" 06:50:08 DEBUG }, 06:50:08 DEBUG "value": [ 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG [...] 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG "pod": "ds-cts-2" 06:50:08 DEBUG }, 06:50:08 DEBUG "value": [ 06:50:08 DEBUG 1713592208, 06:50:08 DEBUG "30.266666666666666" 06:50:08 DEBUG ] 06:50:08 DEBUG } 06:50:08 DEBUG ] 06:50:08 DEBUG } 06:50:08 DEBUG } 06:50:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592208" 06:50:08 INFO [http_cmd]: http status code OK 06:50:08 DEBUG --- status code --- 06:50:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:50:08 DEBUG --- http response --- 06:50:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 06:50:08 DEBUG ----- output ----- 06:50:08 DEBUG { 06:50:08 DEBUG "status": "success", 06:50:08 DEBUG "data": { 06:50:08 DEBUG "resultType": "vector", 06:50:08 DEBUG "result": [ 06:50:08 DEBUG { 06:50:08 DEBUG "metric": { 06:50:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:50:08 DEBUG }, 06:50:08 DEBUG "value": [ 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG [...] 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG "pod": "ds-idrepo-2" 06:50:08 DEBUG }, 06:50:08 DEBUG "value": [ 06:50:08 DEBUG 1713592208, 06:50:08 DEBUG "2900.799192327107" 06:50:08 DEBUG ] 06:50:08 DEBUG } 06:50:08 DEBUG ] 06:50:08 DEBUG } 06:50:08 DEBUG } 06:50:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713592208" 06:50:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592208" 06:50:08 ERROR [http_cmd]: ERROR 06:50:08 ERROR --- status code --- 06:50:08 ERROR http status code is 400 (expected [200, 401, 403, 404]) 06:50:08 ERROR --- http response --- 06:50:08 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 06:50:08 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 06:50:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592208" 06:50:08 INFO [http_cmd]: http status code OK 06:50:08 DEBUG --- status code --- 06:50:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:50:08 DEBUG --- http response --- 06:50:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:50:08 DEBUG ----- output ----- 06:50:08 DEBUG { 06:50:08 DEBUG "status": "success", 06:50:08 DEBUG "data": { 06:50:08 DEBUG "resultType": "vector", 06:50:08 DEBUG "result": [ 06:50:08 DEBUG { 06:50:08 DEBUG "metric": { 06:50:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:50:08 DEBUG }, 06:50:08 DEBUG "value": [ 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG [...] 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:50:08 DEBUG }, 06:50:08 DEBUG "value": [ 06:50:08 DEBUG 1713592208, 06:50:08 DEBUG "480135" 06:50:08 DEBUG ] 06:50:08 DEBUG } 06:50:08 DEBUG ] 06:50:08 DEBUG } 06:50:08 DEBUG } 06:50:08 INFO [http_cmd]: http status code OK 06:50:08 DEBUG --- status code --- 06:50:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:50:08 DEBUG --- http response --- 06:50:08 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 06:50:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592208" 06:50:08 INFO [http_cmd]: http status code OK 06:50:08 DEBUG --- status code --- 06:50:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:50:08 DEBUG --- http response --- 06:50:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:50:08 DEBUG ----- output ----- 06:50:08 DEBUG { 06:50:08 DEBUG "status": "success", 06:50:08 DEBUG "data": { 06:50:08 DEBUG "resultType": "vector", 06:50:08 DEBUG "result": [ 06:50:08 DEBUG { 06:50:08 DEBUG "metric": { 06:50:08 DEBUG "pod": "ds-idrepo-0" 06:50:08 DEBUG }, 06:50:08 DEBUG "value": [ 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG [...] 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG "pod": "ds-idrepo-2" 06:50:08 DEBUG }, 06:50:08 DEBUG "value": [ 06:50:08 DEBUG 1713592208, 06:50:08 DEBUG "0" 06:50:08 DEBUG ] 06:50:08 DEBUG } 06:50:08 DEBUG ] 06:50:08 DEBUG } 06:50:08 DEBUG } 06:50:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592208" 06:50:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592208" 06:50:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592208" 06:50:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592208" 06:50:08 INFO [http_cmd]: http status code OK 06:50:08 INFO [http_cmd]: http status code OK 06:50:08 DEBUG --- status code --- 06:50:08 DEBUG --- status code --- 06:50:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:50:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:50:08 DEBUG --- http response --- 06:50:08 DEBUG --- http response --- 06:50:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:50:08 DEBUG ----- output ----- 06:50:08 DEBUG { 06:50:08 DEBUG "status": "success", 06:50:08 DEBUG "data": { 06:50:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:50:08 DEBUG "resultType": "vector", 06:50:08 DEBUG ----- output ----- 06:50:08 DEBUG "result": [ 06:50:08 DEBUG { 06:50:08 DEBUG { 06:50:08 DEBUG "status": "success", 06:50:08 DEBUG "metric": { 06:50:08 DEBUG "data": { 06:50:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:50:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592208" 06:50:08 DEBUG "resultType": "vector", 06:50:08 DEBUG }, 06:50:08 DEBUG "result": [ 06:50:08 DEBUG "value": [ 06:50:08 DEBUG { 06:50:08 DEBUG 06:50:08 DEBUG "metric": { 06:50:08 DEBUG 06:50:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:50:08 DEBUG 06:50:08 DEBUG }, 06:50:08 DEBUG [...] 06:50:08 DEBUG "value": [ 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:50:08 DEBUG [...] 06:50:08 DEBUG }, 06:50:08 DEBUG 06:50:08 DEBUG "value": [ 06:50:08 DEBUG 06:50:08 DEBUG 1713592208, 06:50:08 DEBUG 06:50:08 DEBUG "2676.633333333333" 06:50:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:50:08 DEBUG ] 06:50:08 DEBUG }, 06:50:08 DEBUG } 06:50:08 DEBUG "value": [ 06:50:08 DEBUG ] 06:50:08 DEBUG 1713592208, 06:50:08 DEBUG } 06:50:08 DEBUG "0" 06:50:08 DEBUG } 06:50:08 DEBUG ] 06:50:08 DEBUG } 06:50:08 DEBUG ] 06:50:08 DEBUG } 06:50:08 DEBUG } 06:50:08 INFO [http_cmd]: http status code OK 06:50:08 INFO [http_cmd]: http status code OK 06:50:08 DEBUG --- status code --- 06:50:08 DEBUG --- status code --- 06:50:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:50:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:50:08 DEBUG --- http response --- 06:50:08 DEBUG --- http response --- 06:50:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:50:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:50:08 DEBUG ----- output ----- 06:50:08 DEBUG ----- output ----- 06:50:08 DEBUG { 06:50:08 DEBUG { 06:50:08 DEBUG "status": "success", 06:50:08 DEBUG "status": "success", 06:50:08 DEBUG "data": { 06:50:08 DEBUG "data": { 06:50:08 DEBUG "resultType": "vector", 06:50:08 DEBUG "resultType": "vector", 06:50:08 DEBUG "result": [ 06:50:08 DEBUG "result": [ 06:50:08 DEBUG { 06:50:08 DEBUG { 06:50:08 DEBUG "metric": { 06:50:08 DEBUG "metric": { 06:50:08 DEBUG "pod": "ds-cts-0" 06:50:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:50:08 DEBUG }, 06:50:08 DEBUG }, 06:50:08 DEBUG "value": [ 06:50:08 DEBUG "value": [ 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG [...] 06:50:08 DEBUG [...] 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592208" 06:50:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592208" 06:50:08 DEBUG 06:50:08 DEBUG "pod": "ds-cts-2" 06:50:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:50:08 DEBUG }, 06:50:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713592208" 06:50:08 DEBUG "value": [ 06:50:08 DEBUG }, 06:50:08 INFO [http_cmd]: http status code OK 06:50:08 DEBUG 1713592208, 06:50:08 DEBUG "value": [ 06:50:08 DEBUG --- status code --- 06:50:08 DEBUG "2677" 06:50:08 DEBUG 1713592208, 06:50:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:50:08 DEBUG ] 06:50:08 DEBUG "0.00026666666666666673" 06:50:08 DEBUG --- http response --- 06:50:08 DEBUG } 06:50:08 DEBUG ] 06:50:08 DEBUG ] 06:50:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:50:08 DEBUG } 06:50:08 DEBUG } 06:50:08 DEBUG ----- output ----- 06:50:08 DEBUG ] 06:50:08 DEBUG } 06:50:08 DEBUG { 06:50:08 DEBUG } 06:50:08 DEBUG "status": "success", 06:50:08 DEBUG } 06:50:08 DEBUG "data": { 06:50:08 DEBUG "resultType": "vector", 06:50:08 DEBUG "result": [ 06:50:08 DEBUG { 06:50:08 DEBUG "metric": { 06:50:08 DEBUG "pod": "ds-cts-0" 06:50:08 DEBUG }, 06:50:08 DEBUG "value": [ 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG [...] 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG "pod": "ds-cts-2" 06:50:08 DEBUG }, 06:50:08 DEBUG "value": [ 06:50:08 DEBUG 1713592208, 06:50:08 DEBUG "0" 06:50:08 DEBUG ] 06:50:08 DEBUG } 06:50:08 DEBUG ] 06:50:08 DEBUG } 06:50:08 DEBUG } 06:50:08 INFO [http_cmd]: http status code OK 06:50:08 DEBUG --- status code --- 06:50:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:50:08 DEBUG --- http response --- 06:50:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 06:50:08 DEBUG ----- output ----- 06:50:08 DEBUG { 06:50:08 DEBUG "status": "success", 06:50:08 DEBUG "data": { 06:50:08 DEBUG "resultType": "vector", 06:50:08 DEBUG "result": [ 06:50:08 DEBUG { 06:50:08 DEBUG "metric": { 06:50:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:50:08 DEBUG }, 06:50:08 DEBUG "value": [ 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG [...] 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG "pod": "ds-idrepo-2" 06:50:08 DEBUG }, 06:50:08 DEBUG "value": [ 06:50:08 DEBUG 1713592208, 06:50:08 DEBUG "126400651.26765811" 06:50:08 DEBUG ] 06:50:08 DEBUG } 06:50:08 DEBUG ] 06:50:08 DEBUG } 06:50:08 DEBUG } 06:50:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592208" 06:50:08 INFO [http_cmd]: http status code OK 06:50:08 DEBUG --- status code --- 06:50:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:50:08 DEBUG --- http response --- 06:50:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:50:08 DEBUG ----- output ----- 06:50:08 DEBUG { 06:50:08 DEBUG "status": "success", 06:50:08 DEBUG "data": { 06:50:08 DEBUG "resultType": "vector", 06:50:08 DEBUG "result": [ 06:50:08 DEBUG { 06:50:08 DEBUG "metric": { 06:50:08 DEBUG "instance": "10.142.0.107:9100" 06:50:08 DEBUG }, 06:50:08 DEBUG "value": [ 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG [...] 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG "instance": "10.142.0.43:9100" 06:50:08 DEBUG }, 06:50:08 DEBUG "value": [ 06:50:08 DEBUG 1713592208, 06:50:08 DEBUG "80856497.8" 06:50:08 DEBUG ] 06:50:08 DEBUG } 06:50:08 DEBUG ] 06:50:08 DEBUG } 06:50:08 DEBUG } 06:50:08 INFO [http_cmd]: http status code OK 06:50:08 DEBUG --- status code --- 06:50:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:50:08 DEBUG --- http response --- 06:50:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:50:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592208" 06:50:08 DEBUG ----- output ----- 06:50:08 DEBUG { 06:50:08 DEBUG "status": "success", 06:50:08 DEBUG "data": { 06:50:08 DEBUG "resultType": "vector", 06:50:08 DEBUG "result": [ 06:50:08 DEBUG { 06:50:08 DEBUG "metric": { 06:50:08 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 06:50:08 DEBUG }, 06:50:08 DEBUG "value": [ 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG [...] 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 06:50:08 DEBUG }, 06:50:08 DEBUG "value": [ 06:50:08 DEBUG 1713592208, 06:50:08 DEBUG "8.041516246571108" 06:50:08 DEBUG ] 06:50:08 DEBUG } 06:50:08 DEBUG ] 06:50:08 DEBUG } 06:50:08 DEBUG } 06:50:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713592208" 06:50:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592208" 06:50:08 INFO [http_cmd]: http status code OK 06:50:08 DEBUG --- status code --- 06:50:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:50:08 DEBUG --- http response --- 06:50:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:50:08 DEBUG ----- output ----- 06:50:08 DEBUG { 06:50:08 DEBUG "status": "success", 06:50:08 DEBUG "data": { 06:50:08 DEBUG "resultType": "vector", 06:50:08 DEBUG "result": [ 06:50:08 DEBUG { 06:50:08 DEBUG "metric": { 06:50:08 DEBUG "pod": "ds-idrepo-0" 06:50:08 DEBUG }, 06:50:08 DEBUG "value": [ 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG [...] 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG "pod": "ds-idrepo-2" 06:50:08 DEBUG }, 06:50:08 DEBUG "value": [ 06:50:08 DEBUG 1713592208, 06:50:08 DEBUG "0" 06:50:08 DEBUG ] 06:50:08 DEBUG } 06:50:08 DEBUG ] 06:50:08 DEBUG } 06:50:08 DEBUG } 06:50:08 INFO [http_cmd]: http status code OK 06:50:08 INFO [http_cmd]: http status code OK 06:50:08 DEBUG --- status code --- 06:50:08 DEBUG --- status code --- 06:50:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:50:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:50:08 DEBUG --- http response --- 06:50:08 DEBUG --- http response --- 06:50:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:50:08 DEBUG ----- output ----- 06:50:08 DEBUG { 06:50:08 DEBUG "status": "success", 06:50:08 DEBUG "data": { 06:50:08 DEBUG "resultType": "vector", 06:50:08 DEBUG "result": [ 06:50:08 DEBUG { 06:50:08 DEBUG "metric": { 06:50:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:50:08 DEBUG }, 06:50:08 DEBUG "value": [ 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG [...] 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:50:08 DEBUG }, 06:50:08 DEBUG "value": [ 06:50:08 DEBUG 1713592208, 06:50:08 DEBUG "2659.8" 06:50:08 DEBUG ] 06:50:08 DEBUG } 06:50:08 DEBUG ] 06:50:08 DEBUG } 06:50:08 DEBUG } 06:50:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:50:08 DEBUG ----- output ----- 06:50:08 DEBUG { 06:50:08 DEBUG "status": "success", 06:50:08 DEBUG "data": { 06:50:08 DEBUG "resultType": "vector", 06:50:08 DEBUG "result": [ 06:50:08 DEBUG { 06:50:08 DEBUG "metric": { 06:50:08 DEBUG "pod": "ds-idrepo-0" 06:50:08 DEBUG }, 06:50:08 DEBUG "value": [ 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG [...] 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG "pod": "ds-cts-2" 06:50:08 DEBUG }, 06:50:08 DEBUG "value": [ 06:50:08 DEBUG 1713592208, 06:50:08 DEBUG "0" 06:50:08 DEBUG ] 06:50:08 DEBUG } 06:50:08 DEBUG ] 06:50:08 DEBUG } 06:50:08 DEBUG } 06:50:08 INFO [http_cmd]: http status code OK 06:50:08 DEBUG --- status code --- 06:50:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:50:08 DEBUG --- http response --- 06:50:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:50:08 DEBUG ----- output ----- 06:50:08 DEBUG { 06:50:08 DEBUG "status": "success", 06:50:08 DEBUG "data": { 06:50:08 DEBUG "resultType": "vector", 06:50:08 DEBUG "result": [ 06:50:08 DEBUG { 06:50:08 DEBUG "metric": { 06:50:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:50:08 DEBUG }, 06:50:08 DEBUG "value": [ 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG [...] 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:50:08 DEBUG }, 06:50:08 DEBUG "value": [ 06:50:08 DEBUG 1713592208, 06:50:08 DEBUG "2676.733333333333" 06:50:08 DEBUG ] 06:50:08 DEBUG } 06:50:08 DEBUG ] 06:50:08 DEBUG } 06:50:08 DEBUG } 06:50:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592208" 06:50:08 INFO [http_cmd]: http status code OK 06:50:08 DEBUG --- status code --- 06:50:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:50:08 DEBUG --- http response --- 06:50:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:50:08 DEBUG ----- output ----- 06:50:08 DEBUG { 06:50:08 DEBUG "status": "success", 06:50:08 DEBUG "data": { 06:50:08 DEBUG "resultType": "vector", 06:50:08 DEBUG "result": [ 06:50:08 DEBUG { 06:50:08 DEBUG "metric": { 06:50:08 DEBUG "instance": "10.142.0.107:9100" 06:50:08 DEBUG }, 06:50:08 DEBUG "value": [ 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG [...] 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG "instance": "10.142.0.43:9100" 06:50:08 DEBUG }, 06:50:08 DEBUG "value": [ 06:50:08 DEBUG 1713592208, 06:50:08 DEBUG "0.2597333333332538" 06:50:08 DEBUG ] 06:50:08 DEBUG } 06:50:08 DEBUG ] 06:50:08 DEBUG } 06:50:08 DEBUG } 06:50:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713592208" 06:50:08 INFO [http_cmd]: http status code OK 06:50:08 DEBUG --- status code --- 06:50:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:50:08 DEBUG --- http response --- 06:50:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:50:08 DEBUG ----- output ----- 06:50:08 DEBUG { 06:50:08 DEBUG "status": "success", 06:50:08 DEBUG "data": { 06:50:08 DEBUG "resultType": "vector", 06:50:08 DEBUG "result": [ 06:50:08 DEBUG { 06:50:08 DEBUG "metric": { 06:50:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:50:08 DEBUG }, 06:50:08 DEBUG "value": [ 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG [...] 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:50:08 DEBUG }, 06:50:08 DEBUG "value": [ 06:50:08 DEBUG 1713592208, 06:50:08 DEBUG "0.096" 06:50:08 DEBUG ] 06:50:08 DEBUG } 06:50:08 DEBUG ] 06:50:08 DEBUG } 06:50:08 DEBUG } 06:50:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592208" 06:50:08 INFO [http_cmd]: http status code OK 06:50:08 DEBUG --- status code --- 06:50:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:50:08 DEBUG --- http response --- 06:50:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:50:08 DEBUG ----- output ----- 06:50:08 DEBUG { 06:50:08 DEBUG "status": "success", 06:50:08 DEBUG "data": { 06:50:08 DEBUG "resultType": "vector", 06:50:08 DEBUG "result": [ 06:50:08 DEBUG { 06:50:08 DEBUG "metric": { 06:50:08 DEBUG "instance": "10.142.0.107:9100" 06:50:08 DEBUG }, 06:50:08 DEBUG "value": [ 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG [...] 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG "instance": "10.142.0.43:9100" 06:50:08 DEBUG }, 06:50:08 DEBUG "value": [ 06:50:08 DEBUG 1713592208, 06:50:08 DEBUG "80853233.46666667" 06:50:08 DEBUG ] 06:50:08 DEBUG } 06:50:08 DEBUG ] 06:50:08 DEBUG } 06:50:08 DEBUG } 06:50:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592208" 06:50:08 INFO [http_cmd]: http status code OK 06:50:08 DEBUG --- status code --- 06:50:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:50:08 DEBUG --- http response --- 06:50:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:50:08 DEBUG ----- output ----- 06:50:08 DEBUG { 06:50:08 DEBUG "status": "success", 06:50:08 DEBUG "data": { 06:50:08 DEBUG "resultType": "vector", 06:50:08 DEBUG "result": [ 06:50:08 DEBUG { 06:50:08 DEBUG "metric": { 06:50:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:50:08 DEBUG }, 06:50:08 DEBUG "value": [ 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG [...] 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG "pod": "ds-idrepo-2" 06:50:08 DEBUG }, 06:50:08 DEBUG "value": [ 06:50:08 DEBUG 1713592208, 06:50:08 DEBUG "5.435963073387576" 06:50:08 DEBUG ] 06:50:08 DEBUG } 06:50:08 DEBUG ] 06:50:08 DEBUG } 06:50:08 DEBUG } 06:50:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592208" 06:50:08 INFO [http_cmd]: http status code OK 06:50:08 DEBUG --- status code --- 06:50:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:50:08 DEBUG --- http response --- 06:50:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:50:08 DEBUG ----- output ----- 06:50:08 DEBUG { 06:50:08 DEBUG "status": "success", 06:50:08 DEBUG "data": { 06:50:08 DEBUG "resultType": "vector", 06:50:08 DEBUG "result": [ 06:50:08 DEBUG { 06:50:08 DEBUG "metric": { 06:50:08 DEBUG "instance": "10.142.0.107:9100" 06:50:08 DEBUG }, 06:50:08 DEBUG "value": [ 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG [...] 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG 06:50:08 DEBUG "instance": "10.142.0.43:9100" 06:50:08 DEBUG }, 06:50:08 DEBUG "value": [ 06:50:08 DEBUG 1713592208, 06:50:08 DEBUG "0.10266666666666045" 06:50:08 DEBUG ] 06:50:08 DEBUG } 06:50:08 DEBUG ] 06:50:08 DEBUG } 06:50:08 DEBUG } 06:50:09 INFO 06:50:09 INFO [loop_until]: kubectl --namespace=xlou top pods 06:50:09 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:50:09 INFO 06:50:09 INFO [loop_until]: kubectl --namespace=xlou top node 06:50:09 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:50:09 INFO [loop_until]: OK (rc = 0) 06:50:09 DEBUG --- stdout --- 06:50:09 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 70m 0% 6282Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 8053m 26% 7425Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 17224m 57% 11360Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 79m 0% 3370Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 18038m 60% 13524Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 155m 0% 5200Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 17492m 58% 11595Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 18816m 63% 3019Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 19546m 65% 10870Mi 9% gke-xlou-cdm-ds-0f601751-j488 8208m 27% 7012Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 17360m 58% 15590Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 8079m 27% 7698Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 19164m 64% 12870Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 8175m 51% 8501Mi 14% 06:50:09 DEBUG --- stderr --- 06:50:09 DEBUG 06:50:09 INFO [loop_until]: OK (rc = 0) 06:50:09 DEBUG --- stdout --- 06:50:09 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 17350m 8652Mi am-767f6b67b5-qcqz2 16955m 8714Mi am-767f6b67b5-vbltz 19180m 8654Mi ds-cts-0 18081m 10391Mi ds-cts-1 17365m 10670Mi ds-cts-2 19479m 10279Mi ds-idrepo-0 7612m 3748Mi ds-idrepo-1 7579m 3802Mi ds-idrepo-2 7810m 3691Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1177Mi idm-b85f9d978-wnhxb 4m 1237Mi lodemon-7cb5b9456d-dskfn 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 8044m 4664Mi 06:50:09 DEBUG --- stderr --- 06:50:09 DEBUG 06:51:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713592268" 06:51:08 INFO [http_cmd]: http status code OK 06:51:08 DEBUG --- status code --- 06:51:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:51:08 DEBUG --- http response --- 06:51:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592268" 06:51:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:51:08 DEBUG ----- output ----- 06:51:08 DEBUG { 06:51:08 DEBUG "status": "success", 06:51:08 DEBUG "data": { 06:51:08 DEBUG "resultType": "vector", 06:51:08 DEBUG "result": [ 06:51:08 DEBUG { 06:51:08 DEBUG "metric": { 06:51:08 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 06:51:08 DEBUG }, 06:51:08 DEBUG "value": [ 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG [...] 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 06:51:08 DEBUG }, 06:51:08 DEBUG "value": [ 06:51:08 DEBUG 1713592268, 06:51:08 DEBUG "4989116416" 06:51:08 DEBUG ] 06:51:08 DEBUG } 06:51:08 DEBUG ] 06:51:08 DEBUG } 06:51:08 DEBUG } 06:51:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592268" 06:51:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592268" 06:51:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592268" 06:51:08 INFO [http_cmd]: http status code OK 06:51:08 DEBUG --- status code --- 06:51:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:51:08 DEBUG --- http response --- 06:51:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 06:51:08 DEBUG ----- output ----- 06:51:08 DEBUG { 06:51:08 DEBUG "status": "success", 06:51:08 DEBUG "data": { 06:51:08 DEBUG "resultType": "vector", 06:51:08 DEBUG "result": [ 06:51:08 DEBUG { 06:51:08 DEBUG "metric": { 06:51:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:51:08 DEBUG }, 06:51:08 DEBUG "value": [ 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG [...] 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 06:51:08 DEBUG }, 06:51:08 DEBUG "value": [ 06:51:08 DEBUG 1713592268, 06:51:08 DEBUG "0" 06:51:08 DEBUG ] 06:51:08 DEBUG } 06:51:08 DEBUG ] 06:51:08 DEBUG } 06:51:08 DEBUG } 06:51:08 INFO [http_cmd]: http status code OK 06:51:08 DEBUG --- status code --- 06:51:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:51:08 DEBUG --- http response --- 06:51:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:51:08 DEBUG ----- output ----- 06:51:08 DEBUG { 06:51:08 DEBUG "status": "success", 06:51:08 DEBUG "data": { 06:51:08 DEBUG "resultType": "vector", 06:51:08 DEBUG "result": [ 06:51:08 DEBUG { 06:51:08 DEBUG "metric": { 06:51:08 DEBUG "pod": "ds-idrepo-0" 06:51:08 DEBUG }, 06:51:08 DEBUG "value": [ 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG [...] 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG "pod": "ds-idrepo-2" 06:51:08 DEBUG }, 06:51:08 DEBUG "value": [ 06:51:08 DEBUG 1713592268, 06:51:08 DEBUG "0" 06:51:08 DEBUG ] 06:51:08 DEBUG } 06:51:08 DEBUG ] 06:51:08 DEBUG } 06:51:08 DEBUG } 06:51:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592268" 06:51:08 INFO [http_cmd]: http status code OK 06:51:08 INFO [http_cmd]: http status code OK 06:51:08 DEBUG --- status code --- 06:51:08 DEBUG --- status code --- 06:51:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:51:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:51:08 DEBUG --- http response --- 06:51:08 DEBUG --- http response --- 06:51:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:51:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:51:08 DEBUG ----- output ----- 06:51:08 DEBUG ----- output ----- 06:51:08 DEBUG { 06:51:08 DEBUG { 06:51:08 DEBUG "status": "success", 06:51:08 DEBUG "status": "success", 06:51:08 DEBUG "data": { 06:51:08 DEBUG "data": { 06:51:08 DEBUG "resultType": "vector", 06:51:08 DEBUG "resultType": "vector", 06:51:08 DEBUG "result": [ 06:51:08 DEBUG "result": [ 06:51:08 DEBUG { 06:51:08 DEBUG { 06:51:08 DEBUG "metric": { 06:51:08 DEBUG "metric": { 06:51:08 DEBUG "pod": "ds-cts-0" 06:51:08 DEBUG "instance": "10.142.0.107:9100" 06:51:08 DEBUG }, 06:51:08 DEBUG }, 06:51:08 DEBUG "value": [ 06:51:08 DEBUG "value": [ 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG [...] 06:51:08 DEBUG [...] 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG "pod": "ds-cts-2" 06:51:08 DEBUG "instance": "10.142.0.43:9100" 06:51:08 DEBUG }, 06:51:08 DEBUG }, 06:51:08 DEBUG "value": [ 06:51:08 DEBUG "value": [ 06:51:08 DEBUG 1713592268, 06:51:08 DEBUG 1713592268, 06:51:08 DEBUG "32.6" 06:51:08 DEBUG "3226262963200" 06:51:08 DEBUG ] 06:51:08 DEBUG ] 06:51:08 DEBUG } 06:51:08 DEBUG } 06:51:08 DEBUG ] 06:51:08 DEBUG ] 06:51:08 DEBUG } 06:51:08 DEBUG } 06:51:08 DEBUG } 06:51:08 DEBUG } 06:51:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592268" 06:51:08 INFO [http_cmd]: http status code OK 06:51:08 DEBUG --- status code --- 06:51:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:51:08 DEBUG --- http response --- 06:51:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 06:51:08 DEBUG ----- output ----- 06:51:08 DEBUG { 06:51:08 DEBUG "status": "success", 06:51:08 DEBUG "data": { 06:51:08 DEBUG "resultType": "vector", 06:51:08 DEBUG "result": [ 06:51:08 DEBUG { 06:51:08 DEBUG "metric": { 06:51:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:51:08 DEBUG }, 06:51:08 DEBUG "value": [ 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG [...] 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 06:51:08 DEBUG }, 06:51:08 DEBUG "value": [ 06:51:08 DEBUG 1713592268, 06:51:08 DEBUG "0" 06:51:08 DEBUG ] 06:51:08 DEBUG } 06:51:08 DEBUG ] 06:51:08 DEBUG } 06:51:08 DEBUG } 06:51:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592268" 06:51:08 INFO [http_cmd]: http status code OK 06:51:08 DEBUG --- status code --- 06:51:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:51:08 DEBUG --- http response --- 06:51:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:51:08 DEBUG ----- output ----- 06:51:08 DEBUG { 06:51:08 DEBUG "status": "success", 06:51:08 DEBUG "data": { 06:51:08 DEBUG "resultType": "vector", 06:51:08 DEBUG "result": [ 06:51:08 DEBUG { 06:51:08 DEBUG "metric": { 06:51:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:51:08 DEBUG }, 06:51:08 DEBUG "value": [ 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG [...] 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:51:08 DEBUG }, 06:51:08 DEBUG "value": [ 06:51:08 DEBUG 1713592268, 06:51:08 DEBUG "480654.5" 06:51:08 DEBUG ] 06:51:08 DEBUG } 06:51:08 DEBUG ] 06:51:08 DEBUG } 06:51:08 DEBUG } 06:51:08 INFO [http_cmd]: http status code OK 06:51:08 DEBUG --- status code --- 06:51:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:51:08 DEBUG --- http response --- 06:51:08 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 06:51:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713592268" 06:51:08 ERROR [http_cmd]: ERROR 06:51:08 ERROR --- status code --- 06:51:08 ERROR http status code is 400 (expected [200, 401, 403, 404]) 06:51:08 ERROR --- http response --- 06:51:08 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 06:51:08 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 06:51:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592268" 06:51:08 INFO [http_cmd]: http status code OK 06:51:08 DEBUG --- status code --- 06:51:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:51:08 DEBUG --- http response --- 06:51:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:51:08 DEBUG ----- output ----- 06:51:08 DEBUG { 06:51:08 DEBUG "status": "success", 06:51:08 DEBUG "data": { 06:51:08 DEBUG "resultType": "vector", 06:51:08 DEBUG "result": [ 06:51:08 DEBUG { 06:51:08 DEBUG "metric": { 06:51:08 DEBUG "pod": "ds-idrepo-0" 06:51:08 DEBUG }, 06:51:08 DEBUG "value": [ 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG [...] 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG "pod": "ds-idrepo-2" 06:51:08 DEBUG }, 06:51:08 DEBUG "value": [ 06:51:08 DEBUG 1713592268, 06:51:08 DEBUG "0" 06:51:08 DEBUG ] 06:51:08 DEBUG } 06:51:08 DEBUG ] 06:51:08 DEBUG } 06:51:08 DEBUG } 06:51:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592268" 06:51:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592268" 06:51:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592268" 06:51:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713592268" 06:51:08 INFO [http_cmd]: http status code OK 06:51:08 INFO [http_cmd]: http status code OK 06:51:08 DEBUG --- status code --- 06:51:08 DEBUG --- status code --- 06:51:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:51:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:51:08 DEBUG --- http response --- 06:51:08 DEBUG --- http response --- 06:51:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:51:08 DEBUG ----- output ----- 06:51:08 DEBUG { 06:51:08 DEBUG "status": "success", 06:51:08 DEBUG "data": { 06:51:08 DEBUG "resultType": "vector", 06:51:08 DEBUG "result": [ 06:51:08 DEBUG { 06:51:08 DEBUG "metric": { 06:51:08 DEBUG "pod": "ds-cts-0" 06:51:08 DEBUG }, 06:51:08 DEBUG "value": [ 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG [...] 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG "pod": "ds-cts-2" 06:51:08 DEBUG }, 06:51:08 DEBUG "value": [ 06:51:08 DEBUG 1713592268, 06:51:08 DEBUG "0" 06:51:08 DEBUG ] 06:51:08 DEBUG } 06:51:08 DEBUG ] 06:51:08 DEBUG } 06:51:08 DEBUG } 06:51:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:51:08 DEBUG ----- output ----- 06:51:08 DEBUG { 06:51:08 DEBUG "status": "success", 06:51:08 DEBUG "data": { 06:51:08 DEBUG "resultType": "vector", 06:51:08 DEBUG "result": [ 06:51:08 DEBUG { 06:51:08 DEBUG "metric": { 06:51:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:51:08 DEBUG }, 06:51:08 DEBUG "value": [ 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG [...] 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:51:08 DEBUG }, 06:51:08 DEBUG "value": [ 06:51:08 DEBUG 1713592268, 06:51:08 DEBUG "2675.5333333333333" 06:51:08 DEBUG ] 06:51:08 DEBUG } 06:51:08 DEBUG ] 06:51:08 DEBUG } 06:51:08 DEBUG } 06:51:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592268" 06:51:08 INFO [http_cmd]: http status code OK 06:51:08 INFO [http_cmd]: http status code OK 06:51:08 DEBUG --- status code --- 06:51:08 DEBUG --- status code --- 06:51:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:51:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:51:08 DEBUG --- http response --- 06:51:08 DEBUG --- http response --- 06:51:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:51:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:51:08 DEBUG ----- output ----- 06:51:08 DEBUG ----- output ----- 06:51:08 DEBUG { 06:51:08 DEBUG { 06:51:08 DEBUG "status": "success", 06:51:08 DEBUG "status": "success", 06:51:08 DEBUG "data": { 06:51:08 DEBUG "data": { 06:51:08 DEBUG "resultType": "vector", 06:51:08 DEBUG "resultType": "vector", 06:51:08 DEBUG "result": [ 06:51:08 DEBUG "result": [ 06:51:08 DEBUG { 06:51:08 DEBUG { 06:51:08 DEBUG "metric": { 06:51:08 DEBUG "metric": { 06:51:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:51:08 DEBUG "pod": "ds-idrepo-0" 06:51:08 DEBUG }, 06:51:08 DEBUG }, 06:51:08 DEBUG "value": [ 06:51:08 DEBUG "value": [ 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG [...] 06:51:08 DEBUG [...] 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:51:08 DEBUG "pod": "ds-cts-2" 06:51:08 DEBUG }, 06:51:08 DEBUG }, 06:51:08 DEBUG "value": [ 06:51:08 DEBUG "value": [ 06:51:08 DEBUG 1713592268, 06:51:08 DEBUG 1713592268, 06:51:08 DEBUG "0" 06:51:08 DEBUG "0" 06:51:08 DEBUG ] 06:51:08 DEBUG ] 06:51:08 DEBUG } 06:51:08 DEBUG } 06:51:08 DEBUG ] 06:51:08 DEBUG ] 06:51:08 DEBUG } 06:51:08 DEBUG } 06:51:08 DEBUG } 06:51:08 DEBUG } 06:51:08 INFO [http_cmd]: http status code OK 06:51:08 DEBUG --- status code --- 06:51:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:51:08 DEBUG --- http response --- 06:51:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:51:08 DEBUG ----- output ----- 06:51:08 DEBUG { 06:51:08 DEBUG "status": "success", 06:51:08 DEBUG "data": { 06:51:08 DEBUG "resultType": "vector", 06:51:08 DEBUG "result": [ 06:51:08 DEBUG { 06:51:08 DEBUG "metric": { 06:51:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:51:08 DEBUG }, 06:51:08 DEBUG "value": [ 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG [...] 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:51:08 DEBUG }, 06:51:08 DEBUG "value": [ 06:51:08 DEBUG 1713592268, 06:51:08 DEBUG "2670.1666666666665" 06:51:08 DEBUG ] 06:51:08 DEBUG } 06:51:08 DEBUG ] 06:51:08 DEBUG } 06:51:08 DEBUG } 06:51:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592268" 06:51:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592268" 06:51:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592268" 06:51:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713592268" 06:51:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592268" 06:51:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592268" 06:51:08 INFO [http_cmd]: http status code OK 06:51:08 DEBUG --- status code --- 06:51:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:51:08 DEBUG --- http response --- 06:51:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:51:08 DEBUG ----- output ----- 06:51:08 DEBUG { 06:51:08 DEBUG "status": "success", 06:51:08 DEBUG "data": { 06:51:08 DEBUG "resultType": "vector", 06:51:08 DEBUG "result": [ 06:51:08 DEBUG { 06:51:08 DEBUG "metric": { 06:51:08 DEBUG "pod": "ds-cts-0" 06:51:08 DEBUG }, 06:51:08 DEBUG "value": [ 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG [...] 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG "pod": "ds-cts-2" 06:51:08 DEBUG }, 06:51:08 DEBUG "value": [ 06:51:08 DEBUG 1713592268, 06:51:08 DEBUG "0.00044084222222222217" 06:51:08 DEBUG ] 06:51:08 DEBUG } 06:51:08 DEBUG ] 06:51:08 DEBUG } 06:51:08 DEBUG } 06:51:08 INFO [http_cmd]: http status code OK 06:51:08 INFO [http_cmd]: http status code OK 06:51:08 DEBUG --- status code --- 06:51:08 DEBUG --- status code --- 06:51:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:51:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:51:08 DEBUG --- http response --- 06:51:08 DEBUG --- http response --- 06:51:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:51:08 DEBUG ----- output ----- 06:51:08 DEBUG { 06:51:08 DEBUG "status": "success", 06:51:08 DEBUG "data": { 06:51:08 DEBUG "resultType": "vector", 06:51:08 DEBUG "result": [ 06:51:08 DEBUG { 06:51:08 DEBUG "metric": { 06:51:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:51:08 DEBUG }, 06:51:08 DEBUG "value": [ 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG [...] 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:51:08 DEBUG }, 06:51:08 DEBUG "value": [ 06:51:08 DEBUG 1713592268, 06:51:08 DEBUG "2664.633333333333" 06:51:08 DEBUG ] 06:51:08 DEBUG } 06:51:08 DEBUG ] 06:51:08 DEBUG } 06:51:08 DEBUG } 06:51:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:51:08 DEBUG ----- output ----- 06:51:08 DEBUG { 06:51:08 DEBUG "status": "success", 06:51:08 DEBUG "data": { 06:51:08 DEBUG "resultType": "vector", 06:51:08 DEBUG "result": [ 06:51:08 DEBUG { 06:51:08 DEBUG "metric": { 06:51:08 DEBUG "instance": "10.142.0.107:9100" 06:51:08 DEBUG }, 06:51:08 DEBUG "value": [ 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG [...] 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG "instance": "10.142.0.43:9100" 06:51:08 DEBUG }, 06:51:08 DEBUG "value": [ 06:51:08 DEBUG 1713592268, 06:51:08 DEBUG "0.14943333333328515" 06:51:08 DEBUG ] 06:51:08 DEBUG } 06:51:08 DEBUG ] 06:51:08 DEBUG } 06:51:08 DEBUG } 06:51:08 INFO [http_cmd]: http status code OK 06:51:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592268" 06:51:08 INFO [http_cmd]: http status code OK 06:51:08 DEBUG --- status code --- 06:51:08 DEBUG --- status code --- 06:51:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:51:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:51:08 DEBUG --- http response --- 06:51:08 DEBUG --- http response --- 06:51:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:51:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:51:08 DEBUG ----- output ----- 06:51:08 DEBUG ----- output ----- 06:51:08 DEBUG { 06:51:08 DEBUG { 06:51:08 DEBUG "status": "success", 06:51:08 DEBUG "status": "success", 06:51:08 DEBUG "data": { 06:51:08 DEBUG "data": { 06:51:08 DEBUG "resultType": "vector", 06:51:08 DEBUG "resultType": "vector", 06:51:08 DEBUG "result": [ 06:51:08 DEBUG "result": [ 06:51:08 DEBUG { 06:51:08 DEBUG { 06:51:08 DEBUG "metric": { 06:51:08 INFO [http_cmd]: http status code OK 06:51:08 DEBUG "metric": { 06:51:08 DEBUG "instance": "10.142.0.107:9100" 06:51:08 DEBUG --- status code --- 06:51:08 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 06:51:08 DEBUG }, 06:51:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:51:08 DEBUG }, 06:51:08 DEBUG "value": [ 06:51:08 DEBUG --- http response --- 06:51:08 DEBUG "value": [ 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG ----- output ----- 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG { 06:51:08 DEBUG [...] 06:51:08 DEBUG [...] 06:51:08 DEBUG "status": "success", 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG "data": { 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG "resultType": "vector", 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG "result": [ 06:51:08 DEBUG "instance": "10.142.0.43:9100" 06:51:08 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 06:51:08 DEBUG { 06:51:08 DEBUG }, 06:51:08 DEBUG }, 06:51:08 DEBUG "metric": { 06:51:08 DEBUG "value": [ 06:51:08 DEBUG "value": [ 06:51:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:51:08 DEBUG 1713592268, 06:51:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592268" 06:51:08 DEBUG 1713592268, 06:51:08 DEBUG }, 06:51:08 DEBUG "80693667.06666666" 06:51:08 DEBUG "8.033302094567546" 06:51:08 DEBUG "value": [ 06:51:08 DEBUG ] 06:51:08 DEBUG ] 06:51:08 DEBUG 06:51:08 DEBUG } 06:51:08 DEBUG } 06:51:08 DEBUG 06:51:08 DEBUG ] 06:51:08 DEBUG ] 06:51:08 DEBUG 06:51:08 DEBUG } 06:51:08 DEBUG } 06:51:08 DEBUG [...] 06:51:08 DEBUG } 06:51:08 DEBUG } 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG "pod": "ds-idrepo-2" 06:51:08 DEBUG }, 06:51:08 DEBUG "value": [ 06:51:08 DEBUG 1713592268, 06:51:08 DEBUG "0" 06:51:08 DEBUG ] 06:51:08 DEBUG } 06:51:08 DEBUG ] 06:51:08 DEBUG } 06:51:08 DEBUG } 06:51:08 INFO [http_cmd]: http status code OK 06:51:08 DEBUG --- status code --- 06:51:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:51:08 DEBUG --- http response --- 06:51:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:51:08 DEBUG ----- output ----- 06:51:08 DEBUG { 06:51:08 DEBUG "status": "success", 06:51:08 DEBUG "data": { 06:51:08 DEBUG "resultType": "vector", 06:51:08 DEBUG "result": [ 06:51:08 DEBUG { 06:51:08 DEBUG "metric": { 06:51:08 DEBUG "pod": "ds-idrepo-0" 06:51:08 DEBUG }, 06:51:08 DEBUG "value": [ 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG [...] 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG "pod": "ds-idrepo-2" 06:51:08 DEBUG }, 06:51:08 DEBUG "value": [ 06:51:08 DEBUG 1713592268, 06:51:08 DEBUG "0" 06:51:08 DEBUG ] 06:51:08 DEBUG } 06:51:08 DEBUG ] 06:51:08 DEBUG } 06:51:08 DEBUG } 06:51:08 INFO [http_cmd]: http status code OK 06:51:08 DEBUG --- status code --- 06:51:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:51:08 DEBUG --- http response --- 06:51:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:51:08 DEBUG ----- output ----- 06:51:08 DEBUG { 06:51:08 DEBUG "status": "success", 06:51:08 DEBUG "data": { 06:51:08 DEBUG "resultType": "vector", 06:51:08 DEBUG "result": [ 06:51:08 DEBUG { 06:51:08 DEBUG "metric": { 06:51:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:51:08 DEBUG }, 06:51:08 DEBUG "value": [ 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG [...] 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:51:08 DEBUG }, 06:51:08 DEBUG "value": [ 06:51:08 DEBUG 1713592268, 06:51:08 DEBUG "2675.8333333333335" 06:51:08 DEBUG ] 06:51:08 DEBUG } 06:51:08 DEBUG ] 06:51:08 DEBUG } 06:51:08 DEBUG } 06:51:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713592268" 06:51:08 INFO [http_cmd]: http status code OK 06:51:08 DEBUG --- status code --- 06:51:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:51:08 DEBUG --- http response --- 06:51:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:51:08 DEBUG ----- output ----- 06:51:08 DEBUG { 06:51:08 DEBUG "status": "success", 06:51:08 DEBUG "data": { 06:51:08 DEBUG "resultType": "vector", 06:51:08 DEBUG "result": [ 06:51:08 DEBUG { 06:51:08 DEBUG "metric": { 06:51:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:51:08 DEBUG }, 06:51:08 DEBUG "value": [ 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG [...] 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:51:08 DEBUG }, 06:51:08 DEBUG "value": [ 06:51:08 DEBUG 1713592268, 06:51:08 DEBUG "0.096" 06:51:08 DEBUG ] 06:51:08 DEBUG } 06:51:08 DEBUG ] 06:51:08 DEBUG } 06:51:08 DEBUG } 06:51:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592268" 06:51:08 INFO [http_cmd]: http status code OK 06:51:08 DEBUG --- status code --- 06:51:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:51:08 DEBUG --- http response --- 06:51:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:51:08 DEBUG ----- output ----- 06:51:08 DEBUG { 06:51:08 DEBUG "status": "success", 06:51:08 DEBUG "data": { 06:51:08 DEBUG "resultType": "vector", 06:51:08 DEBUG "result": [ 06:51:08 DEBUG { 06:51:08 DEBUG "metric": { 06:51:08 DEBUG "instance": "10.142.0.107:9100" 06:51:08 DEBUG }, 06:51:08 DEBUG "value": [ 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG [...] 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG "instance": "10.142.0.43:9100" 06:51:08 DEBUG }, 06:51:08 DEBUG "value": [ 06:51:08 DEBUG 1713592268, 06:51:08 DEBUG "80689728.33333333" 06:51:08 DEBUG ] 06:51:08 DEBUG } 06:51:08 DEBUG ] 06:51:08 DEBUG } 06:51:08 DEBUG } 06:51:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592268" 06:51:08 INFO [http_cmd]: http status code OK 06:51:08 DEBUG --- status code --- 06:51:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:51:08 DEBUG --- http response --- 06:51:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 06:51:08 DEBUG ----- output ----- 06:51:08 DEBUG { 06:51:08 DEBUG "status": "success", 06:51:08 DEBUG "data": { 06:51:08 DEBUG "resultType": "vector", 06:51:08 DEBUG "result": [ 06:51:08 DEBUG { 06:51:08 DEBUG "metric": { 06:51:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:51:08 DEBUG }, 06:51:08 DEBUG "value": [ 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG [...] 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG "pod": "ds-idrepo-2" 06:51:08 DEBUG }, 06:51:08 DEBUG "value": [ 06:51:08 DEBUG 1713592268, 06:51:08 DEBUG "0" 06:51:08 DEBUG ] 06:51:08 DEBUG } 06:51:08 DEBUG ] 06:51:08 DEBUG } 06:51:08 DEBUG } 06:51:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592268" 06:51:08 INFO [http_cmd]: http status code OK 06:51:08 DEBUG --- status code --- 06:51:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:51:08 DEBUG --- http response --- 06:51:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:51:08 DEBUG ----- output ----- 06:51:08 DEBUG { 06:51:08 DEBUG "status": "success", 06:51:08 DEBUG "data": { 06:51:08 DEBUG "resultType": "vector", 06:51:08 DEBUG "result": [ 06:51:08 DEBUG { 06:51:08 DEBUG "metric": { 06:51:08 DEBUG "instance": "10.142.0.107:9100" 06:51:08 DEBUG }, 06:51:08 DEBUG "value": [ 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG [...] 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG 06:51:08 DEBUG "instance": "10.142.0.43:9100" 06:51:08 DEBUG }, 06:51:08 DEBUG "value": [ 06:51:08 DEBUG 1713592268, 06:51:08 DEBUG "0.05833333333332765" 06:51:08 DEBUG ] 06:51:08 DEBUG } 06:51:08 DEBUG ] 06:51:08 DEBUG } 06:51:08 DEBUG } 06:51:09 INFO 06:51:09 INFO [loop_until]: kubectl --namespace=xlou top node 06:51:09 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:51:09 INFO [loop_until]: OK (rc = 0) 06:51:09 DEBUG --- stdout --- 06:51:09 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 72m 0% 6278Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 8017m 26% 7430Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 17235m 57% 11362Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 79m 0% 3347Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 18421m 61% 13535Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 146m 0% 5222Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 17384m 58% 11606Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 18912m 63% 3009Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 19363m 64% 10863Mi 9% gke-xlou-cdm-ds-0f601751-j488 8151m 27% 7022Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 17122m 57% 15594Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 8025m 26% 7694Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 19612m 65% 12881Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 8166m 51% 8501Mi 14% 06:51:09 DEBUG --- stderr --- 06:51:09 DEBUG 06:51:09 INFO 06:51:09 INFO [loop_until]: kubectl --namespace=xlou top pods 06:51:09 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:51:09 INFO [loop_until]: OK (rc = 0) 06:51:09 DEBUG --- stdout --- 06:51:09 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 17297m 8649Mi am-767f6b67b5-qcqz2 17113m 8715Mi am-767f6b67b5-vbltz 18649m 8654Mi ds-cts-0 18246m 10400Mi ds-cts-1 17319m 10666Mi ds-cts-2 19812m 10267Mi ds-idrepo-0 7842m 3758Mi ds-idrepo-1 7582m 3802Mi ds-idrepo-2 7418m 3691Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1177Mi idm-b85f9d978-wnhxb 3m 1236Mi lodemon-7cb5b9456d-dskfn 1m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7933m 4666Mi 06:51:09 DEBUG --- stderr --- 06:51:09 DEBUG 06:52:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592328" 06:52:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713592328" 06:52:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592328" 06:52:08 INFO [http_cmd]: http status code OK 06:52:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592328" 06:52:08 DEBUG --- status code --- 06:52:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:52:08 DEBUG --- http response --- 06:52:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 06:52:08 DEBUG ----- output ----- 06:52:08 DEBUG { 06:52:08 DEBUG "status": "success", 06:52:08 DEBUG "data": { 06:52:08 DEBUG "resultType": "vector", 06:52:08 DEBUG "result": [ 06:52:08 DEBUG { 06:52:08 DEBUG "metric": { 06:52:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:52:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592328" 06:52:08 DEBUG }, 06:52:08 DEBUG "value": [ 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG [...] 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG "pod": "am-767f6b67b5-qcqz2" 06:52:08 DEBUG }, 06:52:08 DEBUG "value": [ 06:52:08 DEBUG 1713592328, 06:52:08 DEBUG "0" 06:52:08 DEBUG ] 06:52:08 DEBUG } 06:52:08 DEBUG ] 06:52:08 DEBUG } 06:52:08 DEBUG } 06:52:08 INFO [http_cmd]: http status code OK 06:52:08 INFO [http_cmd]: http status code OK 06:52:08 DEBUG --- status code --- 06:52:08 DEBUG --- status code --- 06:52:08 INFO [http_cmd]: http status code OK 06:52:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:52:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:52:08 DEBUG --- status code --- 06:52:08 DEBUG --- http response --- 06:52:08 DEBUG --- http response --- 06:52:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:52:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:52:08 DEBUG --- http response --- 06:52:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:52:08 DEBUG ----- output ----- 06:52:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:52:08 DEBUG ----- output ----- 06:52:08 DEBUG { 06:52:08 DEBUG ----- output ----- 06:52:08 DEBUG { 06:52:08 DEBUG "status": "success", 06:52:08 DEBUG { 06:52:08 DEBUG "status": "success", 06:52:08 DEBUG "data": { 06:52:08 DEBUG "status": "success", 06:52:08 DEBUG "data": { 06:52:08 DEBUG "resultType": "vector", 06:52:08 DEBUG "data": { 06:52:08 DEBUG "resultType": "vector", 06:52:08 DEBUG "result": [ 06:52:08 DEBUG "resultType": "vector", 06:52:08 DEBUG "result": [ 06:52:08 DEBUG { 06:52:08 DEBUG "result": [ 06:52:08 DEBUG { 06:52:08 DEBUG "metric": { 06:52:08 DEBUG { 06:52:08 DEBUG "metric": { 06:52:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:52:08 DEBUG "metric": { 06:52:08 DEBUG "instance": "10.142.0.107:9100" 06:52:08 DEBUG }, 06:52:08 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 06:52:08 DEBUG }, 06:52:08 DEBUG "value": [ 06:52:08 DEBUG }, 06:52:08 DEBUG "value": [ 06:52:08 DEBUG 06:52:08 DEBUG "value": [ 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG [...] 06:52:08 DEBUG 06:52:08 DEBUG [...] 06:52:08 DEBUG 06:52:08 DEBUG [...] 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:52:08 DEBUG 06:52:08 DEBUG "instance": "10.142.0.43:9100" 06:52:08 DEBUG }, 06:52:08 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 06:52:08 DEBUG }, 06:52:08 DEBUG "value": [ 06:52:08 DEBUG }, 06:52:08 DEBUG "value": [ 06:52:08 DEBUG 1713592328, 06:52:08 DEBUG "value": [ 06:52:08 DEBUG 1713592328, 06:52:08 DEBUG "480542.5" 06:52:08 DEBUG 1713592328, 06:52:08 DEBUG "3230622786560" 06:52:08 DEBUG ] 06:52:08 DEBUG "4987387904" 06:52:08 DEBUG ] 06:52:08 DEBUG } 06:52:08 DEBUG ] 06:52:08 DEBUG } 06:52:08 DEBUG ] 06:52:08 DEBUG } 06:52:08 DEBUG ] 06:52:08 DEBUG } 06:52:08 DEBUG ] 06:52:08 DEBUG } 06:52:08 DEBUG } 06:52:08 DEBUG } 06:52:08 DEBUG } 06:52:08 DEBUG } 06:52:08 INFO [http_cmd]: http status code OK 06:52:08 DEBUG --- status code --- 06:52:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:52:08 DEBUG --- http response --- 06:52:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:52:08 DEBUG ----- output ----- 06:52:08 DEBUG { 06:52:08 DEBUG "status": "success", 06:52:08 DEBUG "data": { 06:52:08 DEBUG "resultType": "vector", 06:52:08 DEBUG "result": [ 06:52:08 DEBUG { 06:52:08 DEBUG "metric": { 06:52:08 DEBUG "pod": "ds-idrepo-0" 06:52:08 DEBUG }, 06:52:08 DEBUG "value": [ 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG [...] 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG "pod": "ds-idrepo-2" 06:52:08 DEBUG }, 06:52:08 DEBUG "value": [ 06:52:08 DEBUG 1713592328, 06:52:08 DEBUG "0" 06:52:08 DEBUG ] 06:52:08 DEBUG } 06:52:08 DEBUG ] 06:52:08 DEBUG } 06:52:08 DEBUG } 06:52:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592328" 06:52:08 INFO [http_cmd]: http status code OK 06:52:08 DEBUG --- status code --- 06:52:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:52:08 DEBUG --- http response --- 06:52:08 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 06:52:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592328" 06:52:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592328" 06:52:08 INFO [http_cmd]: http status code OK 06:52:08 DEBUG --- status code --- 06:52:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:52:08 DEBUG --- http response --- 06:52:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:52:08 DEBUG ----- output ----- 06:52:08 DEBUG { 06:52:08 DEBUG "status": "success", 06:52:08 DEBUG "data": { 06:52:08 DEBUG "resultType": "vector", 06:52:08 DEBUG "result": [ 06:52:08 DEBUG { 06:52:08 DEBUG "metric": { 06:52:08 DEBUG "pod": "ds-cts-0" 06:52:08 DEBUG }, 06:52:08 DEBUG "value": [ 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG [...] 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG "pod": "ds-cts-2" 06:52:08 DEBUG }, 06:52:08 DEBUG "value": [ 06:52:08 DEBUG 1713592328, 06:52:08 DEBUG "30" 06:52:08 DEBUG ] 06:52:08 DEBUG } 06:52:08 DEBUG ] 06:52:08 DEBUG } 06:52:08 DEBUG } 06:52:08 INFO [http_cmd]: http status code OK 06:52:08 DEBUG --- status code --- 06:52:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:52:08 DEBUG --- http response --- 06:52:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 06:52:08 DEBUG ----- output ----- 06:52:08 DEBUG { 06:52:08 DEBUG "status": "success", 06:52:08 DEBUG "data": { 06:52:08 DEBUG "resultType": "vector", 06:52:08 DEBUG "result": [ 06:52:08 DEBUG { 06:52:08 DEBUG "metric": { 06:52:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:52:08 DEBUG }, 06:52:08 DEBUG "value": [ 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG [...] 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG "pod": "am-767f6b67b5-qcqz2" 06:52:08 DEBUG }, 06:52:08 DEBUG "value": [ 06:52:08 DEBUG 1713592328, 06:52:08 DEBUG "0" 06:52:08 DEBUG ] 06:52:08 DEBUG } 06:52:08 DEBUG ] 06:52:08 DEBUG } 06:52:08 DEBUG } 06:52:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713592328" 06:52:08 ERROR [http_cmd]: ERROR 06:52:08 ERROR --- status code --- 06:52:08 ERROR http status code is 400 (expected [200, 401, 403, 404]) 06:52:08 ERROR --- http response --- 06:52:08 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 06:52:08 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 06:52:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592328" 06:52:08 INFO [http_cmd]: http status code OK 06:52:08 DEBUG --- status code --- 06:52:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:52:08 DEBUG --- http response --- 06:52:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:52:08 DEBUG ----- output ----- 06:52:08 DEBUG { 06:52:08 DEBUG "status": "success", 06:52:08 DEBUG "data": { 06:52:08 DEBUG "resultType": "vector", 06:52:08 DEBUG "result": [ 06:52:08 DEBUG { 06:52:08 DEBUG "metric": { 06:52:08 DEBUG "pod": "ds-idrepo-0" 06:52:08 DEBUG }, 06:52:08 DEBUG "value": [ 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG [...] 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG "pod": "ds-idrepo-2" 06:52:08 DEBUG }, 06:52:08 DEBUG "value": [ 06:52:08 DEBUG 1713592328, 06:52:08 DEBUG "0" 06:52:08 DEBUG ] 06:52:08 DEBUG } 06:52:08 DEBUG ] 06:52:08 DEBUG } 06:52:08 DEBUG } 06:52:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592328" 06:52:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592328" 06:52:08 INFO [http_cmd]: http status code OK 06:52:08 DEBUG --- status code --- 06:52:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:52:08 DEBUG --- http response --- 06:52:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:52:08 DEBUG ----- output ----- 06:52:08 DEBUG { 06:52:08 DEBUG "status": "success", 06:52:08 DEBUG "data": { 06:52:08 DEBUG "resultType": "vector", 06:52:08 DEBUG "result": [ 06:52:08 DEBUG { 06:52:08 DEBUG "metric": { 06:52:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:52:08 DEBUG }, 06:52:08 DEBUG "value": [ 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG [...] 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:52:08 DEBUG }, 06:52:08 DEBUG "value": [ 06:52:08 DEBUG 1713592328, 06:52:08 DEBUG "0" 06:52:08 DEBUG ] 06:52:08 DEBUG } 06:52:08 DEBUG ] 06:52:08 DEBUG } 06:52:08 DEBUG } 06:52:08 INFO [http_cmd]: http status code OK 06:52:08 DEBUG --- status code --- 06:52:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:52:08 DEBUG --- http response --- 06:52:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:52:08 DEBUG ----- output ----- 06:52:08 DEBUG { 06:52:08 DEBUG "status": "success", 06:52:08 DEBUG "data": { 06:52:08 DEBUG "resultType": "vector", 06:52:08 DEBUG "result": [ 06:52:08 DEBUG { 06:52:08 DEBUG "metric": { 06:52:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:52:08 DEBUG }, 06:52:08 DEBUG "value": [ 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG [...] 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG "pod": "am-767f6b67b5-qcqz2" 06:52:08 DEBUG }, 06:52:08 DEBUG "value": [ 06:52:08 DEBUG 1713592328, 06:52:08 DEBUG "52757.58761786159" 06:52:08 DEBUG ] 06:52:08 DEBUG } 06:52:08 DEBUG ] 06:52:08 DEBUG } 06:52:08 DEBUG } 06:52:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592328" 06:52:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592328" 06:52:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592328" 06:52:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713592328" 06:52:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592328" 06:52:08 INFO [http_cmd]: http status code OK 06:52:08 INFO [http_cmd]: http status code OK 06:52:08 INFO [http_cmd]: http status code OK 06:52:08 DEBUG --- status code --- 06:52:08 DEBUG --- status code --- 06:52:08 DEBUG --- status code --- 06:52:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:52:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:52:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:52:08 DEBUG --- http response --- 06:52:08 DEBUG --- http response --- 06:52:08 DEBUG --- http response --- 06:52:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:52:08 DEBUG ----- output ----- 06:52:08 DEBUG { 06:52:08 DEBUG "status": "success", 06:52:08 DEBUG "data": { 06:52:08 DEBUG "resultType": "vector", 06:52:08 DEBUG "result": [ 06:52:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:52:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:52:08 INFO [http_cmd]: http status code OK 06:52:08 DEBUG { 06:52:08 DEBUG ----- output ----- 06:52:08 DEBUG ----- output ----- 06:52:08 DEBUG --- status code --- 06:52:08 DEBUG "metric": { 06:52:08 DEBUG { 06:52:08 DEBUG { 06:52:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:52:08 DEBUG "pod": "ds-cts-0" 06:52:08 DEBUG "status": "success", 06:52:08 DEBUG "status": "success", 06:52:08 DEBUG --- http response --- 06:52:08 DEBUG }, 06:52:08 DEBUG "data": { 06:52:08 DEBUG "data": { 06:52:08 DEBUG "value": [ 06:52:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:52:08 DEBUG "resultType": "vector", 06:52:08 DEBUG "resultType": "vector", 06:52:08 DEBUG 06:52:08 DEBUG ----- output ----- 06:52:08 DEBUG "result": [ 06:52:08 DEBUG "result": [ 06:52:08 DEBUG 06:52:08 DEBUG { 06:52:08 DEBUG { 06:52:08 DEBUG { 06:52:08 DEBUG 06:52:08 DEBUG "status": "success", 06:52:08 DEBUG "metric": { 06:52:08 DEBUG "metric": { 06:52:08 DEBUG [...] 06:52:08 DEBUG "data": { 06:52:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:52:08 DEBUG "instance": "10.142.0.107:9100" 06:52:08 DEBUG 06:52:08 DEBUG "resultType": "vector", 06:52:08 DEBUG }, 06:52:08 DEBUG }, 06:52:08 DEBUG 06:52:08 DEBUG "result": [ 06:52:08 DEBUG "value": [ 06:52:08 DEBUG "value": [ 06:52:08 DEBUG 06:52:08 DEBUG { 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG "pod": "ds-cts-2" 06:52:08 DEBUG "metric": { 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG }, 06:52:08 DEBUG "pod": "ds-idrepo-0" 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG "value": [ 06:52:08 DEBUG }, 06:52:08 DEBUG [...] 06:52:08 DEBUG [...] 06:52:08 DEBUG 1713592328, 06:52:08 DEBUG "value": [ 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG "0" 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG ] 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG } 06:52:08 DEBUG 06:52:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:52:08 DEBUG "instance": "10.142.0.43:9100" 06:52:08 DEBUG ] 06:52:08 DEBUG [...] 06:52:08 DEBUG }, 06:52:08 DEBUG }, 06:52:08 DEBUG } 06:52:08 DEBUG 06:52:08 DEBUG "value": [ 06:52:08 DEBUG "value": [ 06:52:08 DEBUG } 06:52:08 DEBUG 06:52:08 DEBUG 1713592328, 06:52:08 DEBUG 1713592328, 06:52:08 DEBUG 06:52:08 DEBUG "2656" 06:52:08 DEBUG "0.25479999999992625" 06:52:08 DEBUG "pod": "ds-cts-2" 06:52:08 DEBUG ] 06:52:08 DEBUG ] 06:52:08 DEBUG }, 06:52:08 DEBUG } 06:52:08 DEBUG } 06:52:08 DEBUG "value": [ 06:52:08 DEBUG ] 06:52:08 DEBUG ] 06:52:08 DEBUG 1713592328, 06:52:08 DEBUG } 06:52:08 DEBUG } 06:52:08 DEBUG "0" 06:52:08 DEBUG } 06:52:08 DEBUG } 06:52:08 DEBUG ] 06:52:08 DEBUG } 06:52:08 DEBUG ] 06:52:08 DEBUG } 06:52:08 DEBUG } 06:52:08 INFO [http_cmd]: http status code OK 06:52:08 DEBUG --- status code --- 06:52:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:52:08 DEBUG --- http response --- 06:52:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:52:08 DEBUG ----- output ----- 06:52:08 DEBUG { 06:52:08 DEBUG "status": "success", 06:52:08 DEBUG "data": { 06:52:08 DEBUG "resultType": "vector", 06:52:08 DEBUG "result": [ 06:52:08 DEBUG { 06:52:08 DEBUG "metric": { 06:52:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:52:08 DEBUG }, 06:52:08 DEBUG "value": [ 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG [...] 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:52:08 DEBUG }, 06:52:08 DEBUG "value": [ 06:52:08 DEBUG 1713592328, 06:52:08 DEBUG "2656.0666666666666" 06:52:08 DEBUG ] 06:52:08 DEBUG } 06:52:08 DEBUG ] 06:52:08 DEBUG } 06:52:08 DEBUG } 06:52:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592328" 06:52:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592328" 06:52:08 INFO [http_cmd]: http status code OK 06:52:08 DEBUG --- status code --- 06:52:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:52:08 DEBUG --- http response --- 06:52:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:52:08 DEBUG ----- output ----- 06:52:08 DEBUG { 06:52:08 DEBUG "status": "success", 06:52:08 DEBUG "data": { 06:52:08 DEBUG "resultType": "vector", 06:52:08 DEBUG "result": [ 06:52:08 DEBUG { 06:52:08 DEBUG "metric": { 06:52:08 DEBUG "pod": "ds-cts-0" 06:52:08 DEBUG }, 06:52:08 DEBUG "value": [ 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG [...] 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG "pod": "ds-cts-2" 06:52:08 DEBUG }, 06:52:08 DEBUG "value": [ 06:52:08 DEBUG 1713592328, 06:52:08 DEBUG "0.0013805066666666666" 06:52:08 DEBUG ] 06:52:08 DEBUG } 06:52:08 DEBUG ] 06:52:08 DEBUG } 06:52:08 DEBUG } 06:52:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592328" 06:52:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713592328" 06:52:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592328" 06:52:08 INFO [http_cmd]: http status code OK 06:52:08 DEBUG --- status code --- 06:52:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:52:08 DEBUG --- http response --- 06:52:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:52:08 DEBUG ----- output ----- 06:52:08 DEBUG { 06:52:08 DEBUG "status": "success", 06:52:08 DEBUG "data": { 06:52:08 DEBUG "resultType": "vector", 06:52:08 DEBUG "result": [ 06:52:08 DEBUG { 06:52:08 DEBUG "metric": { 06:52:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:52:08 DEBUG }, 06:52:08 DEBUG "value": [ 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG [...] 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:52:08 DEBUG }, 06:52:08 DEBUG "value": [ 06:52:08 DEBUG 1713592328, 06:52:08 DEBUG "2679.0333333333333" 06:52:08 DEBUG ] 06:52:08 DEBUG } 06:52:08 DEBUG ] 06:52:08 DEBUG } 06:52:08 DEBUG } 06:52:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592328" 06:52:08 INFO [http_cmd]: http status code OK 06:52:08 DEBUG --- status code --- 06:52:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:52:08 DEBUG --- http response --- 06:52:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:52:08 DEBUG ----- output ----- 06:52:08 DEBUG { 06:52:08 DEBUG "status": "success", 06:52:08 DEBUG "data": { 06:52:08 DEBUG "resultType": "vector", 06:52:08 DEBUG "result": [ 06:52:08 DEBUG { 06:52:08 DEBUG "metric": { 06:52:08 DEBUG "instance": "10.142.0.107:9100" 06:52:08 DEBUG }, 06:52:08 DEBUG "value": [ 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG [...] 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG "instance": "10.142.0.43:9100" 06:52:08 DEBUG }, 06:52:08 DEBUG "value": [ 06:52:08 DEBUG 1713592328, 06:52:08 DEBUG "80717659.76666667" 06:52:08 DEBUG ] 06:52:08 DEBUG } 06:52:08 DEBUG ] 06:52:08 DEBUG } 06:52:08 DEBUG } 06:52:08 INFO [http_cmd]: http status code OK 06:52:08 DEBUG --- status code --- 06:52:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:52:08 DEBUG --- http response --- 06:52:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:52:08 DEBUG ----- output ----- 06:52:08 DEBUG { 06:52:08 DEBUG "status": "success", 06:52:08 DEBUG "data": { 06:52:08 DEBUG "resultType": "vector", 06:52:08 DEBUG "result": [ 06:52:08 DEBUG { 06:52:08 DEBUG "metric": { 06:52:08 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 06:52:08 DEBUG }, 06:52:08 DEBUG "value": [ 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG [...] 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 06:52:08 DEBUG }, 06:52:08 DEBUG "value": [ 06:52:08 DEBUG 1713592328, 06:52:08 DEBUG "8.054536573052454" 06:52:08 DEBUG ] 06:52:08 DEBUG } 06:52:08 DEBUG ] 06:52:08 DEBUG } 06:52:08 DEBUG } 06:52:08 INFO [http_cmd]: http status code OK 06:52:08 DEBUG --- status code --- 06:52:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:52:08 DEBUG --- http response --- 06:52:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:52:08 DEBUG ----- output ----- 06:52:08 DEBUG { 06:52:08 DEBUG "status": "success", 06:52:08 DEBUG "data": { 06:52:08 DEBUG "resultType": "vector", 06:52:08 DEBUG "result": [ 06:52:08 DEBUG { 06:52:08 DEBUG "metric": { 06:52:08 DEBUG "pod": "ds-idrepo-0" 06:52:08 DEBUG }, 06:52:08 DEBUG "value": [ 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG [...] 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG "pod": "ds-idrepo-2" 06:52:08 DEBUG }, 06:52:08 DEBUG "value": [ 06:52:08 DEBUG 1713592328, 06:52:08 DEBUG "0" 06:52:08 DEBUG ] 06:52:08 DEBUG } 06:52:08 DEBUG ] 06:52:08 DEBUG } 06:52:08 DEBUG } 06:52:08 INFO [http_cmd]: http status code OK 06:52:08 DEBUG --- status code --- 06:52:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:52:08 DEBUG --- http response --- 06:52:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:52:08 DEBUG ----- output ----- 06:52:08 DEBUG { 06:52:08 DEBUG "status": "success", 06:52:08 DEBUG "data": { 06:52:08 DEBUG "resultType": "vector", 06:52:08 DEBUG "result": [ 06:52:08 DEBUG { 06:52:08 DEBUG "metric": { 06:52:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:52:08 DEBUG }, 06:52:08 DEBUG "value": [ 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG [...] 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:52:08 DEBUG }, 06:52:08 DEBUG "value": [ 06:52:08 DEBUG 1713592328, 06:52:08 DEBUG "2655.3333333333335" 06:52:08 DEBUG ] 06:52:08 DEBUG } 06:52:08 DEBUG ] 06:52:08 DEBUG } 06:52:08 DEBUG } 06:52:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713592328" 06:52:08 INFO [http_cmd]: http status code OK 06:52:08 DEBUG --- status code --- 06:52:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:52:08 DEBUG --- http response --- 06:52:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:52:08 DEBUG ----- output ----- 06:52:08 DEBUG { 06:52:08 DEBUG "status": "success", 06:52:08 DEBUG "data": { 06:52:08 DEBUG "resultType": "vector", 06:52:08 DEBUG "result": [ 06:52:08 DEBUG { 06:52:08 DEBUG "metric": { 06:52:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:52:08 DEBUG }, 06:52:08 DEBUG "value": [ 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG [...] 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:52:08 DEBUG }, 06:52:08 DEBUG "value": [ 06:52:08 DEBUG 1713592328, 06:52:08 DEBUG "0.096" 06:52:08 DEBUG ] 06:52:08 DEBUG } 06:52:08 DEBUG ] 06:52:08 DEBUG } 06:52:08 DEBUG } 06:52:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592328" 06:52:08 INFO [http_cmd]: http status code OK 06:52:08 DEBUG --- status code --- 06:52:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:52:08 DEBUG --- http response --- 06:52:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:52:08 DEBUG ----- output ----- 06:52:08 DEBUG { 06:52:08 DEBUG "status": "success", 06:52:08 DEBUG "data": { 06:52:08 DEBUG "resultType": "vector", 06:52:08 DEBUG "result": [ 06:52:08 DEBUG { 06:52:08 DEBUG "metric": { 06:52:08 DEBUG "instance": "10.142.0.107:9100" 06:52:08 DEBUG }, 06:52:08 DEBUG "value": [ 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG [...] 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG "instance": "10.142.0.43:9100" 06:52:08 DEBUG }, 06:52:08 DEBUG "value": [ 06:52:08 DEBUG 1713592328, 06:52:08 DEBUG "80712336.73333333" 06:52:08 DEBUG ] 06:52:08 DEBUG } 06:52:08 DEBUG ] 06:52:08 DEBUG } 06:52:08 DEBUG } 06:52:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592328" 06:52:08 INFO [http_cmd]: http status code OK 06:52:08 DEBUG --- status code --- 06:52:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:52:08 DEBUG --- http response --- 06:52:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:52:08 DEBUG ----- output ----- 06:52:08 DEBUG { 06:52:08 DEBUG "status": "success", 06:52:08 DEBUG "data": { 06:52:08 DEBUG "resultType": "vector", 06:52:08 DEBUG "result": [ 06:52:08 DEBUG { 06:52:08 DEBUG "metric": { 06:52:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:52:08 DEBUG }, 06:52:08 DEBUG "value": [ 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG [...] 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG 06:52:08 DEBUG "pod": "am-767f6b67b5-qcqz2" 06:52:08 DEBUG }, 06:52:08 DEBUG "value": [ 06:52:08 DEBUG 1713592328, 06:52:08 DEBUG "17.237868350826126" 06:52:08 DEBUG ] 06:52:08 DEBUG } 06:52:08 DEBUG ] 06:52:08 DEBUG } 06:52:08 DEBUG } 06:52:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592328" 06:52:08 INFO [http_cmd]: http status code OK 06:52:08 DEBUG --- status code --- 06:52:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:52:08 DEBUG --- http response --- 06:52:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:52:09 DEBUG ----- output ----- 06:52:09 DEBUG { 06:52:09 DEBUG "status": "success", 06:52:09 DEBUG "data": { 06:52:09 DEBUG "resultType": "vector", 06:52:09 DEBUG "result": [ 06:52:09 DEBUG { 06:52:09 DEBUG "metric": { 06:52:09 DEBUG "instance": "10.142.0.107:9100" 06:52:09 DEBUG }, 06:52:09 DEBUG "value": [ 06:52:09 DEBUG 06:52:09 DEBUG 06:52:09 DEBUG 06:52:09 DEBUG [...] 06:52:09 DEBUG 06:52:09 DEBUG 06:52:09 DEBUG 06:52:09 DEBUG "instance": "10.142.0.43:9100" 06:52:09 DEBUG }, 06:52:09 DEBUG "value": [ 06:52:09 DEBUG 1713592328, 06:52:09 DEBUG "0.09866666666666407" 06:52:09 DEBUG ] 06:52:09 DEBUG } 06:52:09 DEBUG ] 06:52:09 DEBUG } 06:52:09 DEBUG } 06:52:09 INFO 06:52:09 INFO [loop_until]: kubectl --namespace=xlou top node 06:52:09 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:52:09 INFO [loop_until]: OK (rc = 0) 06:52:09 DEBUG --- stdout --- 06:52:09 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 70m 0% 6267Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7970m 26% 7433Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 17055m 57% 11357Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 81m 0% 3357Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 18645m 62% 13527Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 151m 0% 5195Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 17374m 58% 11601Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 19083m 63% 3006Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 19459m 65% 10866Mi 9% gke-xlou-cdm-ds-0f601751-j488 8137m 27% 7030Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 17243m 57% 15591Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 7972m 26% 7710Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 19351m 64% 12881Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 8166m 51% 8491Mi 14% 06:52:09 DEBUG --- stderr --- 06:52:09 DEBUG 06:52:09 INFO 06:52:09 INFO [loop_until]: kubectl --namespace=xlou top pods 06:52:09 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:52:09 INFO [loop_until]: OK (rc = 0) 06:52:09 DEBUG --- stdout --- 06:52:09 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 17129m 8647Mi am-767f6b67b5-qcqz2 17084m 8717Mi am-767f6b67b5-vbltz 18430m 8654Mi ds-cts-0 18710m 10388Mi ds-cts-1 17149m 10673Mi ds-cts-2 19315m 10269Mi ds-idrepo-0 7456m 3759Mi ds-idrepo-1 7572m 3803Mi ds-idrepo-2 7653m 3706Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1177Mi idm-b85f9d978-wnhxb 3m 1236Mi lodemon-7cb5b9456d-dskfn 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7903m 4667Mi 06:52:09 DEBUG --- stderr --- 06:52:09 DEBUG 06:53:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592388" 06:53:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592388" 06:53:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592388" 06:53:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592388" 06:53:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592388" 06:53:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592388" 06:53:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713592388" 06:53:08 INFO [http_cmd]: http status code OK 06:53:08 INFO [http_cmd]: http status code OK 06:53:08 DEBUG --- status code --- 06:53:08 DEBUG --- status code --- 06:53:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:53:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:53:08 DEBUG --- http response --- 06:53:08 DEBUG --- http response --- 06:53:08 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 06:53:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:53:08 DEBUG ----- output ----- 06:53:08 DEBUG { 06:53:08 DEBUG "status": "success", 06:53:08 DEBUG "data": { 06:53:08 DEBUG "resultType": "vector", 06:53:08 DEBUG "result": [ 06:53:08 DEBUG { 06:53:08 DEBUG "metric": { 06:53:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:53:08 DEBUG }, 06:53:08 DEBUG "value": [ 06:53:08 INFO [http_cmd]: http status code OK 06:53:08 DEBUG 06:53:08 DEBUG --- status code --- 06:53:08 DEBUG 06:53:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:53:08 DEBUG 06:53:08 DEBUG --- http response --- 06:53:08 DEBUG [...] 06:53:08 DEBUG 06:53:08 DEBUG 06:53:08 DEBUG 06:53:08 DEBUG "pod": "am-767f6b67b5-qcqz2" 06:53:08 DEBUG }, 06:53:08 DEBUG "value": [ 06:53:08 DEBUG 1713592388, 06:53:08 DEBUG "0" 06:53:08 DEBUG ] 06:53:08 DEBUG } 06:53:08 DEBUG ] 06:53:08 DEBUG } 06:53:08 DEBUG } 06:53:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:53:08 DEBUG ----- output ----- 06:53:08 DEBUG { 06:53:08 DEBUG "status": "success", 06:53:08 DEBUG "data": { 06:53:08 DEBUG "resultType": "vector", 06:53:08 DEBUG "result": [ 06:53:08 DEBUG { 06:53:08 DEBUG "metric": { 06:53:08 DEBUG "pod": "ds-idrepo-0" 06:53:08 DEBUG }, 06:53:08 DEBUG "value": [ 06:53:08 DEBUG 06:53:08 DEBUG 06:53:08 DEBUG 06:53:08 DEBUG [...] 06:53:08 DEBUG 06:53:08 DEBUG 06:53:08 DEBUG 06:53:08 DEBUG "pod": "ds-idrepo-2" 06:53:08 DEBUG }, 06:53:08 DEBUG "value": [ 06:53:08 DEBUG 1713592388, 06:53:08 DEBUG "0" 06:53:08 DEBUG ] 06:53:08 DEBUG } 06:53:08 DEBUG ] 06:53:08 DEBUG } 06:53:08 DEBUG } 06:53:08 INFO [http_cmd]: http status code OK 06:53:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713592388" 06:53:08 INFO [http_cmd]: http status code OK 06:53:08 DEBUG --- status code --- 06:53:08 DEBUG --- status code --- 06:53:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:53:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:53:08 DEBUG --- http response --- 06:53:08 DEBUG --- http response --- 06:53:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:53:08 DEBUG ----- output ----- 06:53:08 DEBUG { 06:53:08 DEBUG "status": "success", 06:53:08 DEBUG "data": { 06:53:08 DEBUG "resultType": "vector", 06:53:08 DEBUG "result": [ 06:53:08 DEBUG { 06:53:08 DEBUG "metric": { 06:53:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:53:08 DEBUG }, 06:53:08 DEBUG "value": [ 06:53:08 DEBUG 06:53:08 DEBUG 06:53:08 DEBUG 06:53:08 DEBUG [...] 06:53:08 DEBUG 06:53:08 DEBUG 06:53:08 DEBUG 06:53:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:53:08 DEBUG }, 06:53:08 DEBUG "value": [ 06:53:08 DEBUG 1713592388, 06:53:08 DEBUG "480185" 06:53:08 DEBUG ] 06:53:08 DEBUG } 06:53:08 DEBUG ] 06:53:08 DEBUG } 06:53:08 DEBUG } 06:53:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:53:08 INFO [http_cmd]: http status code OK 06:53:08 DEBUG ----- output ----- 06:53:08 DEBUG --- status code --- 06:53:08 DEBUG { 06:53:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:53:08 DEBUG "status": "success", 06:53:08 DEBUG --- http response --- 06:53:08 DEBUG "data": { 06:53:08 INFO [http_cmd]: http status code OK 06:53:08 DEBUG "resultType": "vector", 06:53:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:53:08 DEBUG --- status code --- 06:53:08 DEBUG "result": [ 06:53:08 DEBUG ----- output ----- 06:53:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:53:08 DEBUG { 06:53:08 DEBUG { 06:53:08 DEBUG --- http response --- 06:53:08 DEBUG "metric": { 06:53:08 DEBUG "status": "success", 06:53:08 DEBUG "pod": "ds-cts-0" 06:53:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:53:08 DEBUG "data": { 06:53:08 DEBUG }, 06:53:08 DEBUG ----- output ----- 06:53:08 DEBUG "resultType": "vector", 06:53:08 DEBUG "value": [ 06:53:08 DEBUG { 06:53:08 DEBUG "result": [ 06:53:08 DEBUG 06:53:08 DEBUG "status": "success", 06:53:08 DEBUG { 06:53:08 DEBUG 06:53:08 DEBUG "data": { 06:53:08 DEBUG "metric": { 06:53:08 DEBUG 06:53:08 DEBUG "resultType": "vector", 06:53:08 DEBUG "instance": "10.142.0.107:9100" 06:53:08 DEBUG [...] 06:53:08 DEBUG "result": [ 06:53:08 DEBUG }, 06:53:08 DEBUG 06:53:08 DEBUG { 06:53:08 DEBUG "value": [ 06:53:08 DEBUG 06:53:08 DEBUG "metric": { 06:53:08 DEBUG 06:53:08 DEBUG 06:53:08 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 06:53:08 DEBUG 06:53:08 DEBUG "pod": "ds-cts-2" 06:53:08 DEBUG }, 06:53:08 DEBUG 06:53:08 DEBUG }, 06:53:08 DEBUG "value": [ 06:53:08 DEBUG [...] 06:53:08 DEBUG "value": [ 06:53:08 DEBUG 06:53:08 DEBUG 06:53:08 DEBUG 1713592388, 06:53:08 DEBUG 06:53:08 DEBUG 06:53:08 DEBUG "30" 06:53:08 DEBUG 06:53:08 DEBUG 06:53:08 DEBUG ] 06:53:08 DEBUG [...] 06:53:08 DEBUG "instance": "10.142.0.43:9100" 06:53:08 DEBUG } 06:53:08 DEBUG 06:53:08 DEBUG }, 06:53:08 DEBUG ] 06:53:08 DEBUG 06:53:08 DEBUG "value": [ 06:53:08 DEBUG } 06:53:08 DEBUG 06:53:08 DEBUG 1713592388, 06:53:08 DEBUG } 06:53:08 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 06:53:08 DEBUG "3234993343488" 06:53:08 DEBUG }, 06:53:08 DEBUG ] 06:53:08 DEBUG "value": [ 06:53:08 DEBUG } 06:53:08 DEBUG 1713592388, 06:53:08 DEBUG ] 06:53:08 DEBUG "4991053824" 06:53:08 DEBUG } 06:53:08 DEBUG ] 06:53:08 DEBUG } 06:53:08 DEBUG } 06:53:08 DEBUG ] 06:53:08 DEBUG } 06:53:08 DEBUG } 06:53:08 ERROR [http_cmd]: ERROR 06:53:08 ERROR --- status code --- 06:53:08 ERROR http status code is 400 (expected [200, 401, 403, 404]) 06:53:08 ERROR --- http response --- 06:53:08 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 06:53:08 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 06:53:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592388" 06:53:08 INFO [http_cmd]: http status code OK 06:53:08 DEBUG --- status code --- 06:53:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:53:08 DEBUG --- http response --- 06:53:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:53:08 DEBUG ----- output ----- 06:53:08 DEBUG { 06:53:08 DEBUG "status": "success", 06:53:08 DEBUG "data": { 06:53:08 DEBUG "resultType": "vector", 06:53:08 DEBUG "result": [ 06:53:08 DEBUG { 06:53:08 DEBUG "metric": { 06:53:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:53:08 DEBUG }, 06:53:08 DEBUG "value": [ 06:53:08 DEBUG 06:53:08 DEBUG 06:53:08 DEBUG 06:53:08 DEBUG [...] 06:53:08 DEBUG 06:53:08 DEBUG 06:53:08 DEBUG 06:53:08 DEBUG "pod": "am-767f6b67b5-qcqz2" 06:53:08 DEBUG }, 06:53:08 DEBUG "value": [ 06:53:08 DEBUG 1713592388, 06:53:08 DEBUG "1369.695210574196" 06:53:08 DEBUG ] 06:53:08 DEBUG } 06:53:08 DEBUG ] 06:53:08 DEBUG } 06:53:08 DEBUG } 06:53:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592388" 06:53:08 INFO [http_cmd]: http status code OK 06:53:08 DEBUG --- status code --- 06:53:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:53:08 DEBUG --- http response --- 06:53:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:53:08 DEBUG ----- output ----- 06:53:08 DEBUG { 06:53:08 DEBUG "status": "success", 06:53:08 DEBUG "data": { 06:53:08 DEBUG "resultType": "vector", 06:53:08 DEBUG "result": [ 06:53:08 DEBUG { 06:53:08 DEBUG "metric": { 06:53:08 DEBUG "pod": "ds-idrepo-0" 06:53:08 DEBUG }, 06:53:08 DEBUG "value": [ 06:53:08 DEBUG 06:53:08 DEBUG 06:53:08 DEBUG 06:53:08 DEBUG [...] 06:53:08 DEBUG 06:53:08 DEBUG 06:53:08 DEBUG 06:53:08 DEBUG "pod": "ds-idrepo-2" 06:53:08 DEBUG }, 06:53:08 DEBUG "value": [ 06:53:08 DEBUG 1713592388, 06:53:08 DEBUG "0" 06:53:08 DEBUG ] 06:53:08 DEBUG } 06:53:08 DEBUG ] 06:53:08 DEBUG } 06:53:08 DEBUG } 06:53:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592388" 06:53:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592388" 06:53:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713592388" 06:53:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592388" 06:53:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592388" 06:53:08 INFO [http_cmd]: http status code OK 06:53:08 INFO [http_cmd]: http status code OK 06:53:08 DEBUG --- status code --- 06:53:08 DEBUG --- status code --- 06:53:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:53:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:53:08 DEBUG --- http response --- 06:53:08 DEBUG --- http response --- 06:53:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:53:08 DEBUG ----- output ----- 06:53:08 DEBUG { 06:53:08 DEBUG "status": "success", 06:53:08 DEBUG "data": { 06:53:08 DEBUG "resultType": "vector", 06:53:08 DEBUG "result": [ 06:53:08 DEBUG { 06:53:08 DEBUG "metric": { 06:53:08 DEBUG "pod": "ds-cts-0" 06:53:08 DEBUG }, 06:53:08 DEBUG "value": [ 06:53:08 DEBUG 06:53:08 DEBUG 06:53:08 DEBUG 06:53:08 DEBUG [...] 06:53:08 DEBUG 06:53:08 DEBUG 06:53:08 DEBUG 06:53:08 DEBUG "pod": "ds-cts-2" 06:53:08 DEBUG }, 06:53:08 DEBUG "value": [ 06:53:08 DEBUG 1713592388, 06:53:08 DEBUG "0" 06:53:08 DEBUG ] 06:53:08 DEBUG } 06:53:08 DEBUG ] 06:53:08 DEBUG } 06:53:08 DEBUG } 06:53:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:53:08 DEBUG ----- output ----- 06:53:08 DEBUG { 06:53:08 DEBUG "status": "success", 06:53:08 DEBUG "data": { 06:53:08 DEBUG "resultType": "vector", 06:53:08 DEBUG "result": [ 06:53:08 DEBUG { 06:53:08 DEBUG "metric": { 06:53:08 DEBUG "pod": "ds-idrepo-0" 06:53:08 DEBUG }, 06:53:08 DEBUG "value": [ 06:53:08 DEBUG 06:53:08 DEBUG 06:53:08 DEBUG 06:53:08 DEBUG [...] 06:53:08 DEBUG 06:53:08 DEBUG 06:53:08 DEBUG 06:53:08 DEBUG "pod": "ds-cts-2" 06:53:08 DEBUG }, 06:53:08 DEBUG "value": [ 06:53:08 DEBUG 1713592388, 06:53:08 DEBUG "0" 06:53:08 DEBUG ] 06:53:08 DEBUG } 06:53:08 DEBUG ] 06:53:08 DEBUG } 06:53:08 DEBUG } 06:53:08 INFO [http_cmd]: http status code OK 06:53:08 DEBUG --- status code --- 06:53:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:53:08 DEBUG --- http response --- 06:53:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:53:08 DEBUG ----- output ----- 06:53:08 DEBUG { 06:53:08 DEBUG "status": "success", 06:53:08 DEBUG "data": { 06:53:08 DEBUG "resultType": "vector", 06:53:08 DEBUG "result": [ 06:53:08 DEBUG { 06:53:08 DEBUG "metric": { 06:53:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:53:08 DEBUG }, 06:53:08 DEBUG "value": [ 06:53:08 DEBUG 06:53:08 DEBUG 06:53:08 DEBUG 06:53:08 DEBUG [...] 06:53:08 DEBUG 06:53:08 DEBUG 06:53:08 DEBUG 06:53:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:53:08 DEBUG }, 06:53:08 DEBUG "value": [ 06:53:08 DEBUG 1713592388, 06:53:08 DEBUG "0" 06:53:08 DEBUG ] 06:53:08 DEBUG } 06:53:08 DEBUG ] 06:53:08 DEBUG } 06:53:08 DEBUG } 06:53:08 INFO [http_cmd]: http status code OK 06:53:08 DEBUG --- status code --- 06:53:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:53:08 DEBUG --- http response --- 06:53:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:53:08 INFO [http_cmd]: http status code OK 06:53:08 DEBUG ----- output ----- 06:53:08 DEBUG --- status code --- 06:53:08 DEBUG { 06:53:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:53:08 DEBUG "status": "success", 06:53:08 DEBUG --- http response --- 06:53:08 DEBUG "data": { 06:53:08 DEBUG "resultType": "vector", 06:53:08 DEBUG "result": [ 06:53:08 DEBUG { 06:53:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 06:53:08 DEBUG "metric": { 06:53:08 DEBUG ----- output ----- 06:53:08 DEBUG "instance": "10.142.0.107:9100" 06:53:08 DEBUG { 06:53:08 DEBUG }, 06:53:08 DEBUG "status": "success", 06:53:08 DEBUG "value": [ 06:53:08 DEBUG "data": { 06:53:08 DEBUG 06:53:08 DEBUG "resultType": "vector", 06:53:08 DEBUG 06:53:08 DEBUG "result": [ 06:53:08 DEBUG 06:53:08 DEBUG { 06:53:08 DEBUG [...] 06:53:08 DEBUG "metric": { 06:53:08 DEBUG 06:53:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:53:08 DEBUG 06:53:08 DEBUG }, 06:53:08 DEBUG 06:53:08 DEBUG "value": [ 06:53:08 DEBUG "instance": "10.142.0.43:9100" 06:53:08 DEBUG 06:53:08 DEBUG }, 06:53:08 DEBUG 06:53:08 DEBUG "value": [ 06:53:08 DEBUG 06:53:08 DEBUG 1713592388, 06:53:08 DEBUG [...] 06:53:08 DEBUG "0.26436666666677033" 06:53:08 DEBUG 06:53:08 DEBUG ] 06:53:08 DEBUG 06:53:08 DEBUG } 06:53:08 DEBUG 06:53:08 DEBUG ] 06:53:08 DEBUG "pod": "am-767f6b67b5-qcqz2" 06:53:08 DEBUG } 06:53:08 DEBUG }, 06:53:08 DEBUG } 06:53:08 DEBUG "value": [ 06:53:08 DEBUG 1713592388, 06:53:08 DEBUG "2853.5316886962414" 06:53:08 DEBUG ] 06:53:08 DEBUG } 06:53:08 DEBUG ] 06:53:08 DEBUG } 06:53:08 DEBUG } 06:53:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592388" 06:53:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592388" 06:53:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592388" 06:53:08 INFO [http_cmd]: http status code OK 06:53:08 DEBUG --- status code --- 06:53:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:53:08 DEBUG --- http response --- 06:53:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:53:08 DEBUG ----- output ----- 06:53:08 DEBUG { 06:53:08 DEBUG "status": "success", 06:53:08 DEBUG "data": { 06:53:08 DEBUG "resultType": "vector", 06:53:08 DEBUG "result": [ 06:53:08 DEBUG { 06:53:08 DEBUG "metric": { 06:53:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:53:08 DEBUG }, 06:53:08 DEBUG "value": [ 06:53:08 DEBUG 06:53:08 DEBUG 06:53:08 DEBUG 06:53:08 DEBUG [...] 06:53:08 DEBUG 06:53:08 DEBUG 06:53:08 DEBUG 06:53:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:53:08 DEBUG }, 06:53:08 DEBUG "value": [ 06:53:08 DEBUG 1713592388, 06:53:08 DEBUG "2673.5" 06:53:08 DEBUG ] 06:53:08 DEBUG } 06:53:08 DEBUG ] 06:53:08 DEBUG } 06:53:08 DEBUG } 06:53:08 INFO [http_cmd]: http status code OK 06:53:08 DEBUG --- status code --- 06:53:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:53:08 DEBUG --- http response --- 06:53:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:53:08 DEBUG ----- output ----- 06:53:08 DEBUG { 06:53:08 DEBUG "status": "success", 06:53:08 DEBUG "data": { 06:53:08 DEBUG "resultType": "vector", 06:53:08 DEBUG "result": [ 06:53:08 DEBUG { 06:53:08 DEBUG "metric": { 06:53:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:53:08 DEBUG }, 06:53:08 DEBUG "value": [ 06:53:08 DEBUG 06:53:08 INFO [http_cmd]: http status code OK 06:53:08 DEBUG 06:53:08 DEBUG --- status code --- 06:53:08 DEBUG 06:53:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:53:08 DEBUG [...] 06:53:08 DEBUG --- http response --- 06:53:08 DEBUG 06:53:08 DEBUG 06:53:08 DEBUG 06:53:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:53:08 DEBUG }, 06:53:08 DEBUG "value": [ 06:53:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:53:08 DEBUG 1713592388, 06:53:08 DEBUG ----- output ----- 06:53:08 DEBUG "2669.4333333333334" 06:53:08 DEBUG { 06:53:08 DEBUG ] 06:53:08 DEBUG "status": "success", 06:53:08 DEBUG } 06:53:08 DEBUG "data": { 06:53:08 DEBUG ] 06:53:08 DEBUG "resultType": "vector", 06:53:08 DEBUG } 06:53:08 DEBUG "result": [ 06:53:08 DEBUG } 06:53:08 DEBUG { 06:53:08 DEBUG "metric": { 06:53:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:53:08 DEBUG }, 06:53:08 DEBUG "value": [ 06:53:08 DEBUG 06:53:08 DEBUG 06:53:08 DEBUG 06:53:08 DEBUG [...] 06:53:08 DEBUG 06:53:08 DEBUG 06:53:08 DEBUG 06:53:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:53:08 DEBUG }, 06:53:08 DEBUG "value": [ 06:53:08 DEBUG 1713592388, 06:53:08 DEBUG "2668.8333333333335" 06:53:08 DEBUG ] 06:53:08 DEBUG } 06:53:08 DEBUG ] 06:53:08 DEBUG } 06:53:08 DEBUG } 06:53:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592388" 06:53:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713592388" 06:53:08 INFO [http_cmd]: http status code OK 06:53:08 DEBUG --- status code --- 06:53:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:53:08 DEBUG --- http response --- 06:53:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:53:08 DEBUG ----- output ----- 06:53:08 DEBUG { 06:53:08 DEBUG "status": "success", 06:53:08 DEBUG "data": { 06:53:08 DEBUG "resultType": "vector", 06:53:08 DEBUG "result": [ 06:53:08 DEBUG { 06:53:08 DEBUG "metric": { 06:53:08 DEBUG "pod": "ds-cts-0" 06:53:08 DEBUG }, 06:53:08 DEBUG "value": [ 06:53:08 DEBUG 06:53:08 DEBUG 06:53:08 DEBUG 06:53:08 DEBUG [...] 06:53:08 DEBUG 06:53:08 DEBUG 06:53:08 DEBUG 06:53:08 DEBUG "pod": "ds-cts-2" 06:53:08 DEBUG }, 06:53:08 DEBUG "value": [ 06:53:08 DEBUG 1713592388, 06:53:08 DEBUG "0.00006666666666666667" 06:53:08 DEBUG ] 06:53:08 DEBUG } 06:53:08 DEBUG ] 06:53:08 DEBUG } 06:53:08 DEBUG } 06:53:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592388" 06:53:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592388" 06:53:08 INFO [http_cmd]: http status code OK 06:53:08 DEBUG --- status code --- 06:53:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:53:08 DEBUG --- http response --- 06:53:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:53:08 DEBUG ----- output ----- 06:53:08 DEBUG { 06:53:08 DEBUG "status": "success", 06:53:08 DEBUG "data": { 06:53:08 DEBUG "resultType": "vector", 06:53:08 DEBUG "result": [ 06:53:08 DEBUG { 06:53:08 DEBUG "metric": { 06:53:08 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 06:53:08 DEBUG }, 06:53:08 DEBUG "value": [ 06:53:08 DEBUG 06:53:08 DEBUG 06:53:08 DEBUG 06:53:08 DEBUG [...] 06:53:08 DEBUG 06:53:08 DEBUG 06:53:08 DEBUG 06:53:08 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 06:53:08 DEBUG }, 06:53:08 DEBUG "value": [ 06:53:08 DEBUG 1713592388, 06:53:08 DEBUG "8.038938763436601" 06:53:08 DEBUG ] 06:53:08 DEBUG } 06:53:08 DEBUG ] 06:53:08 DEBUG } 06:53:08 DEBUG } 06:53:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592388" 06:53:08 INFO [http_cmd]: http status code OK 06:53:08 DEBUG --- status code --- 06:53:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:53:08 DEBUG --- http response --- 06:53:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:53:08 DEBUG ----- output ----- 06:53:08 DEBUG { 06:53:08 DEBUG "status": "success", 06:53:08 DEBUG "data": { 06:53:08 DEBUG "resultType": "vector", 06:53:08 DEBUG "result": [ 06:53:08 DEBUG { 06:53:08 DEBUG "metric": { 06:53:08 DEBUG "pod": "ds-idrepo-0" 06:53:08 DEBUG }, 06:53:08 DEBUG "value": [ 06:53:08 DEBUG 06:53:08 DEBUG 06:53:08 DEBUG 06:53:08 DEBUG [...] 06:53:08 DEBUG 06:53:08 DEBUG 06:53:08 INFO [http_cmd]: http status code OK 06:53:08 DEBUG 06:53:08 DEBUG --- status code --- 06:53:08 DEBUG "pod": "ds-idrepo-2" 06:53:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:53:08 DEBUG }, 06:53:08 DEBUG --- http response --- 06:53:08 DEBUG "value": [ 06:53:08 DEBUG 1713592388, 06:53:08 DEBUG "0" 06:53:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:53:08 DEBUG ] 06:53:08 DEBUG ----- output ----- 06:53:08 DEBUG } 06:53:08 DEBUG { 06:53:08 DEBUG ] 06:53:08 DEBUG "status": "success", 06:53:08 DEBUG } 06:53:08 DEBUG "data": { 06:53:08 DEBUG } 06:53:08 DEBUG "resultType": "vector", 06:53:08 DEBUG "result": [ 06:53:08 DEBUG { 06:53:08 DEBUG "metric": { 06:53:08 DEBUG "instance": "10.142.0.107:9100" 06:53:08 DEBUG }, 06:53:08 DEBUG "value": [ 06:53:08 DEBUG 06:53:08 DEBUG 06:53:08 DEBUG 06:53:08 DEBUG [...] 06:53:08 DEBUG 06:53:08 DEBUG 06:53:08 DEBUG 06:53:08 DEBUG "instance": "10.142.0.43:9100" 06:53:08 DEBUG }, 06:53:08 DEBUG "value": [ 06:53:08 DEBUG 1713592388, 06:53:08 DEBUG "80872580.7" 06:53:08 DEBUG ] 06:53:08 DEBUG } 06:53:08 DEBUG ] 06:53:08 DEBUG } 06:53:08 DEBUG } 06:53:08 INFO [http_cmd]: http status code OK 06:53:08 DEBUG --- status code --- 06:53:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:53:08 DEBUG --- http response --- 06:53:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:53:08 DEBUG ----- output ----- 06:53:08 DEBUG { 06:53:08 DEBUG "status": "success", 06:53:08 DEBUG "data": { 06:53:08 DEBUG "resultType": "vector", 06:53:08 DEBUG "result": [ 06:53:08 DEBUG { 06:53:08 DEBUG "metric": { 06:53:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:53:08 DEBUG }, 06:53:08 DEBUG "value": [ 06:53:08 DEBUG 06:53:08 DEBUG 06:53:08 DEBUG 06:53:08 DEBUG [...] 06:53:08 DEBUG 06:53:08 DEBUG 06:53:08 DEBUG 06:53:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:53:08 DEBUG }, 06:53:08 DEBUG "value": [ 06:53:08 DEBUG 1713592388, 06:53:08 DEBUG "2669.3333333333335" 06:53:08 DEBUG ] 06:53:08 DEBUG } 06:53:08 DEBUG ] 06:53:08 DEBUG } 06:53:08 DEBUG } 06:53:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713592388" 06:53:08 INFO [http_cmd]: http status code OK 06:53:08 DEBUG --- status code --- 06:53:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:53:08 DEBUG --- http response --- 06:53:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:53:08 DEBUG ----- output ----- 06:53:08 DEBUG { 06:53:08 DEBUG "status": "success", 06:53:08 DEBUG "data": { 06:53:08 DEBUG "resultType": "vector", 06:53:08 DEBUG "result": [ 06:53:08 DEBUG { 06:53:08 DEBUG "metric": { 06:53:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:53:08 DEBUG }, 06:53:08 DEBUG "value": [ 06:53:08 DEBUG 06:53:08 DEBUG 06:53:08 DEBUG 06:53:08 DEBUG [...] 06:53:08 DEBUG 06:53:08 DEBUG 06:53:08 DEBUG 06:53:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:53:08 DEBUG }, 06:53:08 DEBUG "value": [ 06:53:08 DEBUG 1713592388, 06:53:08 DEBUG "0.096" 06:53:08 DEBUG ] 06:53:08 DEBUG } 06:53:08 DEBUG ] 06:53:08 DEBUG } 06:53:08 DEBUG } 06:53:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592389" 06:53:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592389" 06:53:09 INFO [http_cmd]: http status code OK 06:53:09 INFO [http_cmd]: http status code OK 06:53:09 DEBUG --- status code --- 06:53:09 DEBUG --- status code --- 06:53:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:53:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:53:09 DEBUG --- http response --- 06:53:09 DEBUG --- http response --- 06:53:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:53:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:53:09 DEBUG ----- output ----- 06:53:09 DEBUG ----- output ----- 06:53:09 DEBUG { 06:53:09 DEBUG { 06:53:09 DEBUG "status": "success", 06:53:09 DEBUG "status": "success", 06:53:09 DEBUG "data": { 06:53:09 DEBUG "data": { 06:53:09 DEBUG "resultType": "vector", 06:53:09 DEBUG "resultType": "vector", 06:53:09 DEBUG "result": [ 06:53:09 DEBUG "result": [ 06:53:09 DEBUG { 06:53:09 DEBUG { 06:53:09 DEBUG "metric": { 06:53:09 DEBUG "metric": { 06:53:09 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:53:09 DEBUG "instance": "10.142.0.107:9100" 06:53:09 DEBUG }, 06:53:09 DEBUG }, 06:53:09 DEBUG "value": [ 06:53:09 DEBUG "value": [ 06:53:09 DEBUG 06:53:09 DEBUG 06:53:09 DEBUG 06:53:09 DEBUG 06:53:09 DEBUG 06:53:09 DEBUG 06:53:09 DEBUG [...] 06:53:09 DEBUG [...] 06:53:09 DEBUG 06:53:09 DEBUG 06:53:09 DEBUG 06:53:09 DEBUG 06:53:09 DEBUG 06:53:09 DEBUG 06:53:09 DEBUG "pod": "am-767f6b67b5-qcqz2" 06:53:09 DEBUG "instance": "10.142.0.43:9100" 06:53:09 DEBUG }, 06:53:09 DEBUG }, 06:53:09 DEBUG "value": [ 06:53:09 DEBUG "value": [ 06:53:09 DEBUG 1713592389, 06:53:09 DEBUG 1713592389, 06:53:09 DEBUG "8.728816477362837" 06:53:09 DEBUG "80739457.56666666" 06:53:09 DEBUG ] 06:53:09 DEBUG ] 06:53:09 DEBUG } 06:53:09 DEBUG } 06:53:09 DEBUG ] 06:53:09 DEBUG ] 06:53:09 DEBUG } 06:53:09 DEBUG } 06:53:09 DEBUG } 06:53:09 DEBUG } 06:53:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592389" 06:53:09 INFO [http_cmd]: http status code OK 06:53:09 DEBUG --- status code --- 06:53:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:53:09 DEBUG --- http response --- 06:53:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:53:09 DEBUG ----- output ----- 06:53:09 DEBUG { 06:53:09 DEBUG "status": "success", 06:53:09 DEBUG "data": { 06:53:09 DEBUG "resultType": "vector", 06:53:09 DEBUG "result": [ 06:53:09 DEBUG { 06:53:09 DEBUG "metric": { 06:53:09 DEBUG "instance": "10.142.0.107:9100" 06:53:09 DEBUG }, 06:53:09 DEBUG "value": [ 06:53:09 DEBUG 06:53:09 DEBUG 06:53:09 DEBUG 06:53:09 DEBUG [...] 06:53:09 DEBUG 06:53:09 DEBUG 06:53:09 DEBUG 06:53:09 DEBUG "instance": "10.142.0.43:9100" 06:53:09 DEBUG }, 06:53:09 DEBUG "value": [ 06:53:09 DEBUG 1713592389, 06:53:09 DEBUG "0.06266666666666271" 06:53:09 DEBUG ] 06:53:09 DEBUG } 06:53:09 DEBUG ] 06:53:09 DEBUG } 06:53:09 DEBUG } 06:53:09 INFO 06:53:09 INFO [loop_until]: kubectl --namespace=xlou top node 06:53:09 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:53:09 INFO 06:53:09 INFO [loop_until]: kubectl --namespace=xlou top pods 06:53:09 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:53:09 INFO [loop_until]: OK (rc = 0) 06:53:09 DEBUG --- stdout --- 06:53:09 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 69m 0% 6261Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7997m 26% 7435Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 17079m 57% 11351Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 81m 0% 3354Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 18635m 62% 13528Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 153m 0% 5220Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 16909m 56% 11596Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 18473m 61% 3017Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 18854m 63% 10864Mi 9% gke-xlou-cdm-ds-0f601751-j488 8145m 27% 7042Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 16842m 56% 15598Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 8050m 26% 7696Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 19001m 63% 12868Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 8166m 51% 8490Mi 14% 06:53:09 DEBUG --- stderr --- 06:53:09 DEBUG 06:53:10 INFO [loop_until]: OK (rc = 0) 06:53:10 DEBUG --- stdout --- 06:53:10 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 17360m 8650Mi am-767f6b67b5-qcqz2 17280m 8716Mi am-767f6b67b5-vbltz 18787m 8653Mi ds-cts-0 18807m 10387Mi ds-cts-1 16826m 10685Mi ds-cts-2 18949m 10269Mi ds-idrepo-0 7733m 3763Mi ds-idrepo-1 7791m 3802Mi ds-idrepo-2 7664m 3705Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1177Mi idm-b85f9d978-wnhxb 3m 1236Mi lodemon-7cb5b9456d-dskfn 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7991m 4668Mi 06:53:10 DEBUG --- stderr --- 06:53:10 DEBUG 06:54:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592448" 06:54:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592448" 06:54:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592448" 06:54:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592448" 06:54:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592448" 06:54:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713592448" 06:54:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592448" 06:54:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713592448" 06:54:08 INFO [http_cmd]: http status code OK 06:54:08 INFO [http_cmd]: http status code OK 06:54:08 DEBUG --- status code --- 06:54:08 INFO [http_cmd]: http status code OK 06:54:08 INFO [http_cmd]: http status code OK 06:54:08 INFO [http_cmd]: http status code OK 06:54:08 INFO [http_cmd]: http status code OK 06:54:08 INFO [http_cmd]: http status code OK 06:54:08 DEBUG --- status code --- 06:54:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:54:08 DEBUG --- status code --- 06:54:08 DEBUG --- status code --- 06:54:08 DEBUG --- status code --- 06:54:08 DEBUG --- status code --- 06:54:08 DEBUG --- status code --- 06:54:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:54:08 DEBUG --- http response --- 06:54:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:54:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:54:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:54:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:54:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:54:08 DEBUG --- http response --- 06:54:08 DEBUG --- http response --- 06:54:08 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 06:54:08 DEBUG --- http response --- 06:54:08 DEBUG --- http response --- 06:54:08 DEBUG --- http response --- 06:54:08 DEBUG --- http response --- 06:54:08 ERROR [http_cmd]: ERROR 06:54:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:54:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:54:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (107 lines): 06:54:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:54:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:54:08 ERROR --- status code --- 06:54:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:54:08 DEBUG ----- output ----- 06:54:08 DEBUG ----- output ----- 06:54:08 DEBUG ----- output ----- 06:54:08 DEBUG ----- output ----- 06:54:08 DEBUG ----- output ----- 06:54:08 ERROR http status code is 400 (expected [200, 401, 403, 404]) 06:54:08 DEBUG ----- output ----- 06:54:08 DEBUG { 06:54:08 DEBUG { 06:54:08 DEBUG { 06:54:08 DEBUG { 06:54:08 DEBUG { 06:54:08 ERROR --- http response --- 06:54:08 DEBUG { 06:54:08 DEBUG "status": "success", 06:54:08 DEBUG "status": "success", 06:54:08 DEBUG "status": "success", 06:54:08 DEBUG "status": "success", 06:54:08 DEBUG "status": "success", 06:54:08 DEBUG "status": "success", 06:54:08 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 06:54:08 DEBUG "data": { 06:54:08 DEBUG "data": { 06:54:08 DEBUG "data": { 06:54:08 DEBUG "data": { 06:54:08 DEBUG "data": { 06:54:08 DEBUG "data": { 06:54:08 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 06:54:08 DEBUG "resultType": "vector", 06:54:08 DEBUG "resultType": "vector", 06:54:08 DEBUG "resultType": "vector", 06:54:08 DEBUG "resultType": "vector", 06:54:08 DEBUG "resultType": "vector", 06:54:08 DEBUG "resultType": "vector", 06:54:08 DEBUG "result": [ 06:54:08 DEBUG "result": [ 06:54:08 DEBUG "result": [ 06:54:08 DEBUG "result": [ 06:54:08 DEBUG "result": [ 06:54:08 DEBUG "result": [ 06:54:08 DEBUG { 06:54:08 DEBUG { 06:54:08 DEBUG { 06:54:08 DEBUG { 06:54:08 DEBUG { 06:54:08 DEBUG { 06:54:08 DEBUG "metric": { 06:54:08 DEBUG "metric": { 06:54:08 DEBUG "metric": { 06:54:08 DEBUG "metric": { 06:54:08 DEBUG "metric": { 06:54:08 DEBUG "metric": { 06:54:08 DEBUG "pod": "ds-cts-0" 06:54:08 DEBUG "pod": "ds-idrepo-0" 06:54:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:54:08 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 06:54:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:54:08 DEBUG "instance": "10.142.0.107:9100" 06:54:08 DEBUG }, 06:54:08 DEBUG }, 06:54:08 DEBUG }, 06:54:08 DEBUG }, 06:54:08 DEBUG }, 06:54:08 DEBUG }, 06:54:08 DEBUG "value": [ 06:54:08 DEBUG "value": [ 06:54:08 DEBUG "value": [ 06:54:08 DEBUG "value": [ 06:54:08 DEBUG "value": [ 06:54:08 DEBUG "value": [ 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG [...] 06:54:08 DEBUG [...] 06:54:08 DEBUG [...] 06:54:08 DEBUG [...] 06:54:08 DEBUG [...] 06:54:08 DEBUG [...] 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG "pod": "ds-cts-2" 06:54:08 DEBUG "pod": "ds-idrepo-2" 06:54:08 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 06:54:08 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 06:54:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:54:08 DEBUG "instance": "10.142.0.43:9100" 06:54:08 DEBUG }, 06:54:08 DEBUG }, 06:54:08 DEBUG }, 06:54:08 DEBUG }, 06:54:08 DEBUG }, 06:54:08 DEBUG }, 06:54:08 DEBUG "value": [ 06:54:08 DEBUG "value": [ 06:54:08 DEBUG "value": [ 06:54:08 DEBUG "value": [ 06:54:08 DEBUG "value": [ 06:54:08 DEBUG "value": [ 06:54:08 DEBUG 1713592448, 06:54:08 DEBUG 1713592448, 06:54:08 DEBUG 1713592448, 06:54:08 DEBUG 1713592448, 06:54:08 DEBUG 1713592448, 06:54:08 DEBUG 1713592448, 06:54:08 DEBUG "25.2" 06:54:08 DEBUG "0" 06:54:08 DEBUG "0" 06:54:08 DEBUG "4994846720" 06:54:08 DEBUG "479994.5" 06:54:08 DEBUG "3239121816576" 06:54:08 DEBUG ] 06:54:08 DEBUG ] 06:54:08 DEBUG ] 06:54:08 DEBUG ] 06:54:08 DEBUG ] 06:54:08 DEBUG ] 06:54:08 DEBUG } 06:54:08 DEBUG } 06:54:08 DEBUG } 06:54:08 DEBUG } 06:54:08 DEBUG } 06:54:08 DEBUG } 06:54:08 DEBUG ] 06:54:08 DEBUG ] 06:54:08 DEBUG ] 06:54:08 DEBUG ] 06:54:08 DEBUG ] 06:54:08 DEBUG ] 06:54:08 DEBUG } 06:54:08 DEBUG } 06:54:08 DEBUG } 06:54:08 DEBUG } 06:54:08 DEBUG } 06:54:08 DEBUG } 06:54:08 DEBUG } 06:54:08 DEBUG } 06:54:08 DEBUG } 06:54:08 DEBUG } 06:54:08 DEBUG } 06:54:08 DEBUG } 06:54:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592448" 06:54:08 INFO [http_cmd]: http status code OK 06:54:08 DEBUG --- status code --- 06:54:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:54:08 DEBUG --- http response --- 06:54:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 06:54:08 DEBUG ----- output ----- 06:54:08 DEBUG { 06:54:08 DEBUG "status": "success", 06:54:08 DEBUG "data": { 06:54:08 DEBUG "resultType": "vector", 06:54:08 DEBUG "result": [ 06:54:08 DEBUG { 06:54:08 DEBUG "metric": { 06:54:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:54:08 DEBUG }, 06:54:08 DEBUG "value": [ 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG [...] 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 06:54:08 DEBUG }, 06:54:08 DEBUG "value": [ 06:54:08 DEBUG 1713592448, 06:54:08 DEBUG "0" 06:54:08 DEBUG ] 06:54:08 DEBUG } 06:54:08 DEBUG ] 06:54:08 DEBUG } 06:54:08 DEBUG } 06:54:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592448" 06:54:08 INFO [http_cmd]: http status code OK 06:54:08 DEBUG --- status code --- 06:54:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:54:08 DEBUG --- http response --- 06:54:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:54:08 DEBUG ----- output ----- 06:54:08 DEBUG { 06:54:08 DEBUG "status": "success", 06:54:08 DEBUG "data": { 06:54:08 DEBUG "resultType": "vector", 06:54:08 DEBUG "result": [ 06:54:08 DEBUG { 06:54:08 DEBUG "metric": { 06:54:08 DEBUG "pod": "ds-idrepo-0" 06:54:08 DEBUG }, 06:54:08 DEBUG "value": [ 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG [...] 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG "pod": "ds-idrepo-2" 06:54:08 DEBUG }, 06:54:08 DEBUG "value": [ 06:54:08 DEBUG 1713592448, 06:54:08 DEBUG "0" 06:54:08 DEBUG ] 06:54:08 DEBUG } 06:54:08 DEBUG ] 06:54:08 DEBUG } 06:54:08 DEBUG } 06:54:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592448" 06:54:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592448" 06:54:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592448" 06:54:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592448" 06:54:08 INFO [http_cmd]: http status code OK 06:54:08 DEBUG --- status code --- 06:54:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:54:08 DEBUG --- http response --- 06:54:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:54:08 DEBUG ----- output ----- 06:54:08 DEBUG { 06:54:08 DEBUG "status": "success", 06:54:08 DEBUG "data": { 06:54:08 DEBUG "resultType": "vector", 06:54:08 DEBUG "result": [ 06:54:08 DEBUG { 06:54:08 DEBUG "metric": { 06:54:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:54:08 DEBUG }, 06:54:08 DEBUG "value": [ 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG [...] 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:54:08 DEBUG }, 06:54:08 DEBUG "value": [ 06:54:08 DEBUG 1713592448, 06:54:08 DEBUG "0" 06:54:08 DEBUG ] 06:54:08 DEBUG } 06:54:08 DEBUG ] 06:54:08 DEBUG } 06:54:08 DEBUG } 06:54:08 INFO [http_cmd]: http status code OK 06:54:08 DEBUG --- status code --- 06:54:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:54:08 DEBUG --- http response --- 06:54:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:54:08 DEBUG ----- output ----- 06:54:08 DEBUG { 06:54:08 DEBUG "status": "success", 06:54:08 DEBUG "data": { 06:54:08 DEBUG "resultType": "vector", 06:54:08 DEBUG "result": [ 06:54:08 DEBUG { 06:54:08 DEBUG "metric": { 06:54:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:54:08 DEBUG }, 06:54:08 DEBUG "value": [ 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG [...] 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG "pod": "ds-cts-2" 06:54:08 DEBUG }, 06:54:08 DEBUG "value": [ 06:54:08 DEBUG 1713592448, 06:54:08 DEBUG "0" 06:54:08 DEBUG ] 06:54:08 DEBUG } 06:54:08 DEBUG ] 06:54:08 DEBUG } 06:54:08 DEBUG } 06:54:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713592448" 06:54:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592448" 06:54:08 INFO [http_cmd]: http status code OK 06:54:08 INFO [http_cmd]: http status code OK 06:54:08 DEBUG --- status code --- 06:54:08 DEBUG --- status code --- 06:54:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:54:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:54:08 DEBUG --- http response --- 06:54:08 DEBUG --- http response --- 06:54:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:54:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:54:08 DEBUG ----- output ----- 06:54:08 DEBUG ----- output ----- 06:54:08 DEBUG { 06:54:08 DEBUG { 06:54:08 DEBUG "status": "success", 06:54:08 DEBUG "status": "success", 06:54:08 DEBUG "data": { 06:54:08 DEBUG "data": { 06:54:08 DEBUG "resultType": "vector", 06:54:08 DEBUG "resultType": "vector", 06:54:08 DEBUG "result": [ 06:54:08 DEBUG "result": [ 06:54:08 DEBUG { 06:54:08 DEBUG { 06:54:08 DEBUG "metric": { 06:54:08 DEBUG "metric": { 06:54:08 DEBUG "pod": "ds-cts-0" 06:54:08 DEBUG "instance": "10.142.0.107:9100" 06:54:08 DEBUG }, 06:54:08 DEBUG }, 06:54:08 DEBUG "value": [ 06:54:08 DEBUG "value": [ 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG [...] 06:54:08 DEBUG [...] 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG "pod": "ds-cts-2" 06:54:08 DEBUG "instance": "10.142.0.43:9100" 06:54:08 DEBUG }, 06:54:08 DEBUG }, 06:54:08 DEBUG "value": [ 06:54:08 DEBUG "value": [ 06:54:08 DEBUG 1713592448, 06:54:08 DEBUG 1713592448, 06:54:08 DEBUG "0" 06:54:08 DEBUG "0.2129" 06:54:08 DEBUG ] 06:54:08 DEBUG ] 06:54:08 DEBUG } 06:54:08 DEBUG } 06:54:08 DEBUG ] 06:54:08 DEBUG ] 06:54:08 DEBUG } 06:54:08 DEBUG } 06:54:08 DEBUG } 06:54:08 DEBUG } 06:54:08 INFO [http_cmd]: http status code OK 06:54:08 INFO [http_cmd]: http status code OK 06:54:08 DEBUG --- status code --- 06:54:08 DEBUG --- status code --- 06:54:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:54:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:54:08 DEBUG --- http response --- 06:54:08 DEBUG --- http response --- 06:54:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:54:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:54:08 DEBUG ----- output ----- 06:54:08 DEBUG ----- output ----- 06:54:08 DEBUG { 06:54:08 DEBUG { 06:54:08 DEBUG "status": "success", 06:54:08 DEBUG "status": "success", 06:54:08 DEBUG "data": { 06:54:08 DEBUG "data": { 06:54:08 DEBUG "resultType": "vector", 06:54:08 DEBUG "resultType": "vector", 06:54:08 DEBUG "result": [ 06:54:08 DEBUG "result": [ 06:54:08 DEBUG { 06:54:08 DEBUG { 06:54:08 DEBUG "metric": { 06:54:08 DEBUG "metric": { 06:54:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:54:08 DEBUG "pod": "ds-idrepo-0" 06:54:08 DEBUG }, 06:54:08 DEBUG }, 06:54:08 DEBUG "value": [ 06:54:08 DEBUG "value": [ 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG [...] 06:54:08 DEBUG [...] 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:54:08 DEBUG "pod": "ds-cts-2" 06:54:08 DEBUG }, 06:54:08 DEBUG }, 06:54:08 DEBUG "value": [ 06:54:08 DEBUG "value": [ 06:54:08 DEBUG 1713592448, 06:54:08 DEBUG 1713592448, 06:54:08 DEBUG "2675.366666666667" 06:54:08 DEBUG "0" 06:54:08 DEBUG ] 06:54:08 DEBUG ] 06:54:08 DEBUG } 06:54:08 DEBUG } 06:54:08 DEBUG ] 06:54:08 DEBUG ] 06:54:08 DEBUG } 06:54:08 DEBUG } 06:54:08 DEBUG } 06:54:08 DEBUG } 06:54:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592448" 06:54:08 INFO [http_cmd]: http status code OK 06:54:08 DEBUG --- status code --- 06:54:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:54:08 DEBUG --- http response --- 06:54:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:54:08 DEBUG ----- output ----- 06:54:08 DEBUG { 06:54:08 DEBUG "status": "success", 06:54:08 DEBUG "data": { 06:54:08 DEBUG "resultType": "vector", 06:54:08 DEBUG "result": [ 06:54:08 DEBUG { 06:54:08 DEBUG "metric": { 06:54:08 DEBUG "instance": "10.142.0.107:9100" 06:54:08 DEBUG }, 06:54:08 DEBUG "value": [ 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG [...] 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG "instance": "10.142.0.43:9100" 06:54:08 DEBUG }, 06:54:08 DEBUG "value": [ 06:54:08 DEBUG 1713592448, 06:54:08 DEBUG "80708899.13333333" 06:54:08 DEBUG ] 06:54:08 DEBUG } 06:54:08 DEBUG ] 06:54:08 DEBUG } 06:54:08 DEBUG } 06:54:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592448" 06:54:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592448" 06:54:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592448" 06:54:08 INFO [http_cmd]: http status code OK 06:54:08 DEBUG --- status code --- 06:54:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:54:08 DEBUG --- http response --- 06:54:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:54:08 DEBUG ----- output ----- 06:54:08 DEBUG { 06:54:08 DEBUG "status": "success", 06:54:08 DEBUG "data": { 06:54:08 DEBUG "resultType": "vector", 06:54:08 DEBUG "result": [ 06:54:08 DEBUG { 06:54:08 DEBUG "metric": { 06:54:08 DEBUG "pod": "ds-idrepo-0" 06:54:08 DEBUG }, 06:54:08 DEBUG "value": [ 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG [...] 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG "pod": "ds-idrepo-2" 06:54:08 DEBUG }, 06:54:08 DEBUG "value": [ 06:54:08 DEBUG 1713592448, 06:54:08 DEBUG "0" 06:54:08 DEBUG ] 06:54:08 DEBUG } 06:54:08 DEBUG ] 06:54:08 DEBUG } 06:54:08 DEBUG } 06:54:08 INFO [http_cmd]: http status code OK 06:54:08 DEBUG --- status code --- 06:54:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:54:08 DEBUG --- http response --- 06:54:08 INFO [http_cmd]: http status code OK 06:54:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:54:08 DEBUG --- status code --- 06:54:08 DEBUG ----- output ----- 06:54:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:54:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592448" 06:54:08 DEBUG { 06:54:08 DEBUG --- http response --- 06:54:08 DEBUG "status": "success", 06:54:08 DEBUG "data": { 06:54:08 DEBUG "resultType": "vector", 06:54:08 DEBUG "result": [ 06:54:08 DEBUG { 06:54:08 DEBUG "metric": { 06:54:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:54:08 DEBUG }, 06:54:08 DEBUG "value": [ 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG [...] 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:54:08 DEBUG }, 06:54:08 DEBUG "value": [ 06:54:08 DEBUG 1713592448, 06:54:08 DEBUG "2664.266666666667" 06:54:08 DEBUG ] 06:54:08 DEBUG } 06:54:08 DEBUG ] 06:54:08 DEBUG } 06:54:08 DEBUG } 06:54:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:54:08 DEBUG ----- output ----- 06:54:08 DEBUG { 06:54:08 DEBUG "status": "success", 06:54:08 DEBUG "data": { 06:54:08 DEBUG "resultType": "vector", 06:54:08 DEBUG "result": [ 06:54:08 DEBUG { 06:54:08 DEBUG "metric": { 06:54:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:54:08 DEBUG }, 06:54:08 DEBUG "value": [ 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG [...] 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:54:08 DEBUG }, 06:54:08 DEBUG "value": [ 06:54:08 DEBUG 1713592448, 06:54:08 DEBUG "2664.2" 06:54:08 DEBUG ] 06:54:08 DEBUG } 06:54:08 DEBUG ] 06:54:08 DEBUG } 06:54:08 DEBUG } 06:54:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713592448" 06:54:08 INFO [http_cmd]: http status code OK 06:54:08 DEBUG --- status code --- 06:54:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:54:08 DEBUG --- http response --- 06:54:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:54:08 DEBUG ----- output ----- 06:54:08 DEBUG { 06:54:08 DEBUG "status": "success", 06:54:08 DEBUG "data": { 06:54:08 DEBUG "resultType": "vector", 06:54:08 DEBUG "result": [ 06:54:08 DEBUG { 06:54:08 DEBUG "metric": { 06:54:08 DEBUG "pod": "ds-cts-0" 06:54:08 DEBUG }, 06:54:08 DEBUG "value": [ 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG [...] 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG 06:54:08 DEBUG "pod": "ds-cts-2" 06:54:08 DEBUG }, 06:54:08 DEBUG "value": [ 06:54:08 DEBUG 1713592448, 06:54:08 DEBUG "0.0006" 06:54:08 DEBUG ] 06:54:08 DEBUG } 06:54:08 DEBUG ] 06:54:08 DEBUG } 06:54:08 DEBUG } 06:54:09 INFO [http_cmd]: http status code OK 06:54:09 DEBUG --- status code --- 06:54:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:54:09 DEBUG --- http response --- 06:54:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:54:09 DEBUG ----- output ----- 06:54:09 DEBUG { 06:54:09 DEBUG "status": "success", 06:54:09 DEBUG "data": { 06:54:09 DEBUG "resultType": "vector", 06:54:09 DEBUG "result": [ 06:54:09 DEBUG { 06:54:09 DEBUG "metric": { 06:54:09 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 06:54:09 DEBUG }, 06:54:09 DEBUG "value": [ 06:54:09 DEBUG 06:54:09 DEBUG 06:54:09 DEBUG 06:54:09 DEBUG [...] 06:54:09 DEBUG 06:54:09 DEBUG 06:54:09 DEBUG 06:54:09 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 06:54:09 DEBUG }, 06:54:09 DEBUG "value": [ 06:54:09 DEBUG 1713592448, 06:54:09 DEBUG "8.040085082071405" 06:54:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592449" 06:54:09 DEBUG ] 06:54:09 DEBUG } 06:54:09 DEBUG ] 06:54:09 DEBUG } 06:54:09 DEBUG } 06:54:09 INFO [http_cmd]: http status code OK 06:54:09 DEBUG --- status code --- 06:54:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:54:09 DEBUG --- http response --- 06:54:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:54:09 DEBUG ----- output ----- 06:54:09 DEBUG { 06:54:09 DEBUG "status": "success", 06:54:09 DEBUG "data": { 06:54:09 DEBUG "resultType": "vector", 06:54:09 DEBUG "result": [ 06:54:09 DEBUG { 06:54:09 DEBUG "metric": { 06:54:09 DEBUG "pod": "am-767f6b67b5-9zw22" 06:54:09 DEBUG }, 06:54:09 DEBUG "value": [ 06:54:09 DEBUG 06:54:09 DEBUG 06:54:09 DEBUG 06:54:09 DEBUG [...] 06:54:09 DEBUG 06:54:09 DEBUG 06:54:09 DEBUG 06:54:09 DEBUG "pod": "am-767f6b67b5-vbltz" 06:54:09 DEBUG }, 06:54:09 DEBUG "value": [ 06:54:09 DEBUG 1713592449, 06:54:09 DEBUG "2663.4333333333334" 06:54:09 DEBUG ] 06:54:09 DEBUG } 06:54:09 DEBUG ] 06:54:09 DEBUG } 06:54:09 DEBUG } 06:54:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713592449" 06:54:09 INFO [http_cmd]: http status code OK 06:54:09 DEBUG --- status code --- 06:54:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:54:09 DEBUG --- http response --- 06:54:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:54:09 DEBUG ----- output ----- 06:54:09 DEBUG { 06:54:09 DEBUG "status": "success", 06:54:09 DEBUG "data": { 06:54:09 DEBUG "resultType": "vector", 06:54:09 DEBUG "result": [ 06:54:09 DEBUG { 06:54:09 DEBUG "metric": { 06:54:09 DEBUG "pod": "am-767f6b67b5-9zw22" 06:54:09 DEBUG }, 06:54:09 DEBUG "value": [ 06:54:09 DEBUG 06:54:09 DEBUG 06:54:09 DEBUG 06:54:09 DEBUG [...] 06:54:09 DEBUG 06:54:09 DEBUG 06:54:09 DEBUG 06:54:09 DEBUG "pod": "am-767f6b67b5-vbltz" 06:54:09 DEBUG }, 06:54:09 DEBUG "value": [ 06:54:09 DEBUG 1713592449, 06:54:09 DEBUG "0.096" 06:54:09 DEBUG ] 06:54:09 DEBUG } 06:54:09 DEBUG ] 06:54:09 DEBUG } 06:54:09 DEBUG } 06:54:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592449" 06:54:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592449" 06:54:09 INFO [http_cmd]: http status code OK 06:54:09 DEBUG --- status code --- 06:54:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:54:09 DEBUG --- http response --- 06:54:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:54:09 DEBUG ----- output ----- 06:54:09 DEBUG { 06:54:09 DEBUG "status": "success", 06:54:09 DEBUG "data": { 06:54:09 DEBUG "resultType": "vector", 06:54:09 DEBUG "result": [ 06:54:09 DEBUG { 06:54:09 DEBUG "metric": { 06:54:09 DEBUG "instance": "10.142.0.107:9100" 06:54:09 DEBUG }, 06:54:09 DEBUG "value": [ 06:54:09 DEBUG 06:54:09 DEBUG 06:54:09 DEBUG 06:54:09 DEBUG [...] 06:54:09 DEBUG 06:54:09 DEBUG 06:54:09 DEBUG 06:54:09 DEBUG "instance": "10.142.0.43:9100" 06:54:09 DEBUG }, 06:54:09 DEBUG "value": [ 06:54:09 DEBUG 1713592449, 06:54:09 DEBUG "80623038.83333334" 06:54:09 DEBUG ] 06:54:09 DEBUG } 06:54:09 DEBUG ] 06:54:09 DEBUG } 06:54:09 DEBUG } 06:54:09 INFO [http_cmd]: http status code OK 06:54:09 DEBUG --- status code --- 06:54:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:54:09 DEBUG --- http response --- 06:54:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:54:09 DEBUG ----- output ----- 06:54:09 DEBUG { 06:54:09 DEBUG "status": "success", 06:54:09 DEBUG "data": { 06:54:09 DEBUG "resultType": "vector", 06:54:09 DEBUG "result": [ 06:54:09 DEBUG { 06:54:09 DEBUG "metric": { 06:54:09 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:54:09 DEBUG }, 06:54:09 DEBUG "value": [ 06:54:09 DEBUG 06:54:09 DEBUG 06:54:09 DEBUG 06:54:09 DEBUG [...] 06:54:09 DEBUG 06:54:09 DEBUG 06:54:09 DEBUG 06:54:09 DEBUG "pod": "ds-cts-2" 06:54:09 DEBUG }, 06:54:09 DEBUG "value": [ 06:54:09 DEBUG 1713592449, 06:54:09 DEBUG "0" 06:54:09 DEBUG ] 06:54:09 DEBUG } 06:54:09 DEBUG ] 06:54:09 DEBUG } 06:54:09 DEBUG } 06:54:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592449" 06:54:09 INFO [http_cmd]: http status code OK 06:54:09 DEBUG --- status code --- 06:54:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:54:09 DEBUG --- http response --- 06:54:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:54:09 DEBUG ----- output ----- 06:54:09 DEBUG { 06:54:09 DEBUG "status": "success", 06:54:09 DEBUG "data": { 06:54:09 DEBUG "resultType": "vector", 06:54:09 DEBUG "result": [ 06:54:09 DEBUG { 06:54:09 DEBUG "metric": { 06:54:09 DEBUG "instance": "10.142.0.107:9100" 06:54:09 DEBUG }, 06:54:09 DEBUG "value": [ 06:54:09 DEBUG 06:54:09 DEBUG 06:54:09 DEBUG 06:54:09 DEBUG [...] 06:54:09 DEBUG 06:54:09 DEBUG 06:54:09 DEBUG 06:54:09 DEBUG "instance": "10.142.0.43:9100" 06:54:09 DEBUG }, 06:54:09 DEBUG "value": [ 06:54:09 DEBUG 1713592449, 06:54:09 DEBUG "0.07499999999999055" 06:54:09 DEBUG ] 06:54:09 DEBUG } 06:54:09 DEBUG ] 06:54:09 DEBUG } 06:54:09 DEBUG } 06:54:10 INFO 06:54:10 INFO [loop_until]: kubectl --namespace=xlou top node 06:54:10 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:54:10 INFO 06:54:10 INFO [loop_until]: kubectl --namespace=xlou top pods 06:54:10 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:54:10 INFO [loop_until]: OK (rc = 0) 06:54:10 DEBUG --- stdout --- 06:54:10 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 71m 0% 6266Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7988m 26% 7453Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 17200m 57% 11345Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 79m 0% 3360Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 18609m 62% 13530Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 146m 0% 5219Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 17370m 58% 11600Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 19130m 64% 3017Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 19368m 64% 10861Mi 9% gke-xlou-cdm-ds-0f601751-j488 8236m 27% 7031Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 17299m 57% 15599Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 8038m 26% 7707Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 19448m 65% 12876Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 8160m 51% 8494Mi 14% 06:54:10 DEBUG --- stderr --- 06:54:10 DEBUG 06:54:10 INFO [loop_until]: OK (rc = 0) 06:54:10 DEBUG --- stdout --- 06:54:10 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 17341m 8649Mi am-767f6b67b5-qcqz2 17843m 8717Mi am-767f6b67b5-vbltz 18569m 8656Mi ds-cts-0 18765m 10399Mi ds-cts-1 17586m 10667Mi ds-cts-2 19546m 10270Mi ds-idrepo-0 7659m 3782Mi ds-idrepo-1 7661m 3802Mi ds-idrepo-2 7824m 3706Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1177Mi idm-b85f9d978-wnhxb 4m 1236Mi lodemon-7cb5b9456d-dskfn 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7976m 4672Mi 06:54:10 DEBUG --- stderr --- 06:54:10 DEBUG 06:55:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592508" 06:55:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592508" 06:55:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592508" 06:55:08 INFO [http_cmd]: http status code OK 06:55:08 DEBUG --- status code --- 06:55:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:55:08 DEBUG --- http response --- 06:55:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 06:55:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592508" 06:55:08 DEBUG ----- output ----- 06:55:08 DEBUG { 06:55:08 DEBUG "status": "success", 06:55:08 DEBUG "data": { 06:55:08 INFO [http_cmd]: http status code OK 06:55:08 DEBUG "resultType": "vector", 06:55:08 DEBUG --- status code --- 06:55:08 DEBUG "result": [ 06:55:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:55:08 DEBUG { 06:55:08 DEBUG --- http response --- 06:55:08 DEBUG "metric": { 06:55:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:55:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:55:08 DEBUG }, 06:55:08 DEBUG ----- output ----- 06:55:08 DEBUG "value": [ 06:55:08 DEBUG { 06:55:08 DEBUG 06:55:08 DEBUG "status": "success", 06:55:08 DEBUG 06:55:08 DEBUG "data": { 06:55:08 DEBUG 06:55:08 DEBUG "resultType": "vector", 06:55:08 DEBUG [...] 06:55:08 DEBUG "result": [ 06:55:08 DEBUG 06:55:08 DEBUG { 06:55:08 DEBUG 06:55:08 DEBUG "metric": { 06:55:08 DEBUG 06:55:08 DEBUG "pod": "ds-cts-0" 06:55:08 DEBUG "pod": "ds-cts-2" 06:55:08 DEBUG }, 06:55:08 DEBUG }, 06:55:08 DEBUG "value": [ 06:55:08 DEBUG "value": [ 06:55:08 DEBUG 06:55:08 DEBUG 1713592508, 06:55:08 DEBUG 06:55:08 DEBUG "48754505.257175334" 06:55:08 DEBUG 06:55:08 DEBUG ] 06:55:08 DEBUG [...] 06:55:08 DEBUG } 06:55:08 DEBUG 06:55:08 DEBUG ] 06:55:08 DEBUG 06:55:08 DEBUG } 06:55:08 DEBUG 06:55:08 DEBUG } 06:55:08 DEBUG "pod": "ds-cts-2" 06:55:08 DEBUG }, 06:55:08 DEBUG "value": [ 06:55:08 DEBUG 1713592508, 06:55:08 DEBUG "29.933333333333334" 06:55:08 DEBUG ] 06:55:08 DEBUG } 06:55:08 DEBUG ] 06:55:08 DEBUG } 06:55:08 DEBUG } 06:55:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592508" 06:55:08 INFO [http_cmd]: http status code OK 06:55:08 DEBUG --- status code --- 06:55:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:55:08 DEBUG --- http response --- 06:55:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:55:08 DEBUG ----- output ----- 06:55:08 DEBUG { 06:55:08 DEBUG "status": "success", 06:55:08 DEBUG "data": { 06:55:08 DEBUG "resultType": "vector", 06:55:08 DEBUG "result": [ 06:55:08 DEBUG { 06:55:08 DEBUG "metric": { 06:55:08 DEBUG "instance": "10.142.0.107:9100" 06:55:08 DEBUG }, 06:55:08 DEBUG "value": [ 06:55:08 DEBUG 06:55:08 DEBUG 06:55:08 DEBUG 06:55:08 DEBUG [...] 06:55:08 DEBUG 06:55:08 DEBUG 06:55:08 DEBUG 06:55:08 DEBUG "instance": "10.142.0.43:9100" 06:55:08 DEBUG }, 06:55:08 DEBUG "value": [ 06:55:08 DEBUG 1713592508, 06:55:08 DEBUG "3243185642496" 06:55:08 DEBUG ] 06:55:08 DEBUG } 06:55:08 DEBUG ] 06:55:08 DEBUG } 06:55:08 DEBUG } 06:55:08 INFO [http_cmd]: http status code OK 06:55:08 INFO [http_cmd]: http status code OK 06:55:08 DEBUG --- status code --- 06:55:08 DEBUG --- status code --- 06:55:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:55:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:55:08 DEBUG --- http response --- 06:55:08 DEBUG --- http response --- 06:55:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:55:08 DEBUG ----- output ----- 06:55:08 DEBUG { 06:55:08 DEBUG "status": "success", 06:55:08 DEBUG "data": { 06:55:08 DEBUG "resultType": "vector", 06:55:08 DEBUG "result": [ 06:55:08 DEBUG { 06:55:08 DEBUG "metric": { 06:55:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:55:08 DEBUG }, 06:55:08 DEBUG "value": [ 06:55:08 DEBUG 06:55:08 DEBUG 06:55:08 DEBUG 06:55:08 DEBUG [...] 06:55:08 DEBUG 06:55:08 DEBUG 06:55:08 DEBUG 06:55:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:55:08 DEBUG }, 06:55:08 DEBUG "value": [ 06:55:08 DEBUG 1713592508, 06:55:08 DEBUG "479838" 06:55:08 DEBUG ] 06:55:08 DEBUG } 06:55:08 DEBUG ] 06:55:08 DEBUG } 06:55:08 DEBUG } 06:55:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:55:08 DEBUG ----- output ----- 06:55:08 DEBUG { 06:55:08 DEBUG "status": "success", 06:55:08 DEBUG "data": { 06:55:08 DEBUG "resultType": "vector", 06:55:08 DEBUG "result": [ 06:55:08 DEBUG { 06:55:08 DEBUG "metric": { 06:55:08 DEBUG "pod": "ds-idrepo-0" 06:55:08 DEBUG }, 06:55:08 DEBUG "value": [ 06:55:08 DEBUG 06:55:08 DEBUG 06:55:08 DEBUG 06:55:08 DEBUG [...] 06:55:08 DEBUG 06:55:08 DEBUG 06:55:08 DEBUG 06:55:08 DEBUG "pod": "ds-idrepo-2" 06:55:08 DEBUG }, 06:55:08 DEBUG "value": [ 06:55:08 DEBUG 1713592508, 06:55:08 DEBUG "0" 06:55:08 DEBUG ] 06:55:08 DEBUG } 06:55:08 DEBUG ] 06:55:08 DEBUG } 06:55:08 DEBUG } 06:55:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713592508" 06:55:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592508" 06:55:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713592508" 06:55:08 ERROR [http_cmd]: ERROR 06:55:08 INFO [http_cmd]: http status code OK 06:55:08 ERROR --- status code --- 06:55:08 DEBUG --- status code --- 06:55:08 ERROR http status code is 400 (expected [200, 401, 403, 404]) 06:55:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:55:08 ERROR --- http response --- 06:55:08 DEBUG --- http response --- 06:55:08 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 06:55:08 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 06:55:08 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 06:55:08 INFO [http_cmd]: http status code OK 06:55:08 DEBUG --- status code --- 06:55:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:55:08 DEBUG --- http response --- 06:55:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:55:08 DEBUG ----- output ----- 06:55:08 DEBUG { 06:55:08 DEBUG "status": "success", 06:55:08 DEBUG "data": { 06:55:08 DEBUG "resultType": "vector", 06:55:08 DEBUG "result": [ 06:55:08 DEBUG { 06:55:08 DEBUG "metric": { 06:55:08 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 06:55:08 DEBUG }, 06:55:08 DEBUG "value": [ 06:55:08 DEBUG 06:55:08 DEBUG 06:55:08 DEBUG 06:55:08 DEBUG [...] 06:55:08 DEBUG 06:55:08 DEBUG 06:55:08 DEBUG 06:55:08 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 06:55:08 DEBUG }, 06:55:08 DEBUG "value": [ 06:55:08 DEBUG 1713592508, 06:55:08 DEBUG "4996608000" 06:55:08 DEBUG ] 06:55:08 DEBUG } 06:55:08 DEBUG ] 06:55:08 DEBUG } 06:55:08 DEBUG } 06:55:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592508" 06:55:08 INFO [http_cmd]: http status code OK 06:55:08 DEBUG --- status code --- 06:55:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:55:08 DEBUG --- http response --- 06:55:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 06:55:08 DEBUG ----- output ----- 06:55:08 DEBUG { 06:55:08 DEBUG "status": "success", 06:55:08 DEBUG "data": { 06:55:08 DEBUG "resultType": "vector", 06:55:08 DEBUG "result": [ 06:55:08 DEBUG { 06:55:08 DEBUG "metric": { 06:55:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:55:08 DEBUG }, 06:55:08 DEBUG "value": [ 06:55:08 DEBUG 06:55:08 DEBUG 06:55:08 DEBUG 06:55:08 DEBUG [...] 06:55:08 DEBUG 06:55:08 DEBUG 06:55:08 DEBUG 06:55:08 DEBUG "pod": "ds-cts-2" 06:55:08 DEBUG }, 06:55:08 DEBUG "value": [ 06:55:08 DEBUG 1713592508, 06:55:08 DEBUG "75396983.23387326" 06:55:08 DEBUG ] 06:55:08 DEBUG } 06:55:08 DEBUG ] 06:55:08 DEBUG } 06:55:08 DEBUG } 06:55:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592508" 06:55:08 INFO [http_cmd]: http status code OK 06:55:08 DEBUG --- status code --- 06:55:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:55:08 DEBUG --- http response --- 06:55:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:55:08 DEBUG ----- output ----- 06:55:08 DEBUG { 06:55:08 DEBUG "status": "success", 06:55:08 DEBUG "data": { 06:55:08 DEBUG "resultType": "vector", 06:55:08 DEBUG "result": [ 06:55:08 DEBUG { 06:55:08 DEBUG "metric": { 06:55:08 DEBUG "pod": "ds-idrepo-0" 06:55:08 DEBUG }, 06:55:08 DEBUG "value": [ 06:55:08 DEBUG 06:55:08 DEBUG 06:55:08 DEBUG 06:55:08 DEBUG [...] 06:55:08 DEBUG 06:55:08 DEBUG 06:55:08 DEBUG 06:55:08 DEBUG "pod": "ds-idrepo-2" 06:55:08 DEBUG }, 06:55:08 DEBUG "value": [ 06:55:08 DEBUG 1713592508, 06:55:08 DEBUG "0" 06:55:08 DEBUG ] 06:55:08 DEBUG } 06:55:08 DEBUG ] 06:55:08 DEBUG } 06:55:08 DEBUG } 06:55:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592508" 06:55:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713592508" 06:55:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592508" 06:55:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592508" 06:55:09 INFO [http_cmd]: http status code OK 06:55:09 INFO [http_cmd]: http status code OK 06:55:09 INFO [http_cmd]: http status code OK 06:55:09 DEBUG --- status code --- 06:55:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592509" 06:55:09 DEBUG --- status code --- 06:55:09 DEBUG --- status code --- 06:55:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:55:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:55:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:55:09 DEBUG --- http response --- 06:55:09 DEBUG --- http response --- 06:55:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592509" 06:55:09 DEBUG --- http response --- 06:55:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:55:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:55:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:55:09 DEBUG ----- output ----- 06:55:09 DEBUG ----- output ----- 06:55:09 DEBUG ----- output ----- 06:55:09 DEBUG { 06:55:09 DEBUG { 06:55:09 DEBUG { 06:55:09 DEBUG "status": "success", 06:55:09 DEBUG "status": "success", 06:55:09 DEBUG "status": "success", 06:55:09 DEBUG "data": { 06:55:09 DEBUG "data": { 06:55:09 DEBUG "data": { 06:55:09 DEBUG "resultType": "vector", 06:55:09 DEBUG "resultType": "vector", 06:55:09 DEBUG "resultType": "vector", 06:55:09 DEBUG "result": [ 06:55:09 DEBUG "result": [ 06:55:09 DEBUG "result": [ 06:55:09 DEBUG { 06:55:09 DEBUG { 06:55:09 DEBUG { 06:55:09 DEBUG "metric": { 06:55:09 DEBUG "metric": { 06:55:09 DEBUG "metric": { 06:55:09 DEBUG "pod": "ds-idrepo-0" 06:55:09 DEBUG "pod": "am-767f6b67b5-9zw22" 06:55:09 DEBUG "pod": "ds-idrepo-0" 06:55:09 INFO [http_cmd]: http status code OK 06:55:09 DEBUG }, 06:55:09 DEBUG }, 06:55:09 DEBUG }, 06:55:09 DEBUG --- status code --- 06:55:09 DEBUG "value": [ 06:55:09 DEBUG "value": [ 06:55:09 DEBUG "value": [ 06:55:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:55:09 DEBUG 06:55:09 DEBUG 06:55:09 DEBUG 06:55:09 DEBUG --- http response --- 06:55:09 DEBUG 06:55:09 DEBUG 06:55:09 DEBUG 06:55:09 DEBUG 06:55:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 06:55:09 DEBUG 06:55:09 DEBUG 06:55:09 DEBUG [...] 06:55:09 DEBUG ----- output ----- 06:55:09 DEBUG [...] 06:55:09 DEBUG [...] 06:55:09 DEBUG 06:55:09 DEBUG { 06:55:09 DEBUG 06:55:09 DEBUG 06:55:09 DEBUG 06:55:09 DEBUG "status": "success", 06:55:09 DEBUG 06:55:09 DEBUG 06:55:09 DEBUG 06:55:09 DEBUG "data": { 06:55:09 DEBUG 06:55:09 DEBUG 06:55:09 DEBUG "pod": "ds-cts-2" 06:55:09 DEBUG "resultType": "vector", 06:55:09 DEBUG "pod": "am-767f6b67b5-vbltz" 06:55:09 DEBUG "pod": "ds-idrepo-2" 06:55:09 DEBUG }, 06:55:09 DEBUG "result": [ 06:55:09 DEBUG }, 06:55:09 DEBUG }, 06:55:09 DEBUG "value": [ 06:55:09 DEBUG { 06:55:09 DEBUG "value": [ 06:55:09 DEBUG "value": [ 06:55:09 DEBUG 1713592508, 06:55:09 DEBUG "metric": { 06:55:09 DEBUG 1713592508, 06:55:09 DEBUG 1713592508, 06:55:09 DEBUG "0" 06:55:09 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:55:09 DEBUG "0" 06:55:09 DEBUG "0" 06:55:09 DEBUG ] 06:55:09 DEBUG }, 06:55:09 DEBUG ] 06:55:09 DEBUG ] 06:55:09 DEBUG } 06:55:09 DEBUG "value": [ 06:55:09 DEBUG } 06:55:09 DEBUG } 06:55:09 DEBUG ] 06:55:09 DEBUG 06:55:09 DEBUG ] 06:55:09 DEBUG ] 06:55:09 DEBUG } 06:55:09 DEBUG 06:55:09 DEBUG } 06:55:09 DEBUG } 06:55:09 DEBUG } 06:55:09 DEBUG 06:55:09 DEBUG } 06:55:09 DEBUG } 06:55:09 DEBUG [...] 06:55:09 DEBUG 06:55:09 DEBUG 06:55:09 DEBUG 06:55:09 DEBUG "pod": "ds-cts-2" 06:55:09 DEBUG }, 06:55:09 DEBUG "value": [ 06:55:09 DEBUG 1713592508, 06:55:09 DEBUG "382620262.5745951" 06:55:09 DEBUG ] 06:55:09 DEBUG } 06:55:09 DEBUG ] 06:55:09 DEBUG } 06:55:09 DEBUG } 06:55:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592509" 06:55:09 INFO [http_cmd]: http status code OK 06:55:09 INFO [http_cmd]: http status code OK 06:55:09 DEBUG --- status code --- 06:55:09 DEBUG --- status code --- 06:55:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:55:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:55:09 DEBUG --- http response --- 06:55:09 DEBUG --- http response --- 06:55:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:55:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:55:09 DEBUG ----- output ----- 06:55:09 DEBUG ----- output ----- 06:55:09 DEBUG { 06:55:09 DEBUG { 06:55:09 DEBUG "status": "success", 06:55:09 DEBUG "status": "success", 06:55:09 DEBUG "data": { 06:55:09 DEBUG "data": { 06:55:09 DEBUG "resultType": "vector", 06:55:09 DEBUG "resultType": "vector", 06:55:09 DEBUG "result": [ 06:55:09 DEBUG "result": [ 06:55:09 DEBUG { 06:55:09 DEBUG { 06:55:09 DEBUG "metric": { 06:55:09 DEBUG "metric": { 06:55:09 DEBUG "pod": "ds-cts-0" 06:55:09 DEBUG "instance": "10.142.0.107:9100" 06:55:09 DEBUG }, 06:55:09 DEBUG }, 06:55:09 DEBUG "value": [ 06:55:09 DEBUG "value": [ 06:55:09 DEBUG 06:55:09 DEBUG 06:55:09 DEBUG 06:55:09 DEBUG 06:55:09 DEBUG 06:55:09 DEBUG 06:55:09 DEBUG [...] 06:55:09 DEBUG [...] 06:55:09 DEBUG 06:55:09 DEBUG 06:55:09 DEBUG 06:55:09 DEBUG 06:55:09 DEBUG 06:55:09 DEBUG 06:55:09 DEBUG "pod": "ds-cts-2" 06:55:09 DEBUG "instance": "10.142.0.43:9100" 06:55:09 DEBUG }, 06:55:09 DEBUG }, 06:55:09 DEBUG "value": [ 06:55:09 DEBUG "value": [ 06:55:09 DEBUG 1713592509, 06:55:09 DEBUG 1713592509, 06:55:09 DEBUG "0" 06:55:09 DEBUG "0.27953333333336733" 06:55:09 DEBUG ] 06:55:09 DEBUG ] 06:55:09 DEBUG } 06:55:09 DEBUG } 06:55:09 DEBUG ] 06:55:09 DEBUG ] 06:55:09 DEBUG } 06:55:09 DEBUG } 06:55:09 DEBUG } 06:55:09 DEBUG } 06:55:09 INFO [http_cmd]: http status code OK 06:55:09 DEBUG --- status code --- 06:55:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:55:09 DEBUG --- http response --- 06:55:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:55:09 DEBUG ----- output ----- 06:55:09 DEBUG { 06:55:09 DEBUG "status": "success", 06:55:09 DEBUG "data": { 06:55:09 DEBUG "resultType": "vector", 06:55:09 DEBUG "result": [ 06:55:09 DEBUG { 06:55:09 DEBUG "metric": { 06:55:09 DEBUG "pod": "am-767f6b67b5-9zw22" 06:55:09 DEBUG }, 06:55:09 DEBUG "value": [ 06:55:09 DEBUG 06:55:09 DEBUG 06:55:09 DEBUG 06:55:09 DEBUG [...] 06:55:09 DEBUG 06:55:09 DEBUG 06:55:09 DEBUG 06:55:09 DEBUG "pod": "am-767f6b67b5-vbltz" 06:55:09 DEBUG }, 06:55:09 DEBUG "value": [ 06:55:09 DEBUG 1713592509, 06:55:09 DEBUG "2657.9333333333334" 06:55:09 DEBUG ] 06:55:09 DEBUG } 06:55:09 DEBUG ] 06:55:09 DEBUG } 06:55:09 DEBUG } 06:55:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592509" 06:55:09 INFO [http_cmd]: http status code OK 06:55:09 DEBUG --- status code --- 06:55:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:55:09 DEBUG --- http response --- 06:55:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:55:09 DEBUG ----- output ----- 06:55:09 DEBUG { 06:55:09 DEBUG "status": "success", 06:55:09 DEBUG "data": { 06:55:09 DEBUG "resultType": "vector", 06:55:09 DEBUG "result": [ 06:55:09 DEBUG { 06:55:09 DEBUG "metric": { 06:55:09 DEBUG "instance": "10.142.0.107:9100" 06:55:09 DEBUG }, 06:55:09 DEBUG "value": [ 06:55:09 DEBUG 06:55:09 DEBUG 06:55:09 DEBUG 06:55:09 DEBUG [...] 06:55:09 DEBUG 06:55:09 DEBUG 06:55:09 DEBUG 06:55:09 DEBUG "instance": "10.142.0.43:9100" 06:55:09 DEBUG }, 06:55:09 DEBUG "value": [ 06:55:09 DEBUG 1713592509, 06:55:09 DEBUG "80710995.1" 06:55:09 DEBUG ] 06:55:09 DEBUG } 06:55:09 DEBUG ] 06:55:09 DEBUG } 06:55:09 DEBUG } 06:55:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592509" 06:55:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592509" 06:55:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592509" 06:55:09 INFO [http_cmd]: http status code OK 06:55:09 INFO [http_cmd]: http status code OK 06:55:09 DEBUG --- status code --- 06:55:09 DEBUG --- status code --- 06:55:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:55:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:55:09 DEBUG --- http response --- 06:55:09 DEBUG --- http response --- 06:55:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:55:09 DEBUG ----- output ----- 06:55:09 DEBUG { 06:55:09 DEBUG "status": "success", 06:55:09 DEBUG "data": { 06:55:09 DEBUG "resultType": "vector", 06:55:09 DEBUG "result": [ 06:55:09 DEBUG { 06:55:09 DEBUG "metric": { 06:55:09 DEBUG "pod": "am-767f6b67b5-9zw22" 06:55:09 DEBUG }, 06:55:09 DEBUG "value": [ 06:55:09 DEBUG 06:55:09 DEBUG 06:55:09 DEBUG 06:55:09 DEBUG [...] 06:55:09 DEBUG 06:55:09 DEBUG 06:55:09 DEBUG 06:55:09 DEBUG "pod": "am-767f6b67b5-vbltz" 06:55:09 DEBUG }, 06:55:09 DEBUG "value": [ 06:55:09 DEBUG 1713592509, 06:55:09 DEBUG "2655.2666666666664" 06:55:09 DEBUG ] 06:55:09 DEBUG } 06:55:09 DEBUG ] 06:55:09 DEBUG } 06:55:09 DEBUG } 06:55:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:55:09 DEBUG ----- output ----- 06:55:09 DEBUG { 06:55:09 DEBUG "status": "success", 06:55:09 DEBUG "data": { 06:55:09 DEBUG "resultType": "vector", 06:55:09 DEBUG "result": [ 06:55:09 DEBUG { 06:55:09 DEBUG "metric": { 06:55:09 DEBUG "pod": "am-767f6b67b5-9zw22" 06:55:09 DEBUG }, 06:55:09 DEBUG "value": [ 06:55:09 DEBUG 06:55:09 DEBUG 06:55:09 DEBUG 06:55:09 DEBUG [...] 06:55:09 DEBUG 06:55:09 DEBUG 06:55:09 DEBUG 06:55:09 DEBUG "pod": "am-767f6b67b5-vbltz" 06:55:09 DEBUG }, 06:55:09 DEBUG "value": [ 06:55:09 DEBUG 1713592509, 06:55:09 DEBUG "2655.266666666667" 06:55:09 DEBUG ] 06:55:09 DEBUG } 06:55:09 DEBUG ] 06:55:09 DEBUG } 06:55:09 DEBUG } 06:55:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713592509" 06:55:09 INFO [http_cmd]: http status code OK 06:55:09 DEBUG --- status code --- 06:55:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:55:09 DEBUG --- http response --- 06:55:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:55:09 DEBUG ----- output ----- 06:55:09 DEBUG { 06:55:09 DEBUG "status": "success", 06:55:09 DEBUG "data": { 06:55:09 DEBUG "resultType": "vector", 06:55:09 DEBUG "result": [ 06:55:09 DEBUG { 06:55:09 DEBUG "metric": { 06:55:09 DEBUG "pod": "ds-cts-0" 06:55:09 DEBUG }, 06:55:09 DEBUG "value": [ 06:55:09 DEBUG 06:55:09 DEBUG 06:55:09 DEBUG 06:55:09 DEBUG [...] 06:55:09 DEBUG 06:55:09 DEBUG 06:55:09 DEBUG 06:55:09 DEBUG "pod": "ds-cts-2" 06:55:09 DEBUG }, 06:55:09 DEBUG "value": [ 06:55:09 DEBUG 1713592509, 06:55:09 DEBUG "0.00023333333333333339" 06:55:09 DEBUG ] 06:55:09 DEBUG } 06:55:09 DEBUG ] 06:55:09 DEBUG } 06:55:09 DEBUG } 06:55:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713592509" 06:55:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592509" 06:55:09 INFO [http_cmd]: http status code OK 06:55:09 DEBUG --- status code --- 06:55:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:55:09 DEBUG --- http response --- 06:55:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:55:09 DEBUG ----- output ----- 06:55:09 DEBUG { 06:55:09 DEBUG "status": "success", 06:55:09 DEBUG "data": { 06:55:09 DEBUG "resultType": "vector", 06:55:09 DEBUG "result": [ 06:55:09 DEBUG { 06:55:09 DEBUG "metric": { 06:55:09 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 06:55:09 DEBUG }, 06:55:09 DEBUG "value": [ 06:55:09 DEBUG 06:55:09 DEBUG 06:55:09 DEBUG 06:55:09 DEBUG [...] 06:55:09 DEBUG 06:55:09 DEBUG 06:55:09 DEBUG 06:55:09 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 06:55:09 DEBUG }, 06:55:09 DEBUG "value": [ 06:55:09 DEBUG 1713592509, 06:55:09 DEBUG "8.035444564040876" 06:55:09 DEBUG ] 06:55:09 DEBUG } 06:55:09 DEBUG ] 06:55:09 DEBUG } 06:55:09 DEBUG } 06:55:09 INFO [http_cmd]: http status code OK 06:55:09 INFO [http_cmd]: http status code OK 06:55:09 DEBUG --- status code --- 06:55:09 DEBUG --- status code --- 06:55:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:55:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:55:09 DEBUG --- http response --- 06:55:09 DEBUG --- http response --- 06:55:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:55:09 DEBUG ----- output ----- 06:55:09 DEBUG { 06:55:09 DEBUG "status": "success", 06:55:09 DEBUG "data": { 06:55:09 DEBUG "resultType": "vector", 06:55:09 DEBUG "result": [ 06:55:09 DEBUG { 06:55:09 DEBUG "metric": { 06:55:09 DEBUG "pod": "am-767f6b67b5-9zw22" 06:55:09 DEBUG }, 06:55:09 DEBUG "value": [ 06:55:09 DEBUG 06:55:09 DEBUG 06:55:09 DEBUG 06:55:09 DEBUG [...] 06:55:09 DEBUG 06:55:09 DEBUG 06:55:09 DEBUG 06:55:09 DEBUG "pod": "am-767f6b67b5-vbltz" 06:55:09 DEBUG }, 06:55:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:55:09 DEBUG "value": [ 06:55:09 DEBUG ----- output ----- 06:55:09 DEBUG 1713592509, 06:55:09 DEBUG { 06:55:09 DEBUG "0.096" 06:55:09 DEBUG "status": "success", 06:55:09 DEBUG ] 06:55:09 DEBUG "data": { 06:55:09 DEBUG } 06:55:09 DEBUG "resultType": "vector", 06:55:09 DEBUG ] 06:55:09 DEBUG "result": [ 06:55:09 DEBUG } 06:55:09 DEBUG { 06:55:09 DEBUG } 06:55:09 DEBUG "metric": { 06:55:09 DEBUG "pod": "am-767f6b67b5-9zw22" 06:55:09 DEBUG }, 06:55:09 DEBUG "value": [ 06:55:09 DEBUG 06:55:09 DEBUG 06:55:09 DEBUG 06:55:09 DEBUG [...] 06:55:09 DEBUG 06:55:09 DEBUG 06:55:09 DEBUG 06:55:09 DEBUG "pod": "am-767f6b67b5-vbltz" 06:55:09 DEBUG }, 06:55:09 DEBUG "value": [ 06:55:09 DEBUG 1713592509, 06:55:09 DEBUG "2655.133333333333" 06:55:09 DEBUG ] 06:55:09 DEBUG } 06:55:09 DEBUG ] 06:55:09 DEBUG } 06:55:09 DEBUG } 06:55:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592509" 06:55:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592509" 06:55:09 INFO [http_cmd]: http status code OK 06:55:09 INFO [http_cmd]: http status code OK 06:55:09 DEBUG --- status code --- 06:55:09 DEBUG --- status code --- 06:55:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:55:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:55:09 DEBUG --- http response --- 06:55:09 DEBUG --- http response --- 06:55:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 06:55:09 DEBUG ----- output ----- 06:55:09 DEBUG { 06:55:09 DEBUG "status": "success", 06:55:09 DEBUG "data": { 06:55:09 DEBUG "resultType": "vector", 06:55:09 DEBUG "result": [ 06:55:09 DEBUG { 06:55:09 DEBUG "metric": { 06:55:09 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:55:09 DEBUG }, 06:55:09 DEBUG "value": [ 06:55:09 DEBUG 06:55:09 DEBUG 06:55:09 DEBUG 06:55:09 DEBUG [...] 06:55:09 DEBUG 06:55:09 DEBUG 06:55:09 DEBUG 06:55:09 DEBUG "pod": "ds-cts-2" 06:55:09 DEBUG }, 06:55:09 DEBUG "value": [ 06:55:09 DEBUG 1713592509, 06:55:09 DEBUG "19.422466965049438" 06:55:09 DEBUG ] 06:55:09 DEBUG } 06:55:09 DEBUG ] 06:55:09 DEBUG } 06:55:09 DEBUG } 06:55:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:55:09 DEBUG ----- output ----- 06:55:09 DEBUG { 06:55:09 DEBUG "status": "success", 06:55:09 DEBUG "data": { 06:55:09 DEBUG "resultType": "vector", 06:55:09 DEBUG "result": [ 06:55:09 DEBUG { 06:55:09 DEBUG "metric": { 06:55:09 DEBUG "instance": "10.142.0.107:9100" 06:55:09 DEBUG }, 06:55:09 DEBUG "value": [ 06:55:09 DEBUG 06:55:09 DEBUG 06:55:09 DEBUG 06:55:09 DEBUG [...] 06:55:09 DEBUG 06:55:09 DEBUG 06:55:09 DEBUG 06:55:09 DEBUG "instance": "10.142.0.43:9100" 06:55:09 DEBUG }, 06:55:09 DEBUG "value": [ 06:55:09 DEBUG 1713592509, 06:55:09 DEBUG "80711374.23333332" 06:55:09 DEBUG ] 06:55:09 DEBUG } 06:55:09 DEBUG ] 06:55:09 DEBUG } 06:55:09 DEBUG } 06:55:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592509" 06:55:09 INFO [http_cmd]: http status code OK 06:55:09 DEBUG --- status code --- 06:55:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:55:09 DEBUG --- http response --- 06:55:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:55:09 DEBUG ----- output ----- 06:55:09 DEBUG { 06:55:09 DEBUG "status": "success", 06:55:09 DEBUG "data": { 06:55:09 DEBUG "resultType": "vector", 06:55:09 DEBUG "result": [ 06:55:09 DEBUG { 06:55:09 DEBUG "metric": { 06:55:09 DEBUG "instance": "10.142.0.107:9100" 06:55:09 DEBUG }, 06:55:09 DEBUG "value": [ 06:55:09 DEBUG 06:55:09 DEBUG 06:55:09 DEBUG 06:55:09 DEBUG [...] 06:55:09 DEBUG 06:55:09 DEBUG 06:55:09 DEBUG 06:55:09 DEBUG "instance": "10.142.0.43:9100" 06:55:09 DEBUG }, 06:55:09 DEBUG "value": [ 06:55:09 DEBUG 1713592509, 06:55:09 DEBUG "0.10166666666666703" 06:55:09 DEBUG ] 06:55:09 DEBUG } 06:55:09 DEBUG ] 06:55:09 DEBUG } 06:55:09 DEBUG } 06:55:10 INFO 06:55:10 INFO [loop_until]: kubectl --namespace=xlou top node 06:55:10 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:55:10 INFO 06:55:10 INFO [loop_until]: kubectl --namespace=xlou top pods 06:55:10 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:55:10 INFO [loop_until]: OK (rc = 0) 06:55:10 DEBUG --- stdout --- 06:55:10 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 69m 0% 6283Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7912m 26% 7453Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 17212m 57% 11359Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 88m 0% 3364Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 18767m 62% 13521Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 158m 0% 5212Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 17224m 57% 11602Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 19074m 63% 3010Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 19420m 65% 10866Mi 9% gke-xlou-cdm-ds-0f601751-j488 8160m 27% 7030Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 17492m 58% 15597Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 8011m 26% 7695Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 19381m 64% 12890Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 7911m 49% 8491Mi 14% 06:55:10 DEBUG --- stderr --- 06:55:10 DEBUG 06:55:10 INFO [loop_until]: OK (rc = 0) 06:55:10 DEBUG --- stdout --- 06:55:10 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 17193m 8651Mi am-767f6b67b5-qcqz2 17663m 8716Mi am-767f6b67b5-vbltz 18323m 8654Mi ds-cts-0 18749m 10380Mi ds-cts-1 17516m 10676Mi ds-cts-2 19013m 10279Mi ds-idrepo-0 7750m 3781Mi ds-idrepo-1 7461m 3803Mi ds-idrepo-2 7771m 3706Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1177Mi idm-b85f9d978-wnhxb 3m 1236Mi lodemon-7cb5b9456d-dskfn 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7792m 4674Mi 06:55:10 DEBUG --- stderr --- 06:55:10 DEBUG 06:56:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592568" 06:56:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592568" 06:56:08 INFO [http_cmd]: http status code OK 06:56:08 DEBUG --- status code --- 06:56:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:56:08 DEBUG --- http response --- 06:56:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:56:08 DEBUG ----- output ----- 06:56:08 DEBUG { 06:56:08 DEBUG "status": "success", 06:56:08 DEBUG "data": { 06:56:08 DEBUG "resultType": "vector", 06:56:08 DEBUG "result": [ 06:56:08 DEBUG { 06:56:08 DEBUG "metric": { 06:56:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:56:08 DEBUG }, 06:56:08 DEBUG "value": [ 06:56:08 DEBUG 06:56:08 DEBUG 06:56:08 DEBUG 06:56:08 DEBUG [...] 06:56:08 DEBUG 06:56:08 DEBUG 06:56:08 DEBUG 06:56:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:56:08 DEBUG }, 06:56:08 DEBUG "value": [ 06:56:08 DEBUG 1713592568, 06:56:08 DEBUG "479459.5" 06:56:08 DEBUG ] 06:56:08 DEBUG } 06:56:08 DEBUG ] 06:56:08 DEBUG } 06:56:08 DEBUG } 06:56:08 INFO [http_cmd]: http status code OK 06:56:08 DEBUG --- status code --- 06:56:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:56:08 DEBUG --- http response --- 06:56:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:56:08 DEBUG ----- output ----- 06:56:08 DEBUG { 06:56:08 DEBUG "status": "success", 06:56:08 DEBUG "data": { 06:56:08 DEBUG "resultType": "vector", 06:56:08 DEBUG "result": [ 06:56:08 DEBUG { 06:56:08 DEBUG "metric": { 06:56:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:56:08 DEBUG }, 06:56:08 DEBUG "value": [ 06:56:08 DEBUG 06:56:08 DEBUG 06:56:08 DEBUG 06:56:08 DEBUG [...] 06:56:08 DEBUG 06:56:08 DEBUG 06:56:08 DEBUG 06:56:08 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 06:56:08 DEBUG }, 06:56:08 DEBUG "value": [ 06:56:08 DEBUG 1713592568, 06:56:08 DEBUG "0" 06:56:08 DEBUG ] 06:56:08 DEBUG } 06:56:08 DEBUG ] 06:56:08 DEBUG } 06:56:08 DEBUG } 06:56:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713592568" 06:56:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592568" 06:56:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592568" 06:56:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713592568" 06:56:08 INFO [http_cmd]: http status code OK 06:56:08 DEBUG --- status code --- 06:56:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:56:08 DEBUG --- http response --- 06:56:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:56:08 DEBUG ----- output ----- 06:56:08 DEBUG { 06:56:08 DEBUG "status": "success", 06:56:08 DEBUG "data": { 06:56:08 DEBUG "resultType": "vector", 06:56:08 DEBUG "result": [ 06:56:08 DEBUG { 06:56:08 DEBUG "metric": { 06:56:08 DEBUG "pod": "ds-cts-0" 06:56:08 DEBUG }, 06:56:08 DEBUG "value": [ 06:56:08 DEBUG 06:56:08 DEBUG 06:56:08 DEBUG 06:56:08 DEBUG [...] 06:56:08 DEBUG 06:56:08 DEBUG 06:56:08 DEBUG 06:56:08 DEBUG "pod": "ds-cts-2" 06:56:08 DEBUG }, 06:56:08 DEBUG "value": [ 06:56:08 DEBUG 1713592568, 06:56:08 DEBUG "35.2" 06:56:08 DEBUG ] 06:56:08 DEBUG } 06:56:08 DEBUG ] 06:56:08 DEBUG } 06:56:08 DEBUG } 06:56:08 INFO [http_cmd]: http status code OK 06:56:08 DEBUG --- status code --- 06:56:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:56:08 DEBUG --- http response --- 06:56:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592568" 06:56:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:56:08 DEBUG ----- output ----- 06:56:08 DEBUG { 06:56:08 DEBUG "status": "success", 06:56:08 DEBUG "data": { 06:56:08 DEBUG "resultType": "vector", 06:56:08 DEBUG "result": [ 06:56:08 DEBUG { 06:56:08 DEBUG "metric": { 06:56:08 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 06:56:08 DEBUG }, 06:56:08 DEBUG "value": [ 06:56:08 DEBUG 06:56:08 DEBUG 06:56:08 DEBUG 06:56:08 DEBUG [...] 06:56:08 DEBUG 06:56:08 DEBUG 06:56:08 DEBUG 06:56:08 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 06:56:08 DEBUG }, 06:56:08 DEBUG "value": [ 06:56:08 DEBUG 1713592568, 06:56:08 DEBUG "4996390912" 06:56:08 DEBUG ] 06:56:08 DEBUG } 06:56:08 DEBUG ] 06:56:08 DEBUG } 06:56:08 DEBUG } 06:56:08 INFO [http_cmd]: http status code OK 06:56:08 DEBUG --- status code --- 06:56:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:56:08 DEBUG --- http response --- 06:56:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:56:08 DEBUG ----- output ----- 06:56:08 DEBUG { 06:56:08 DEBUG "status": "success", 06:56:08 DEBUG "data": { 06:56:08 DEBUG "resultType": "vector", 06:56:08 DEBUG "result": [ 06:56:08 DEBUG { 06:56:08 DEBUG "metric": { 06:56:08 DEBUG "instance": "10.142.0.107:9100" 06:56:08 DEBUG }, 06:56:08 DEBUG "value": [ 06:56:08 DEBUG 06:56:08 DEBUG 06:56:08 DEBUG 06:56:08 DEBUG [...] 06:56:08 DEBUG 06:56:08 DEBUG 06:56:08 DEBUG 06:56:08 DEBUG "instance": "10.142.0.43:9100" 06:56:08 DEBUG }, 06:56:08 DEBUG "value": [ 06:56:08 DEBUG 1713592568, 06:56:08 DEBUG "3247708793856" 06:56:08 DEBUG ] 06:56:08 DEBUG } 06:56:08 DEBUG ] 06:56:08 DEBUG } 06:56:08 DEBUG } 06:56:08 ERROR [http_cmd]: ERROR 06:56:08 ERROR --- status code --- 06:56:08 ERROR http status code is 400 (expected [200, 401, 403, 404]) 06:56:08 ERROR --- http response --- 06:56:08 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 06:56:08 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 06:56:08 INFO [http_cmd]: http status code OK 06:56:08 DEBUG --- status code --- 06:56:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:56:08 DEBUG --- http response --- 06:56:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:56:08 DEBUG ----- output ----- 06:56:08 DEBUG { 06:56:08 DEBUG "status": "success", 06:56:08 DEBUG "data": { 06:56:08 DEBUG "resultType": "vector", 06:56:08 DEBUG "result": [ 06:56:08 DEBUG { 06:56:08 DEBUG "metric": { 06:56:08 DEBUG "pod": "ds-idrepo-0" 06:56:08 DEBUG }, 06:56:08 DEBUG "value": [ 06:56:08 DEBUG 06:56:08 DEBUG 06:56:08 DEBUG 06:56:08 DEBUG [...] 06:56:08 DEBUG 06:56:08 DEBUG 06:56:08 DEBUG 06:56:08 DEBUG "pod": "ds-idrepo-2" 06:56:08 DEBUG }, 06:56:08 DEBUG "value": [ 06:56:08 DEBUG 1713592568, 06:56:08 DEBUG "0" 06:56:08 DEBUG ] 06:56:08 DEBUG } 06:56:08 DEBUG ] 06:56:08 DEBUG } 06:56:08 DEBUG } 06:56:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592568" 06:56:08 INFO [http_cmd]: http status code OK 06:56:08 DEBUG --- status code --- 06:56:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:56:08 DEBUG --- http response --- 06:56:08 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 06:56:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592568" 06:56:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592568" 06:56:08 INFO [http_cmd]: http status code OK 06:56:08 DEBUG --- status code --- 06:56:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:56:08 DEBUG --- http response --- 06:56:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:56:08 DEBUG ----- output ----- 06:56:08 DEBUG { 06:56:08 DEBUG "status": "success", 06:56:08 DEBUG "data": { 06:56:08 DEBUG "resultType": "vector", 06:56:08 DEBUG "result": [ 06:56:08 DEBUG { 06:56:08 DEBUG "metric": { 06:56:08 DEBUG "pod": "ds-idrepo-0" 06:56:08 DEBUG }, 06:56:08 DEBUG "value": [ 06:56:08 DEBUG 06:56:08 DEBUG 06:56:08 DEBUG 06:56:08 DEBUG [...] 06:56:08 DEBUG 06:56:08 DEBUG 06:56:08 DEBUG 06:56:08 DEBUG "pod": "ds-idrepo-2" 06:56:08 DEBUG }, 06:56:08 DEBUG "value": [ 06:56:08 DEBUG 1713592568, 06:56:08 DEBUG "0" 06:56:08 DEBUG ] 06:56:08 DEBUG } 06:56:08 DEBUG ] 06:56:08 DEBUG } 06:56:08 DEBUG } 06:56:08 INFO [http_cmd]: http status code OK 06:56:08 DEBUG --- status code --- 06:56:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:56:08 DEBUG --- http response --- 06:56:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:56:08 DEBUG ----- output ----- 06:56:08 DEBUG { 06:56:08 DEBUG "status": "success", 06:56:08 DEBUG "data": { 06:56:08 DEBUG "resultType": "vector", 06:56:08 DEBUG "result": [ 06:56:08 DEBUG { 06:56:08 DEBUG "metric": { 06:56:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:56:08 DEBUG }, 06:56:08 DEBUG "value": [ 06:56:08 DEBUG 06:56:08 DEBUG 06:56:08 DEBUG 06:56:08 DEBUG [...] 06:56:08 DEBUG 06:56:08 DEBUG 06:56:08 DEBUG 06:56:08 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 06:56:08 DEBUG }, 06:56:08 DEBUG "value": [ 06:56:08 DEBUG 1713592568, 06:56:08 DEBUG "0" 06:56:08 DEBUG ] 06:56:08 DEBUG } 06:56:08 DEBUG ] 06:56:08 DEBUG } 06:56:08 DEBUG } 06:56:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713592569" 06:56:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592569" 06:56:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592569" 06:56:09 INFO [http_cmd]: http status code OK 06:56:09 INFO [http_cmd]: http status code OK 06:56:09 DEBUG --- status code --- 06:56:09 DEBUG --- status code --- 06:56:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:56:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:56:09 DEBUG --- http response --- 06:56:09 DEBUG --- http response --- 06:56:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:56:09 DEBUG ----- output ----- 06:56:09 DEBUG { 06:56:09 DEBUG "status": "success", 06:56:09 DEBUG "data": { 06:56:09 DEBUG "resultType": "vector", 06:56:09 DEBUG "result": [ 06:56:09 DEBUG { 06:56:09 DEBUG "metric": { 06:56:09 DEBUG "pod": "ds-idrepo-0" 06:56:09 DEBUG }, 06:56:09 DEBUG "value": [ 06:56:09 DEBUG 06:56:09 DEBUG 06:56:09 DEBUG 06:56:09 DEBUG [...] 06:56:09 DEBUG 06:56:09 DEBUG 06:56:09 DEBUG 06:56:09 DEBUG "pod": "ds-cts-2" 06:56:09 DEBUG }, 06:56:09 DEBUG "value": [ 06:56:09 DEBUG 1713592569, 06:56:09 DEBUG "0" 06:56:09 DEBUG ] 06:56:09 DEBUG } 06:56:09 DEBUG ] 06:56:09 DEBUG } 06:56:09 DEBUG } 06:56:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:56:09 DEBUG ----- output ----- 06:56:09 DEBUG { 06:56:09 DEBUG "status": "success", 06:56:09 DEBUG "data": { 06:56:09 DEBUG "resultType": "vector", 06:56:09 DEBUG "result": [ 06:56:09 DEBUG { 06:56:09 DEBUG "metric": { 06:56:09 DEBUG "instance": "10.142.0.107:9100" 06:56:09 DEBUG }, 06:56:09 DEBUG "value": [ 06:56:09 DEBUG 06:56:09 DEBUG 06:56:09 DEBUG 06:56:09 DEBUG [...] 06:56:09 DEBUG 06:56:09 DEBUG 06:56:09 DEBUG 06:56:09 DEBUG "instance": "10.142.0.43:9100" 06:56:09 DEBUG }, 06:56:09 DEBUG "value": [ 06:56:09 DEBUG 1713592569, 06:56:09 DEBUG "0.22576666666664666" 06:56:09 DEBUG ] 06:56:09 DEBUG } 06:56:09 DEBUG ] 06:56:09 DEBUG } 06:56:09 DEBUG } 06:56:09 INFO [http_cmd]: http status code OK 06:56:09 DEBUG --- status code --- 06:56:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:56:09 DEBUG --- http response --- 06:56:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:56:09 DEBUG ----- output ----- 06:56:09 DEBUG { 06:56:09 DEBUG "status": "success", 06:56:09 DEBUG "data": { 06:56:09 DEBUG "resultType": "vector", 06:56:09 DEBUG "result": [ 06:56:09 DEBUG { 06:56:09 DEBUG "metric": { 06:56:09 DEBUG "pod": "ds-idrepo-0" 06:56:09 DEBUG }, 06:56:09 DEBUG "value": [ 06:56:09 DEBUG 06:56:09 DEBUG 06:56:09 DEBUG 06:56:09 DEBUG [...] 06:56:09 DEBUG 06:56:09 DEBUG 06:56:09 DEBUG 06:56:09 DEBUG "pod": "ds-idrepo-2" 06:56:09 DEBUG }, 06:56:09 DEBUG "value": [ 06:56:09 DEBUG 1713592569, 06:56:09 DEBUG "0" 06:56:09 DEBUG ] 06:56:09 DEBUG } 06:56:09 DEBUG ] 06:56:09 DEBUG } 06:56:09 DEBUG } 06:56:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592569" 06:56:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592569" 06:56:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592569" 06:56:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592569" 06:56:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592569" 06:56:09 INFO [http_cmd]: http status code OK 06:56:09 DEBUG --- status code --- 06:56:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:56:09 DEBUG --- http response --- 06:56:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:56:09 DEBUG ----- output ----- 06:56:09 DEBUG { 06:56:09 DEBUG "status": "success", 06:56:09 DEBUG "data": { 06:56:09 DEBUG "resultType": "vector", 06:56:09 DEBUG "result": [ 06:56:09 DEBUG { 06:56:09 DEBUG "metric": { 06:56:09 DEBUG "pod": "am-767f6b67b5-9zw22" 06:56:09 DEBUG }, 06:56:09 DEBUG "value": [ 06:56:09 DEBUG 06:56:09 DEBUG 06:56:09 DEBUG 06:56:09 DEBUG [...] 06:56:09 DEBUG 06:56:09 DEBUG 06:56:09 DEBUG 06:56:09 DEBUG "pod": "am-767f6b67b5-vbltz" 06:56:09 DEBUG }, 06:56:09 DEBUG "value": [ 06:56:09 DEBUG 1713592569, 06:56:09 DEBUG "0" 06:56:09 DEBUG ] 06:56:09 DEBUG } 06:56:09 DEBUG ] 06:56:09 DEBUG } 06:56:09 DEBUG } 06:56:09 INFO [http_cmd]: http status code OK 06:56:09 DEBUG --- status code --- 06:56:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:56:09 DEBUG --- http response --- 06:56:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:56:09 DEBUG ----- output ----- 06:56:09 DEBUG { 06:56:09 DEBUG "status": "success", 06:56:09 DEBUG "data": { 06:56:09 DEBUG "resultType": "vector", 06:56:09 DEBUG "result": [ 06:56:09 DEBUG { 06:56:09 DEBUG "metric": { 06:56:09 DEBUG "pod": "am-767f6b67b5-9zw22" 06:56:09 DEBUG }, 06:56:09 DEBUG "value": [ 06:56:09 DEBUG 06:56:09 DEBUG 06:56:09 DEBUG 06:56:09 DEBUG [...] 06:56:09 DEBUG 06:56:09 DEBUG 06:56:09 DEBUG 06:56:09 DEBUG "pod": "am-767f6b67b5-vbltz" 06:56:09 DEBUG }, 06:56:09 DEBUG "value": [ 06:56:09 DEBUG 1713592569, 06:56:09 DEBUG "2669.4666666666667" 06:56:09 DEBUG ] 06:56:09 DEBUG } 06:56:09 DEBUG ] 06:56:09 DEBUG } 06:56:09 DEBUG } 06:56:09 INFO [http_cmd]: http status code OK 06:56:09 DEBUG --- status code --- 06:56:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:56:09 DEBUG --- http response --- 06:56:09 INFO [http_cmd]: http status code OK 06:56:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 06:56:09 DEBUG --- status code --- 06:56:09 DEBUG ----- output ----- 06:56:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:56:09 DEBUG { 06:56:09 DEBUG --- http response --- 06:56:09 DEBUG "status": "success", 06:56:09 DEBUG "data": { 06:56:09 DEBUG "resultType": "vector", 06:56:09 DEBUG "result": [ 06:56:09 DEBUG { 06:56:09 DEBUG "metric": { 06:56:09 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:56:09 DEBUG }, 06:56:09 DEBUG "value": [ 06:56:09 DEBUG 06:56:09 DEBUG 06:56:09 DEBUG 06:56:09 DEBUG [...] 06:56:09 DEBUG 06:56:09 DEBUG 06:56:09 DEBUG 06:56:09 DEBUG "pod": "am-767f6b67b5-qcqz2" 06:56:09 DEBUG }, 06:56:09 DEBUG "value": [ 06:56:09 DEBUG 1713592569, 06:56:09 DEBUG "0" 06:56:09 DEBUG ] 06:56:09 DEBUG } 06:56:09 DEBUG ] 06:56:09 DEBUG } 06:56:09 DEBUG } 06:56:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:56:09 DEBUG ----- output ----- 06:56:09 DEBUG { 06:56:09 DEBUG "status": "success", 06:56:09 DEBUG "data": { 06:56:09 DEBUG "resultType": "vector", 06:56:09 DEBUG "result": [ 06:56:09 DEBUG { 06:56:09 DEBUG "metric": { 06:56:09 DEBUG "instance": "10.142.0.107:9100" 06:56:09 DEBUG }, 06:56:09 DEBUG "value": [ 06:56:09 DEBUG 06:56:09 DEBUG 06:56:09 DEBUG 06:56:09 DEBUG [...] 06:56:09 DEBUG 06:56:09 DEBUG 06:56:09 DEBUG 06:56:09 DEBUG "instance": "10.142.0.43:9100" 06:56:09 DEBUG }, 06:56:09 DEBUG "value": [ 06:56:09 DEBUG 1713592569, 06:56:09 DEBUG "80667038.63333333" 06:56:09 DEBUG ] 06:56:09 DEBUG } 06:56:09 DEBUG ] 06:56:09 DEBUG } 06:56:09 DEBUG } 06:56:09 INFO [http_cmd]: http status code OK 06:56:09 DEBUG --- status code --- 06:56:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:56:09 DEBUG --- http response --- 06:56:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:56:09 DEBUG ----- output ----- 06:56:09 DEBUG { 06:56:09 DEBUG "status": "success", 06:56:09 DEBUG "data": { 06:56:09 DEBUG "resultType": "vector", 06:56:09 DEBUG "result": [ 06:56:09 DEBUG { 06:56:09 DEBUG "metric": { 06:56:09 DEBUG "pod": "ds-cts-0" 06:56:09 DEBUG }, 06:56:09 DEBUG "value": [ 06:56:09 DEBUG 06:56:09 DEBUG 06:56:09 DEBUG 06:56:09 DEBUG [...] 06:56:09 DEBUG 06:56:09 DEBUG 06:56:09 DEBUG 06:56:09 DEBUG "pod": "ds-cts-2" 06:56:09 DEBUG }, 06:56:09 DEBUG "value": [ 06:56:09 DEBUG 1713592569, 06:56:09 DEBUG "0" 06:56:09 DEBUG ] 06:56:09 DEBUG } 06:56:09 DEBUG ] 06:56:09 DEBUG } 06:56:09 DEBUG } 06:56:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592569" 06:56:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592569" 06:56:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592569" 06:56:09 INFO [http_cmd]: http status code OK 06:56:09 DEBUG --- status code --- 06:56:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:56:09 DEBUG --- http response --- 06:56:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:56:09 DEBUG ----- output ----- 06:56:09 DEBUG { 06:56:09 DEBUG "status": "success", 06:56:09 DEBUG "data": { 06:56:09 DEBUG "resultType": "vector", 06:56:09 DEBUG "result": [ 06:56:09 DEBUG { 06:56:09 DEBUG "metric": { 06:56:09 DEBUG "pod": "ds-cts-0" 06:56:09 DEBUG }, 06:56:09 DEBUG "value": [ 06:56:09 DEBUG 06:56:09 DEBUG 06:56:09 DEBUG 06:56:09 DEBUG [...] 06:56:09 DEBUG 06:56:09 DEBUG 06:56:09 DEBUG 06:56:09 DEBUG "pod": "ds-cts-2" 06:56:09 DEBUG }, 06:56:09 DEBUG "value": [ 06:56:09 DEBUG 1713592569, 06:56:09 DEBUG "0.0010333333333333334" 06:56:09 DEBUG ] 06:56:09 DEBUG } 06:56:09 DEBUG ] 06:56:09 DEBUG } 06:56:09 DEBUG } 06:56:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713592569" 06:56:09 INFO [http_cmd]: http status code OK 06:56:09 DEBUG --- status code --- 06:56:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:56:09 DEBUG --- http response --- 06:56:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:56:09 DEBUG ----- output ----- 06:56:09 DEBUG { 06:56:09 DEBUG "status": "success", 06:56:09 DEBUG "data": { 06:56:09 DEBUG "resultType": "vector", 06:56:09 DEBUG "result": [ 06:56:09 DEBUG { 06:56:09 DEBUG "metric": { 06:56:09 DEBUG "pod": "am-767f6b67b5-9zw22" 06:56:09 DEBUG }, 06:56:09 DEBUG "value": [ 06:56:09 DEBUG 06:56:09 DEBUG 06:56:09 DEBUG 06:56:09 DEBUG [...] 06:56:09 DEBUG 06:56:09 DEBUG 06:56:09 DEBUG 06:56:09 DEBUG "pod": "am-767f6b67b5-vbltz" 06:56:09 DEBUG }, 06:56:09 DEBUG "value": [ 06:56:09 DEBUG 1713592569, 06:56:09 DEBUG "2662.3333333333335" 06:56:09 DEBUG ] 06:56:09 DEBUG } 06:56:09 DEBUG ] 06:56:09 DEBUG } 06:56:09 DEBUG } 06:56:09 INFO [http_cmd]: http status code OK 06:56:09 DEBUG --- status code --- 06:56:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:56:09 DEBUG --- http response --- 06:56:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:56:09 DEBUG ----- output ----- 06:56:09 DEBUG { 06:56:09 DEBUG "status": "success", 06:56:09 DEBUG "data": { 06:56:09 DEBUG "resultType": "vector", 06:56:09 DEBUG "result": [ 06:56:09 DEBUG { 06:56:09 DEBUG "metric": { 06:56:09 DEBUG "pod": "am-767f6b67b5-9zw22" 06:56:09 DEBUG }, 06:56:09 DEBUG "value": [ 06:56:09 DEBUG 06:56:09 DEBUG 06:56:09 DEBUG 06:56:09 DEBUG [...] 06:56:09 DEBUG 06:56:09 DEBUG 06:56:09 DEBUG 06:56:09 DEBUG "pod": "am-767f6b67b5-vbltz" 06:56:09 DEBUG }, 06:56:09 DEBUG "value": [ 06:56:09 DEBUG 1713592569, 06:56:09 DEBUG "2664.1" 06:56:09 DEBUG ] 06:56:09 DEBUG } 06:56:09 DEBUG ] 06:56:09 DEBUG } 06:56:09 DEBUG } 06:56:09 INFO [http_cmd]: http status code OK 06:56:09 DEBUG --- status code --- 06:56:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:56:09 DEBUG --- http response --- 06:56:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:56:09 DEBUG ----- output ----- 06:56:09 DEBUG { 06:56:09 DEBUG "status": "success", 06:56:09 DEBUG "data": { 06:56:09 DEBUG "resultType": "vector", 06:56:09 DEBUG "result": [ 06:56:09 DEBUG { 06:56:09 DEBUG "metric": { 06:56:09 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 06:56:09 DEBUG }, 06:56:09 DEBUG "value": [ 06:56:09 DEBUG 06:56:09 DEBUG 06:56:09 DEBUG 06:56:09 DEBUG [...] 06:56:09 DEBUG 06:56:09 DEBUG 06:56:09 DEBUG 06:56:09 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 06:56:09 DEBUG }, 06:56:09 DEBUG "value": [ 06:56:09 DEBUG 1713592569, 06:56:09 DEBUG "8.027467992789399" 06:56:09 DEBUG ] 06:56:09 DEBUG } 06:56:09 DEBUG ] 06:56:09 DEBUG } 06:56:09 DEBUG } 06:56:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713592569" 06:56:09 INFO [http_cmd]: http status code OK 06:56:09 DEBUG --- status code --- 06:56:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:56:09 DEBUG --- http response --- 06:56:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:56:09 DEBUG ----- output ----- 06:56:09 DEBUG { 06:56:09 DEBUG "status": "success", 06:56:09 DEBUG "data": { 06:56:09 DEBUG "resultType": "vector", 06:56:09 DEBUG "result": [ 06:56:09 DEBUG { 06:56:09 DEBUG "metric": { 06:56:09 DEBUG "pod": "am-767f6b67b5-9zw22" 06:56:09 DEBUG }, 06:56:09 DEBUG "value": [ 06:56:09 DEBUG 06:56:09 DEBUG 06:56:09 DEBUG 06:56:09 DEBUG [...] 06:56:09 DEBUG 06:56:09 DEBUG 06:56:09 DEBUG 06:56:09 DEBUG "pod": "am-767f6b67b5-vbltz" 06:56:09 DEBUG }, 06:56:09 DEBUG "value": [ 06:56:09 DEBUG 1713592569, 06:56:09 DEBUG "0.097" 06:56:09 DEBUG ] 06:56:09 DEBUG } 06:56:09 DEBUG ] 06:56:09 DEBUG } 06:56:09 DEBUG } 06:56:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592569" 06:56:09 INFO [http_cmd]: http status code OK 06:56:09 DEBUG --- status code --- 06:56:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:56:09 DEBUG --- http response --- 06:56:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:56:09 DEBUG ----- output ----- 06:56:09 DEBUG { 06:56:09 DEBUG "status": "success", 06:56:09 DEBUG "data": { 06:56:09 DEBUG "resultType": "vector", 06:56:09 DEBUG "result": [ 06:56:09 DEBUG { 06:56:09 DEBUG "metric": { 06:56:09 DEBUG "pod": "am-767f6b67b5-9zw22" 06:56:09 DEBUG }, 06:56:09 DEBUG "value": [ 06:56:09 DEBUG 06:56:09 DEBUG 06:56:09 DEBUG 06:56:09 DEBUG [...] 06:56:09 DEBUG 06:56:09 DEBUG 06:56:09 DEBUG 06:56:09 DEBUG "pod": "am-767f6b67b5-vbltz" 06:56:09 DEBUG }, 06:56:09 DEBUG "value": [ 06:56:09 DEBUG 1713592569, 06:56:09 DEBUG "2663.366666666667" 06:56:09 DEBUG ] 06:56:09 DEBUG } 06:56:09 DEBUG ] 06:56:09 DEBUG } 06:56:09 DEBUG } 06:56:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592569" 06:56:09 INFO [http_cmd]: http status code OK 06:56:09 DEBUG --- status code --- 06:56:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:56:09 DEBUG --- http response --- 06:56:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 06:56:09 DEBUG ----- output ----- 06:56:09 DEBUG { 06:56:09 DEBUG "status": "success", 06:56:09 DEBUG "data": { 06:56:09 DEBUG "resultType": "vector", 06:56:09 DEBUG "result": [ 06:56:09 DEBUG { 06:56:09 DEBUG "metric": { 06:56:09 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:56:09 DEBUG }, 06:56:09 DEBUG "value": [ 06:56:09 DEBUG 06:56:09 DEBUG 06:56:09 DEBUG 06:56:09 DEBUG [...] 06:56:09 DEBUG 06:56:09 DEBUG 06:56:09 DEBUG 06:56:09 DEBUG "pod": "am-767f6b67b5-qcqz2" 06:56:09 DEBUG }, 06:56:09 DEBUG "value": [ 06:56:09 DEBUG 1713592569, 06:56:09 DEBUG "0" 06:56:09 DEBUG ] 06:56:09 DEBUG } 06:56:09 DEBUG ] 06:56:09 DEBUG } 06:56:09 DEBUG } 06:56:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592569" 06:56:09 INFO [http_cmd]: http status code OK 06:56:09 DEBUG --- status code --- 06:56:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:56:09 DEBUG --- http response --- 06:56:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:56:09 DEBUG ----- output ----- 06:56:09 DEBUG { 06:56:09 DEBUG "status": "success", 06:56:09 DEBUG "data": { 06:56:09 DEBUG "resultType": "vector", 06:56:09 DEBUG "result": [ 06:56:09 DEBUG { 06:56:09 DEBUG "metric": { 06:56:09 DEBUG "instance": "10.142.0.107:9100" 06:56:09 DEBUG }, 06:56:09 DEBUG "value": [ 06:56:09 DEBUG 06:56:09 DEBUG 06:56:09 DEBUG 06:56:09 DEBUG [...] 06:56:09 DEBUG 06:56:09 DEBUG 06:56:09 DEBUG 06:56:09 DEBUG "instance": "10.142.0.43:9100" 06:56:09 DEBUG }, 06:56:09 DEBUG "value": [ 06:56:09 DEBUG 1713592569, 06:56:09 DEBUG "80666968.03333333" 06:56:09 DEBUG ] 06:56:09 DEBUG } 06:56:09 DEBUG ] 06:56:09 DEBUG } 06:56:09 DEBUG } 06:56:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592569" 06:56:09 INFO [http_cmd]: http status code OK 06:56:09 DEBUG --- status code --- 06:56:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:56:09 DEBUG --- http response --- 06:56:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:56:09 DEBUG ----- output ----- 06:56:09 DEBUG { 06:56:09 DEBUG "status": "success", 06:56:09 DEBUG "data": { 06:56:09 DEBUG "resultType": "vector", 06:56:09 DEBUG "result": [ 06:56:09 DEBUG { 06:56:09 DEBUG "metric": { 06:56:09 DEBUG "instance": "10.142.0.107:9100" 06:56:09 DEBUG }, 06:56:09 DEBUG "value": [ 06:56:09 DEBUG 06:56:09 DEBUG 06:56:09 DEBUG 06:56:09 DEBUG [...] 06:56:09 DEBUG 06:56:09 DEBUG 06:56:09 DEBUG 06:56:09 DEBUG "instance": "10.142.0.43:9100" 06:56:09 DEBUG }, 06:56:09 DEBUG "value": [ 06:56:09 DEBUG 1713592569, 06:56:09 DEBUG "0.08399999999999562" 06:56:09 DEBUG ] 06:56:09 DEBUG } 06:56:09 DEBUG ] 06:56:09 DEBUG } 06:56:09 DEBUG } 06:56:10 INFO 06:56:10 INFO [loop_until]: kubectl --namespace=xlou top node 06:56:10 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:56:10 INFO 06:56:10 INFO [loop_until]: kubectl --namespace=xlou top pods 06:56:10 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:56:10 INFO [loop_until]: OK (rc = 0) 06:56:10 DEBUG --- stdout --- 06:56:10 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 68m 0% 6271Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7978m 26% 7459Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 16646m 55% 11364Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 77m 0% 3373Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 18720m 62% 13579Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 148m 0% 5216Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 17460m 58% 11603Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 19090m 63% 3018Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 19492m 65% 10868Mi 9% gke-xlou-cdm-ds-0f601751-j488 8139m 27% 7027Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 16791m 56% 15603Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 7990m 26% 7690Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 19384m 64% 12879Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 8162m 51% 8493Mi 14% 06:56:10 DEBUG --- stderr --- 06:56:10 DEBUG 06:56:10 INFO [loop_until]: OK (rc = 0) 06:56:10 DEBUG --- stdout --- 06:56:10 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 17637m 8651Mi am-767f6b67b5-qcqz2 17418m 8717Mi am-767f6b67b5-vbltz 18601m 8654Mi ds-cts-0 18758m 10428Mi ds-cts-1 17995m 10678Mi ds-cts-2 19779m 10258Mi ds-idrepo-0 7683m 3786Mi ds-idrepo-1 7652m 3802Mi ds-idrepo-2 7749m 3706Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1177Mi idm-b85f9d978-wnhxb 4m 1236Mi lodemon-7cb5b9456d-dskfn 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7897m 4675Mi 06:56:10 DEBUG --- stderr --- 06:56:10 DEBUG 06:57:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592628" 06:57:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592628" 06:57:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592628" 06:57:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713592628" 06:57:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592628" 06:57:08 INFO [http_cmd]: http status code OK 06:57:08 DEBUG --- status code --- 06:57:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:57:08 DEBUG --- http response --- 06:57:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:57:08 DEBUG ----- output ----- 06:57:08 DEBUG { 06:57:08 DEBUG "status": "success", 06:57:08 DEBUG "data": { 06:57:08 DEBUG "resultType": "vector", 06:57:08 DEBUG "result": [ 06:57:08 DEBUG { 06:57:08 DEBUG "metric": { 06:57:08 DEBUG "pod": "ds-idrepo-0" 06:57:08 DEBUG }, 06:57:08 DEBUG "value": [ 06:57:08 DEBUG 06:57:08 DEBUG 06:57:08 DEBUG 06:57:08 DEBUG [...] 06:57:08 DEBUG 06:57:08 DEBUG 06:57:08 DEBUG 06:57:08 DEBUG "pod": "ds-idrepo-2" 06:57:08 DEBUG }, 06:57:08 DEBUG "value": [ 06:57:08 DEBUG 1713592628, 06:57:08 DEBUG "0" 06:57:08 DEBUG ] 06:57:08 DEBUG } 06:57:08 DEBUG ] 06:57:08 DEBUG } 06:57:08 DEBUG } 06:57:08 INFO [http_cmd]: http status code OK 06:57:08 DEBUG --- status code --- 06:57:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:57:08 DEBUG --- http response --- 06:57:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592628" 06:57:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:57:08 DEBUG ----- output ----- 06:57:08 DEBUG { 06:57:08 DEBUG "status": "success", 06:57:08 DEBUG "data": { 06:57:08 DEBUG "resultType": "vector", 06:57:08 DEBUG "result": [ 06:57:08 DEBUG { 06:57:08 DEBUG "metric": { 06:57:08 DEBUG "pod": "am-767f6b67b5-9zw22" 06:57:08 DEBUG }, 06:57:08 DEBUG "value": [ 06:57:08 DEBUG 06:57:08 DEBUG 06:57:08 DEBUG 06:57:08 DEBUG [...] 06:57:08 DEBUG 06:57:08 DEBUG 06:57:08 DEBUG 06:57:08 DEBUG "pod": "am-767f6b67b5-vbltz" 06:57:08 DEBUG }, 06:57:08 DEBUG "value": [ 06:57:08 DEBUG 1713592628, 06:57:08 DEBUG "479684" 06:57:08 DEBUG ] 06:57:08 DEBUG } 06:57:08 DEBUG ] 06:57:08 DEBUG } 06:57:08 DEBUG } 06:57:08 INFO [http_cmd]: http status code OK 06:57:08 DEBUG --- status code --- 06:57:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:57:08 DEBUG --- http response --- 06:57:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:57:08 DEBUG ----- output ----- 06:57:08 DEBUG { 06:57:08 DEBUG "status": "success", 06:57:08 DEBUG "data": { 06:57:08 DEBUG "resultType": "vector", 06:57:08 DEBUG "result": [ 06:57:08 DEBUG { 06:57:08 DEBUG "metric": { 06:57:08 DEBUG "instance": "10.142.0.107:9100" 06:57:08 DEBUG }, 06:57:08 DEBUG "value": [ 06:57:08 DEBUG 06:57:08 DEBUG 06:57:08 DEBUG 06:57:08 DEBUG [...] 06:57:08 DEBUG 06:57:08 DEBUG 06:57:08 DEBUG 06:57:08 DEBUG "instance": "10.142.0.43:9100" 06:57:08 DEBUG }, 06:57:08 DEBUG "value": [ 06:57:08 DEBUG 1713592628, 06:57:08 DEBUG "3251690224640" 06:57:08 DEBUG ] 06:57:08 DEBUG } 06:57:08 DEBUG ] 06:57:08 DEBUG } 06:57:08 DEBUG } 06:57:08 INFO [http_cmd]: http status code OK 06:57:08 DEBUG --- status code --- 06:57:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:57:08 DEBUG --- http response --- 06:57:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:57:08 DEBUG ----- output ----- 06:57:08 DEBUG { 06:57:08 DEBUG "status": "success", 06:57:08 DEBUG "data": { 06:57:08 DEBUG "resultType": "vector", 06:57:08 DEBUG "result": [ 06:57:08 DEBUG { 06:57:08 DEBUG "metric": { 06:57:08 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:57:08 DEBUG }, 06:57:08 DEBUG "value": [ 06:57:08 DEBUG 06:57:08 DEBUG 06:57:08 DEBUG 06:57:08 DEBUG [...] 06:57:08 DEBUG 06:57:08 DEBUG 06:57:08 DEBUG 06:57:08 DEBUG "pod": "ds-idrepo-2" 06:57:08 DEBUG }, 06:57:08 DEBUG "value": [ 06:57:08 DEBUG 1713592628, 06:57:08 DEBUG "0" 06:57:08 DEBUG ] 06:57:08 DEBUG } 06:57:08 DEBUG ] 06:57:08 DEBUG } 06:57:08 DEBUG } 06:57:08 INFO [http_cmd]: http status code OK 06:57:08 DEBUG --- status code --- 06:57:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:57:08 DEBUG --- http response --- 06:57:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:57:08 DEBUG ----- output ----- 06:57:08 DEBUG { 06:57:08 DEBUG "status": "success", 06:57:08 DEBUG "data": { 06:57:08 DEBUG "resultType": "vector", 06:57:08 DEBUG "result": [ 06:57:08 DEBUG { 06:57:08 DEBUG "metric": { 06:57:08 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 06:57:08 DEBUG }, 06:57:08 DEBUG "value": [ 06:57:08 DEBUG 06:57:08 DEBUG 06:57:08 DEBUG 06:57:08 DEBUG [...] 06:57:08 DEBUG 06:57:08 DEBUG 06:57:08 DEBUG 06:57:08 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 06:57:08 DEBUG }, 06:57:08 DEBUG "value": [ 06:57:08 DEBUG 1713592628, 06:57:08 DEBUG "5000572928" 06:57:08 DEBUG ] 06:57:08 DEBUG } 06:57:08 DEBUG ] 06:57:08 DEBUG } 06:57:08 DEBUG } 06:57:08 INFO [http_cmd]: http status code OK 06:57:08 DEBUG --- status code --- 06:57:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:57:08 DEBUG --- http response --- 06:57:08 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 06:57:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592628" 06:57:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713592628" 06:57:08 INFO [http_cmd]: http status code OK 06:57:08 DEBUG --- status code --- 06:57:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:57:08 DEBUG --- http response --- 06:57:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:57:08 DEBUG ----- output ----- 06:57:08 DEBUG { 06:57:08 DEBUG "status": "success", 06:57:08 DEBUG "data": { 06:57:08 DEBUG "resultType": "vector", 06:57:08 DEBUG "result": [ 06:57:08 DEBUG { 06:57:08 DEBUG "metric": { 06:57:08 DEBUG "pod": "ds-cts-0" 06:57:08 DEBUG }, 06:57:08 DEBUG "value": [ 06:57:08 DEBUG 06:57:08 DEBUG 06:57:08 DEBUG 06:57:08 DEBUG [...] 06:57:08 DEBUG 06:57:08 DEBUG 06:57:08 DEBUG 06:57:08 DEBUG "pod": "ds-cts-2" 06:57:08 DEBUG }, 06:57:08 DEBUG "value": [ 06:57:08 DEBUG 1713592628, 06:57:08 DEBUG "26" 06:57:08 DEBUG ] 06:57:08 DEBUG } 06:57:08 DEBUG ] 06:57:08 DEBUG } 06:57:08 DEBUG } 06:57:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592628" 06:57:08 ERROR [http_cmd]: ERROR 06:57:08 ERROR --- status code --- 06:57:08 ERROR http status code is 400 (expected [200, 401, 403, 404]) 06:57:08 ERROR --- http response --- 06:57:08 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 06:57:08 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 06:57:08 INFO [http_cmd]: http status code OK 06:57:08 DEBUG --- status code --- 06:57:08 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:57:08 DEBUG --- http response --- 06:57:08 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:57:08 DEBUG ----- output ----- 06:57:08 DEBUG { 06:57:08 DEBUG "status": "success", 06:57:08 DEBUG "data": { 06:57:08 DEBUG "resultType": "vector", 06:57:08 DEBUG "result": [ 06:57:08 DEBUG { 06:57:08 DEBUG "metric": { 06:57:08 DEBUG "pod": "ds-idrepo-0" 06:57:08 DEBUG }, 06:57:08 DEBUG "value": [ 06:57:08 DEBUG 06:57:08 DEBUG 06:57:08 DEBUG 06:57:08 DEBUG [...] 06:57:08 DEBUG 06:57:08 DEBUG 06:57:08 DEBUG 06:57:08 DEBUG "pod": "ds-idrepo-2" 06:57:08 DEBUG }, 06:57:08 DEBUG "value": [ 06:57:08 DEBUG 1713592628, 06:57:08 DEBUG "0" 06:57:08 DEBUG ] 06:57:08 DEBUG } 06:57:08 DEBUG ] 06:57:08 DEBUG } 06:57:08 DEBUG } 06:57:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592629" 06:57:09 INFO [http_cmd]: http status code OK 06:57:09 DEBUG --- status code --- 06:57:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:57:09 DEBUG --- http response --- 06:57:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:57:09 DEBUG ----- output ----- 06:57:09 DEBUG { 06:57:09 DEBUG "status": "success", 06:57:09 DEBUG "data": { 06:57:09 DEBUG "resultType": "vector", 06:57:09 DEBUG "result": [ 06:57:09 DEBUG { 06:57:09 DEBUG "metric": { 06:57:09 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:57:09 DEBUG }, 06:57:09 DEBUG "value": [ 06:57:09 DEBUG 06:57:09 DEBUG 06:57:09 DEBUG 06:57:09 DEBUG [...] 06:57:09 DEBUG 06:57:09 DEBUG 06:57:09 DEBUG 06:57:09 DEBUG "pod": "ds-idrepo-2" 06:57:09 DEBUG }, 06:57:09 DEBUG "value": [ 06:57:09 DEBUG 1713592629, 06:57:09 DEBUG "4428.407324819244" 06:57:09 DEBUG ] 06:57:09 DEBUG } 06:57:09 DEBUG ] 06:57:09 DEBUG } 06:57:09 DEBUG } 06:57:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592629" 06:57:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592629" 06:57:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592629" 06:57:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592629" 06:57:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713592629" 06:57:09 INFO [http_cmd]: http status code OK 06:57:09 DEBUG --- status code --- 06:57:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:57:09 DEBUG --- http response --- 06:57:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:57:09 DEBUG ----- output ----- 06:57:09 DEBUG { 06:57:09 DEBUG "status": "success", 06:57:09 DEBUG "data": { 06:57:09 DEBUG "resultType": "vector", 06:57:09 DEBUG "result": [ 06:57:09 DEBUG { 06:57:09 DEBUG "metric": { 06:57:09 DEBUG "instance": "10.142.0.107:9100" 06:57:09 DEBUG }, 06:57:09 DEBUG "value": [ 06:57:09 DEBUG 06:57:09 DEBUG 06:57:09 DEBUG 06:57:09 DEBUG [...] 06:57:09 DEBUG 06:57:09 DEBUG 06:57:09 DEBUG 06:57:09 DEBUG "instance": "10.142.0.43:9100" 06:57:09 DEBUG }, 06:57:09 DEBUG "value": [ 06:57:09 DEBUG 1713592629, 06:57:09 DEBUG "80595013.6" 06:57:09 DEBUG ] 06:57:09 DEBUG } 06:57:09 DEBUG ] 06:57:09 DEBUG } 06:57:09 DEBUG } 06:57:09 INFO [http_cmd]: http status code OK 06:57:09 INFO [http_cmd]: http status code OK 06:57:09 DEBUG --- status code --- 06:57:09 DEBUG --- status code --- 06:57:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:57:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:57:09 DEBUG --- http response --- 06:57:09 DEBUG --- http response --- 06:57:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:57:09 DEBUG ----- output ----- 06:57:09 DEBUG { 06:57:09 DEBUG "status": "success", 06:57:09 DEBUG "data": { 06:57:09 DEBUG "resultType": "vector", 06:57:09 DEBUG "result": [ 06:57:09 DEBUG { 06:57:09 DEBUG "metric": { 06:57:09 DEBUG "pod": "ds-cts-0" 06:57:09 DEBUG }, 06:57:09 DEBUG "value": [ 06:57:09 DEBUG 06:57:09 DEBUG 06:57:09 DEBUG 06:57:09 DEBUG [...] 06:57:09 DEBUG 06:57:09 DEBUG 06:57:09 DEBUG 06:57:09 DEBUG "pod": "ds-cts-2" 06:57:09 DEBUG }, 06:57:09 DEBUG "value": [ 06:57:09 DEBUG 1713592629, 06:57:09 DEBUG "0" 06:57:09 DEBUG ] 06:57:09 DEBUG } 06:57:09 DEBUG ] 06:57:09 DEBUG } 06:57:09 DEBUG } 06:57:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592629" 06:57:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:57:09 DEBUG ----- output ----- 06:57:09 DEBUG { 06:57:09 DEBUG "status": "success", 06:57:09 DEBUG "data": { 06:57:09 DEBUG "resultType": "vector", 06:57:09 DEBUG "result": [ 06:57:09 DEBUG { 06:57:09 DEBUG "metric": { 06:57:09 DEBUG "pod": "am-767f6b67b5-9zw22" 06:57:09 DEBUG }, 06:57:09 DEBUG "value": [ 06:57:09 DEBUG 06:57:09 DEBUG 06:57:09 DEBUG 06:57:09 DEBUG [...] 06:57:09 DEBUG 06:57:09 DEBUG 06:57:09 DEBUG 06:57:09 DEBUG "pod": "am-767f6b67b5-vbltz" 06:57:09 DEBUG }, 06:57:09 DEBUG "value": [ 06:57:09 DEBUG 1713592629, 06:57:09 DEBUG "2664.1" 06:57:09 DEBUG ] 06:57:09 DEBUG } 06:57:09 DEBUG ] 06:57:09 DEBUG } 06:57:09 DEBUG } 06:57:09 INFO [http_cmd]: http status code OK 06:57:09 INFO [http_cmd]: http status code OK 06:57:09 DEBUG --- status code --- 06:57:09 DEBUG --- status code --- 06:57:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:57:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:57:09 DEBUG --- http response --- 06:57:09 DEBUG --- http response --- 06:57:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:57:09 DEBUG ----- output ----- 06:57:09 DEBUG { 06:57:09 DEBUG "status": "success", 06:57:09 DEBUG "data": { 06:57:09 DEBUG "resultType": "vector", 06:57:09 DEBUG "result": [ 06:57:09 DEBUG { 06:57:09 DEBUG "metric": { 06:57:09 DEBUG "pod": "ds-idrepo-0" 06:57:09 DEBUG }, 06:57:09 DEBUG "value": [ 06:57:09 DEBUG 06:57:09 DEBUG 06:57:09 DEBUG 06:57:09 DEBUG [...] 06:57:09 DEBUG 06:57:09 DEBUG 06:57:09 DEBUG 06:57:09 DEBUG "pod": "ds-cts-2" 06:57:09 DEBUG }, 06:57:09 DEBUG "value": [ 06:57:09 DEBUG 1713592629, 06:57:09 DEBUG "0" 06:57:09 DEBUG ] 06:57:09 DEBUG } 06:57:09 DEBUG ] 06:57:09 DEBUG } 06:57:09 DEBUG } 06:57:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:57:09 DEBUG ----- output ----- 06:57:09 DEBUG { 06:57:09 DEBUG "status": "success", 06:57:09 DEBUG "data": { 06:57:09 DEBUG "resultType": "vector", 06:57:09 DEBUG "result": [ 06:57:09 DEBUG { 06:57:09 DEBUG "metric": { 06:57:09 DEBUG "instance": "10.142.0.107:9100" 06:57:09 DEBUG }, 06:57:09 DEBUG "value": [ 06:57:09 DEBUG 06:57:09 DEBUG 06:57:09 DEBUG 06:57:09 DEBUG [...] 06:57:09 DEBUG 06:57:09 DEBUG 06:57:09 DEBUG 06:57:09 DEBUG "instance": "10.142.0.43:9100" 06:57:09 DEBUG }, 06:57:09 DEBUG "value": [ 06:57:09 DEBUG 1713592629, 06:57:09 DEBUG "0.17150000000006096" 06:57:09 DEBUG ] 06:57:09 DEBUG } 06:57:09 DEBUG ] 06:57:09 DEBUG } 06:57:09 DEBUG } 06:57:09 INFO [http_cmd]: http status code OK 06:57:09 DEBUG --- status code --- 06:57:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:57:09 DEBUG --- http response --- 06:57:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:57:09 DEBUG ----- output ----- 06:57:09 DEBUG { 06:57:09 DEBUG "status": "success", 06:57:09 DEBUG "data": { 06:57:09 DEBUG "resultType": "vector", 06:57:09 DEBUG "result": [ 06:57:09 DEBUG { 06:57:09 DEBUG "metric": { 06:57:09 DEBUG "pod": "ds-idrepo-0" 06:57:09 DEBUG }, 06:57:09 DEBUG "value": [ 06:57:09 DEBUG 06:57:09 DEBUG 06:57:09 DEBUG 06:57:09 DEBUG [...] 06:57:09 DEBUG 06:57:09 DEBUG 06:57:09 DEBUG 06:57:09 DEBUG "pod": "ds-idrepo-2" 06:57:09 DEBUG }, 06:57:09 DEBUG "value": [ 06:57:09 DEBUG 1713592629, 06:57:09 DEBUG "0" 06:57:09 DEBUG ] 06:57:09 DEBUG } 06:57:09 DEBUG ] 06:57:09 DEBUG } 06:57:09 DEBUG } 06:57:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592629" 06:57:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592629" 06:57:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592629" 06:57:09 INFO [http_cmd]: http status code OK 06:57:09 DEBUG --- status code --- 06:57:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:57:09 DEBUG --- http response --- 06:57:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:57:09 DEBUG ----- output ----- 06:57:09 DEBUG { 06:57:09 DEBUG "status": "success", 06:57:09 DEBUG "data": { 06:57:09 DEBUG "resultType": "vector", 06:57:09 DEBUG "result": [ 06:57:09 DEBUG { 06:57:09 DEBUG "metric": { 06:57:09 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:57:09 DEBUG }, 06:57:09 DEBUG "value": [ 06:57:09 DEBUG 06:57:09 DEBUG 06:57:09 DEBUG 06:57:09 DEBUG [...] 06:57:09 DEBUG 06:57:09 DEBUG 06:57:09 DEBUG 06:57:09 DEBUG "pod": "ds-idrepo-2" 06:57:09 DEBUG }, 06:57:09 DEBUG "value": [ 06:57:09 DEBUG 1713592629, 06:57:09 DEBUG "205644.16514629367" 06:57:09 DEBUG ] 06:57:09 DEBUG } 06:57:09 DEBUG ] 06:57:09 DEBUG } 06:57:09 DEBUG } 06:57:09 INFO [http_cmd]: http status code OK 06:57:09 DEBUG --- status code --- 06:57:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:57:09 DEBUG --- http response --- 06:57:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:57:09 DEBUG ----- output ----- 06:57:09 DEBUG { 06:57:09 DEBUG "status": "success", 06:57:09 DEBUG "data": { 06:57:09 DEBUG "resultType": "vector", 06:57:09 DEBUG "result": [ 06:57:09 DEBUG { 06:57:09 DEBUG "metric": { 06:57:09 DEBUG "pod": "am-767f6b67b5-9zw22" 06:57:09 DEBUG }, 06:57:09 DEBUG "value": [ 06:57:09 DEBUG 06:57:09 DEBUG 06:57:09 DEBUG 06:57:09 DEBUG [...] 06:57:09 DEBUG 06:57:09 DEBUG 06:57:09 DEBUG 06:57:09 DEBUG "pod": "am-767f6b67b5-vbltz" 06:57:09 DEBUG }, 06:57:09 DEBUG "value": [ 06:57:09 DEBUG 1713592629, 06:57:09 DEBUG "2675.866666666667" 06:57:09 DEBUG ] 06:57:09 DEBUG } 06:57:09 DEBUG ] 06:57:09 DEBUG } 06:57:09 DEBUG } 06:57:09 INFO [http_cmd]: http status code OK 06:57:09 DEBUG --- status code --- 06:57:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:57:09 DEBUG --- http response --- 06:57:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:57:09 DEBUG ----- output ----- 06:57:09 DEBUG { 06:57:09 DEBUG "status": "success", 06:57:09 DEBUG "data": { 06:57:09 DEBUG "resultType": "vector", 06:57:09 DEBUG "result": [ 06:57:09 DEBUG { 06:57:09 DEBUG "metric": { 06:57:09 DEBUG "pod": "am-767f6b67b5-9zw22" 06:57:09 DEBUG }, 06:57:09 DEBUG "value": [ 06:57:09 DEBUG 06:57:09 DEBUG 06:57:09 DEBUG 06:57:09 DEBUG [...] 06:57:09 DEBUG 06:57:09 DEBUG 06:57:09 DEBUG 06:57:09 DEBUG "pod": "am-767f6b67b5-vbltz" 06:57:09 DEBUG }, 06:57:09 DEBUG "value": [ 06:57:09 DEBUG 1713592629, 06:57:09 DEBUG "0" 06:57:09 DEBUG ] 06:57:09 DEBUG } 06:57:09 DEBUG ] 06:57:09 DEBUG } 06:57:09 DEBUG } 06:57:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592629" 06:57:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592629" 06:57:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713592629" 06:57:09 INFO [http_cmd]: http status code OK 06:57:09 DEBUG --- status code --- 06:57:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:57:09 DEBUG --- http response --- 06:57:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:57:09 DEBUG ----- output ----- 06:57:09 DEBUG { 06:57:09 DEBUG "status": "success", 06:57:09 DEBUG "data": { 06:57:09 DEBUG "resultType": "vector", 06:57:09 DEBUG "result": [ 06:57:09 DEBUG { 06:57:09 DEBUG "metric": { 06:57:09 DEBUG "pod": "ds-cts-0" 06:57:09 DEBUG }, 06:57:09 DEBUG "value": [ 06:57:09 DEBUG 06:57:09 DEBUG 06:57:09 DEBUG 06:57:09 DEBUG [...] 06:57:09 DEBUG 06:57:09 DEBUG 06:57:09 DEBUG 06:57:09 DEBUG "pod": "ds-cts-2" 06:57:09 DEBUG }, 06:57:09 DEBUG "value": [ 06:57:09 DEBUG 1713592629, 06:57:09 DEBUG "0.0009622577777777778" 06:57:09 DEBUG ] 06:57:09 DEBUG } 06:57:09 DEBUG ] 06:57:09 DEBUG } 06:57:09 DEBUG } 06:57:09 INFO [http_cmd]: http status code OK 06:57:09 DEBUG --- status code --- 06:57:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:57:09 DEBUG --- http response --- 06:57:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:57:09 DEBUG ----- output ----- 06:57:09 DEBUG { 06:57:09 DEBUG "status": "success", 06:57:09 DEBUG "data": { 06:57:09 DEBUG "resultType": "vector", 06:57:09 DEBUG "result": [ 06:57:09 DEBUG { 06:57:09 DEBUG "metric": { 06:57:09 DEBUG "pod": "am-767f6b67b5-9zw22" 06:57:09 DEBUG }, 06:57:09 DEBUG "value": [ 06:57:09 DEBUG 06:57:09 DEBUG 06:57:09 DEBUG 06:57:09 DEBUG [...] 06:57:09 DEBUG 06:57:09 DEBUG 06:57:09 DEBUG 06:57:09 DEBUG "pod": "am-767f6b67b5-vbltz" 06:57:09 DEBUG }, 06:57:09 DEBUG "value": [ 06:57:09 DEBUG 1713592629, 06:57:09 DEBUG "2672.9666666666667" 06:57:09 DEBUG ] 06:57:09 DEBUG } 06:57:09 DEBUG ] 06:57:09 DEBUG } 06:57:09 DEBUG } 06:57:09 INFO [http_cmd]: http status code OK 06:57:09 DEBUG --- status code --- 06:57:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:57:09 DEBUG --- http response --- 06:57:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:57:09 DEBUG ----- output ----- 06:57:09 DEBUG { 06:57:09 DEBUG "status": "success", 06:57:09 DEBUG "data": { 06:57:09 DEBUG "resultType": "vector", 06:57:09 DEBUG "result": [ 06:57:09 DEBUG { 06:57:09 DEBUG "metric": { 06:57:09 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 06:57:09 DEBUG }, 06:57:09 DEBUG "value": [ 06:57:09 DEBUG 06:57:09 DEBUG 06:57:09 DEBUG 06:57:09 DEBUG [...] 06:57:09 DEBUG 06:57:09 DEBUG 06:57:09 DEBUG 06:57:09 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 06:57:09 DEBUG }, 06:57:09 DEBUG "value": [ 06:57:09 DEBUG 1713592629, 06:57:09 DEBUG "8.03290224320012" 06:57:09 DEBUG ] 06:57:09 DEBUG } 06:57:09 DEBUG ] 06:57:09 DEBUG } 06:57:09 DEBUG } 06:57:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713592629" 06:57:09 INFO [http_cmd]: http status code OK 06:57:09 DEBUG --- status code --- 06:57:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:57:09 DEBUG --- http response --- 06:57:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:57:09 DEBUG ----- output ----- 06:57:09 DEBUG { 06:57:09 DEBUG "status": "success", 06:57:09 DEBUG "data": { 06:57:09 DEBUG "resultType": "vector", 06:57:09 DEBUG "result": [ 06:57:09 DEBUG { 06:57:09 DEBUG "metric": { 06:57:09 DEBUG "pod": "am-767f6b67b5-9zw22" 06:57:09 DEBUG }, 06:57:09 DEBUG "value": [ 06:57:09 DEBUG 06:57:09 DEBUG 06:57:09 DEBUG 06:57:09 DEBUG [...] 06:57:09 DEBUG 06:57:09 DEBUG 06:57:09 DEBUG 06:57:09 DEBUG "pod": "am-767f6b67b5-vbltz" 06:57:09 DEBUG }, 06:57:09 DEBUG "value": [ 06:57:09 DEBUG 1713592629, 06:57:09 DEBUG "0.097" 06:57:09 DEBUG ] 06:57:09 DEBUG } 06:57:09 DEBUG ] 06:57:09 DEBUG } 06:57:09 DEBUG } 06:57:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592629" 06:57:09 INFO [http_cmd]: http status code OK 06:57:09 DEBUG --- status code --- 06:57:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:57:09 DEBUG --- http response --- 06:57:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:57:09 DEBUG ----- output ----- 06:57:09 DEBUG { 06:57:09 DEBUG "status": "success", 06:57:09 DEBUG "data": { 06:57:09 DEBUG "resultType": "vector", 06:57:09 DEBUG "result": [ 06:57:09 DEBUG { 06:57:09 DEBUG "metric": { 06:57:09 DEBUG "pod": "am-767f6b67b5-9zw22" 06:57:09 DEBUG }, 06:57:09 DEBUG "value": [ 06:57:09 DEBUG 06:57:09 DEBUG 06:57:09 DEBUG 06:57:09 DEBUG [...] 06:57:09 DEBUG 06:57:09 DEBUG 06:57:09 DEBUG 06:57:09 DEBUG "pod": "am-767f6b67b5-vbltz" 06:57:09 DEBUG }, 06:57:09 DEBUG "value": [ 06:57:09 DEBUG 1713592629, 06:57:09 DEBUG "2676.5333333333333" 06:57:09 DEBUG ] 06:57:09 DEBUG } 06:57:09 DEBUG ] 06:57:09 DEBUG } 06:57:09 DEBUG } 06:57:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592629" 06:57:09 INFO [http_cmd]: http status code OK 06:57:09 DEBUG --- status code --- 06:57:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:57:09 DEBUG --- http response --- 06:57:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:57:09 DEBUG ----- output ----- 06:57:09 DEBUG { 06:57:09 DEBUG "status": "success", 06:57:09 DEBUG "data": { 06:57:09 DEBUG "resultType": "vector", 06:57:09 DEBUG "result": [ 06:57:09 DEBUG { 06:57:09 DEBUG "metric": { 06:57:09 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:57:09 DEBUG }, 06:57:09 DEBUG "value": [ 06:57:09 DEBUG 06:57:09 DEBUG 06:57:09 DEBUG 06:57:09 DEBUG [...] 06:57:09 DEBUG 06:57:09 DEBUG 06:57:09 DEBUG 06:57:09 DEBUG "pod": "ds-idrepo-2" 06:57:09 DEBUG }, 06:57:09 DEBUG "value": [ 06:57:09 DEBUG 1713592629, 06:57:09 DEBUG "7.830710047974914" 06:57:09 DEBUG ] 06:57:09 DEBUG } 06:57:09 DEBUG ] 06:57:09 DEBUG } 06:57:09 DEBUG } 06:57:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592629" 06:57:09 INFO [http_cmd]: http status code OK 06:57:09 DEBUG --- status code --- 06:57:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:57:09 DEBUG --- http response --- 06:57:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:57:09 DEBUG ----- output ----- 06:57:09 DEBUG { 06:57:09 DEBUG "status": "success", 06:57:09 DEBUG "data": { 06:57:09 DEBUG "resultType": "vector", 06:57:09 DEBUG "result": [ 06:57:09 DEBUG { 06:57:09 DEBUG "metric": { 06:57:09 DEBUG "instance": "10.142.0.107:9100" 06:57:09 DEBUG }, 06:57:09 DEBUG "value": [ 06:57:09 DEBUG 06:57:09 DEBUG 06:57:09 DEBUG 06:57:09 DEBUG [...] 06:57:09 DEBUG 06:57:09 DEBUG 06:57:09 DEBUG 06:57:09 DEBUG "instance": "10.142.0.43:9100" 06:57:09 DEBUG }, 06:57:09 DEBUG "value": [ 06:57:09 DEBUG 1713592629, 06:57:09 DEBUG "80595884.63333333" 06:57:09 DEBUG ] 06:57:09 DEBUG } 06:57:09 DEBUG ] 06:57:09 DEBUG } 06:57:09 DEBUG } 06:57:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592629" 06:57:09 INFO [http_cmd]: http status code OK 06:57:09 DEBUG --- status code --- 06:57:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:57:09 DEBUG --- http response --- 06:57:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:57:09 DEBUG ----- output ----- 06:57:09 DEBUG { 06:57:09 DEBUG "status": "success", 06:57:09 DEBUG "data": { 06:57:09 DEBUG "resultType": "vector", 06:57:09 DEBUG "result": [ 06:57:09 DEBUG { 06:57:09 DEBUG "metric": { 06:57:09 DEBUG "instance": "10.142.0.107:9100" 06:57:09 DEBUG }, 06:57:09 DEBUG "value": [ 06:57:09 DEBUG 06:57:09 DEBUG 06:57:09 DEBUG 06:57:09 DEBUG [...] 06:57:09 DEBUG 06:57:09 DEBUG 06:57:09 DEBUG 06:57:09 DEBUG "instance": "10.142.0.43:9100" 06:57:09 DEBUG }, 06:57:09 DEBUG "value": [ 06:57:09 DEBUG 1713592629, 06:57:09 DEBUG "0.07000000000000833" 06:57:09 DEBUG ] 06:57:09 DEBUG } 06:57:09 DEBUG ] 06:57:09 DEBUG } 06:57:09 DEBUG } 06:57:10 INFO 06:57:10 INFO [loop_until]: kubectl --namespace=xlou top node 06:57:10 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:57:10 INFO 06:57:10 INFO [loop_until]: kubectl --namespace=xlou top pods 06:57:10 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:57:10 INFO [loop_until]: OK (rc = 0) 06:57:10 DEBUG --- stdout --- 06:57:10 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 69m 0% 6264Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 8031m 26% 7462Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 17211m 57% 11366Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 78m 0% 3376Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 18037m 60% 13564Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 164m 0% 5222Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 17485m 58% 11607Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 18770m 62% 3018Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 19406m 65% 10864Mi 9% gke-xlou-cdm-ds-0f601751-j488 8235m 27% 7043Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 17096m 57% 15601Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 8033m 26% 7699Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 19565m 65% 12880Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 8174m 51% 8491Mi 14% 06:57:10 DEBUG --- stderr --- 06:57:10 DEBUG 06:57:10 INFO [loop_until]: OK (rc = 0) 06:57:10 DEBUG --- stdout --- 06:57:10 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 16893m 8651Mi am-767f6b67b5-qcqz2 17489m 8718Mi am-767f6b67b5-vbltz 18809m 8653Mi ds-cts-0 18525m 10435Mi ds-cts-1 17160m 10674Mi ds-cts-2 19762m 10273Mi ds-idrepo-0 7757m 3787Mi ds-idrepo-1 7479m 3803Mi ds-idrepo-2 7866m 3712Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1177Mi idm-b85f9d978-wnhxb 3m 1236Mi lodemon-7cb5b9456d-dskfn 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7975m 4677Mi 06:57:10 DEBUG --- stderr --- 06:57:10 DEBUG 06:58:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592688" 06:58:08 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592688" 06:58:09 INFO [http_cmd]: http status code OK 06:58:09 DEBUG --- status code --- 06:58:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:58:09 DEBUG --- http response --- 06:58:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:58:09 DEBUG ----- output ----- 06:58:09 DEBUG { 06:58:09 DEBUG "status": "success", 06:58:09 DEBUG "data": { 06:58:09 DEBUG "resultType": "vector", 06:58:09 DEBUG "result": [ 06:58:09 DEBUG { 06:58:09 DEBUG "metric": { 06:58:09 DEBUG "instance": "10.142.0.107:9100" 06:58:09 DEBUG }, 06:58:09 DEBUG "value": [ 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG [...] 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG "instance": "10.142.0.43:9100" 06:58:09 DEBUG }, 06:58:09 DEBUG "value": [ 06:58:09 DEBUG 1713592688, 06:58:09 DEBUG "3255678217216" 06:58:09 DEBUG ] 06:58:09 DEBUG } 06:58:09 DEBUG ] 06:58:09 DEBUG } 06:58:09 DEBUG } 06:58:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592689" 06:58:09 INFO [http_cmd]: http status code OK 06:58:09 DEBUG --- status code --- 06:58:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:58:09 DEBUG --- http response --- 06:58:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592689" 06:58:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:58:09 DEBUG ----- output ----- 06:58:09 DEBUG { 06:58:09 DEBUG "status": "success", 06:58:09 DEBUG "data": { 06:58:09 DEBUG "resultType": "vector", 06:58:09 DEBUG "result": [ 06:58:09 DEBUG { 06:58:09 DEBUG "metric": { 06:58:09 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:58:09 DEBUG }, 06:58:09 DEBUG "value": [ 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG [...] 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 06:58:09 DEBUG }, 06:58:09 DEBUG "value": [ 06:58:09 DEBUG 1713592688, 06:58:09 DEBUG "0" 06:58:09 DEBUG ] 06:58:09 DEBUG } 06:58:09 DEBUG ] 06:58:09 DEBUG } 06:58:09 DEBUG } 06:58:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713592689" 06:58:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592689" 06:58:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592689" 06:58:09 INFO [http_cmd]: http status code OK 06:58:09 INFO [http_cmd]: http status code OK 06:58:09 DEBUG --- status code --- 06:58:09 DEBUG --- status code --- 06:58:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:58:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:58:09 DEBUG --- http response --- 06:58:09 DEBUG --- http response --- 06:58:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:58:09 DEBUG ----- output ----- 06:58:09 DEBUG { 06:58:09 DEBUG "status": "success", 06:58:09 DEBUG "data": { 06:58:09 DEBUG "resultType": "vector", 06:58:09 DEBUG "result": [ 06:58:09 DEBUG { 06:58:09 DEBUG "metric": { 06:58:09 DEBUG "pod": "ds-idrepo-0" 06:58:09 DEBUG }, 06:58:09 DEBUG "value": [ 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG [...] 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG "pod": "ds-idrepo-2" 06:58:09 DEBUG }, 06:58:09 DEBUG "value": [ 06:58:09 DEBUG 1713592689, 06:58:09 DEBUG "0" 06:58:09 DEBUG ] 06:58:09 DEBUG } 06:58:09 DEBUG ] 06:58:09 DEBUG } 06:58:09 DEBUG } 06:58:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:58:09 DEBUG ----- output ----- 06:58:09 DEBUG { 06:58:09 DEBUG "status": "success", 06:58:09 DEBUG "data": { 06:58:09 DEBUG "resultType": "vector", 06:58:09 DEBUG "result": [ 06:58:09 DEBUG { 06:58:09 DEBUG "metric": { 06:58:09 DEBUG "pod": "am-767f6b67b5-9zw22" 06:58:09 DEBUG }, 06:58:09 DEBUG "value": [ 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG [...] 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG "pod": "am-767f6b67b5-vbltz" 06:58:09 DEBUG }, 06:58:09 DEBUG "value": [ 06:58:09 DEBUG 1713592689, 06:58:09 DEBUG "479893" 06:58:09 DEBUG ] 06:58:09 DEBUG } 06:58:09 DEBUG ] 06:58:09 DEBUG } 06:58:09 DEBUG } 06:58:09 INFO [http_cmd]: http status code OK 06:58:09 DEBUG --- status code --- 06:58:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:58:09 DEBUG --- http response --- 06:58:09 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 06:58:09 INFO [http_cmd]: http status code OK 06:58:09 DEBUG --- status code --- 06:58:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:58:09 DEBUG --- http response --- 06:58:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:58:09 DEBUG ----- output ----- 06:58:09 DEBUG { 06:58:09 DEBUG "status": "success", 06:58:09 DEBUG "data": { 06:58:09 DEBUG "resultType": "vector", 06:58:09 DEBUG "result": [ 06:58:09 DEBUG { 06:58:09 DEBUG "metric": { 06:58:09 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 06:58:09 DEBUG }, 06:58:09 DEBUG "value": [ 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG [...] 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 06:58:09 DEBUG }, 06:58:09 DEBUG "value": [ 06:58:09 DEBUG 1713592689, 06:58:09 DEBUG "5001818112" 06:58:09 DEBUG ] 06:58:09 DEBUG } 06:58:09 DEBUG ] 06:58:09 DEBUG } 06:58:09 DEBUG } 06:58:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592689" 06:58:09 INFO [http_cmd]: http status code OK 06:58:09 DEBUG --- status code --- 06:58:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:58:09 DEBUG --- http response --- 06:58:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:58:09 DEBUG ----- output ----- 06:58:09 DEBUG { 06:58:09 DEBUG "status": "success", 06:58:09 DEBUG "data": { 06:58:09 DEBUG "resultType": "vector", 06:58:09 DEBUG "result": [ 06:58:09 DEBUG { 06:58:09 DEBUG "metric": { 06:58:09 DEBUG "pod": "ds-idrepo-0" 06:58:09 DEBUG }, 06:58:09 DEBUG "value": [ 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG [...] 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG "pod": "ds-idrepo-2" 06:58:09 DEBUG }, 06:58:09 DEBUG "value": [ 06:58:09 DEBUG 1713592689, 06:58:09 DEBUG "0" 06:58:09 DEBUG ] 06:58:09 DEBUG } 06:58:09 DEBUG ] 06:58:09 DEBUG } 06:58:09 DEBUG } 06:58:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713592689" 06:58:09 INFO [http_cmd]: http status code OK 06:58:09 DEBUG --- status code --- 06:58:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:58:09 DEBUG --- http response --- 06:58:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:58:09 DEBUG ----- output ----- 06:58:09 DEBUG { 06:58:09 DEBUG "status": "success", 06:58:09 DEBUG "data": { 06:58:09 DEBUG "resultType": "vector", 06:58:09 DEBUG "result": [ 06:58:09 DEBUG { 06:58:09 DEBUG "metric": { 06:58:09 DEBUG "pod": "ds-cts-0" 06:58:09 DEBUG }, 06:58:09 DEBUG "value": [ 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG [...] 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG "pod": "ds-cts-2" 06:58:09 DEBUG }, 06:58:09 DEBUG "value": [ 06:58:09 DEBUG 1713592689, 06:58:09 DEBUG "29.966666666666665" 06:58:09 DEBUG ] 06:58:09 DEBUG } 06:58:09 DEBUG ] 06:58:09 DEBUG } 06:58:09 DEBUG } 06:58:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592689" 06:58:09 ERROR [http_cmd]: ERROR 06:58:09 ERROR --- status code --- 06:58:09 ERROR http status code is 400 (expected [200, 401, 403, 404]) 06:58:09 ERROR --- http response --- 06:58:09 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 06:58:09 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 06:58:09 INFO [http_cmd]: http status code OK 06:58:09 DEBUG --- status code --- 06:58:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:58:09 DEBUG --- http response --- 06:58:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 06:58:09 DEBUG ----- output ----- 06:58:09 DEBUG { 06:58:09 DEBUG "status": "success", 06:58:09 DEBUG "data": { 06:58:09 DEBUG "resultType": "vector", 06:58:09 DEBUG "result": [ 06:58:09 DEBUG { 06:58:09 DEBUG "metric": { 06:58:09 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:58:09 DEBUG }, 06:58:09 DEBUG "value": [ 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG [...] 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG "pod": "ds-idrepo-2" 06:58:09 DEBUG }, 06:58:09 DEBUG "value": [ 06:58:09 DEBUG 1713592689, 06:58:09 DEBUG "2460.7853712364035" 06:58:09 DEBUG ] 06:58:09 DEBUG } 06:58:09 DEBUG ] 06:58:09 DEBUG } 06:58:09 DEBUG } 06:58:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713592689" 06:58:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592689" 06:58:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592689" 06:58:09 INFO [http_cmd]: http status code OK 06:58:09 DEBUG --- status code --- 06:58:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:58:09 DEBUG --- http response --- 06:58:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:58:09 DEBUG ----- output ----- 06:58:09 DEBUG { 06:58:09 DEBUG "status": "success", 06:58:09 DEBUG "data": { 06:58:09 DEBUG "resultType": "vector", 06:58:09 DEBUG "result": [ 06:58:09 DEBUG { 06:58:09 DEBUG "metric": { 06:58:09 DEBUG "pod": "ds-idrepo-0" 06:58:09 DEBUG }, 06:58:09 DEBUG "value": [ 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG [...] 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG "pod": "ds-cts-2" 06:58:09 DEBUG }, 06:58:09 DEBUG "value": [ 06:58:09 DEBUG 1713592689, 06:58:09 DEBUG "0" 06:58:09 DEBUG ] 06:58:09 DEBUG } 06:58:09 DEBUG ] 06:58:09 DEBUG } 06:58:09 DEBUG } 06:58:09 INFO [http_cmd]: http status code OK 06:58:09 DEBUG --- status code --- 06:58:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:58:09 DEBUG --- http response --- 06:58:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592689" 06:58:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:58:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592689" 06:58:09 INFO [http_cmd]: http status code OK 06:58:09 DEBUG ----- output ----- 06:58:09 DEBUG --- status code --- 06:58:09 DEBUG { 06:58:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:58:09 DEBUG "status": "success", 06:58:09 DEBUG --- http response --- 06:58:09 DEBUG "data": { 06:58:09 DEBUG "resultType": "vector", 06:58:09 DEBUG "result": [ 06:58:09 DEBUG { 06:58:09 DEBUG "metric": { 06:58:09 DEBUG "instance": "10.142.0.107:9100" 06:58:09 DEBUG }, 06:58:09 DEBUG "value": [ 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG [...] 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG "instance": "10.142.0.43:9100" 06:58:09 DEBUG }, 06:58:09 DEBUG "value": [ 06:58:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:58:09 DEBUG 1713592689, 06:58:09 DEBUG ----- output ----- 06:58:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592689" 06:58:09 DEBUG "80770787.19999999" 06:58:09 DEBUG { 06:58:09 DEBUG ] 06:58:09 DEBUG "status": "success", 06:58:09 DEBUG } 06:58:09 DEBUG "data": { 06:58:09 DEBUG ] 06:58:09 DEBUG "resultType": "vector", 06:58:09 DEBUG } 06:58:09 DEBUG "result": [ 06:58:09 DEBUG } 06:58:09 DEBUG { 06:58:09 DEBUG "metric": { 06:58:09 DEBUG "pod": "am-767f6b67b5-9zw22" 06:58:09 DEBUG }, 06:58:09 DEBUG "value": [ 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG [...] 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG "pod": "am-767f6b67b5-vbltz" 06:58:09 DEBUG }, 06:58:09 DEBUG "value": [ 06:58:09 DEBUG 1713592689, 06:58:09 DEBUG "2669.5666666666666" 06:58:09 DEBUG ] 06:58:09 DEBUG } 06:58:09 DEBUG ] 06:58:09 DEBUG } 06:58:09 DEBUG } 06:58:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592689" 06:58:09 INFO [http_cmd]: http status code OK 06:58:09 INFO [http_cmd]: http status code OK 06:58:09 DEBUG --- status code --- 06:58:09 DEBUG --- status code --- 06:58:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:58:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:58:09 DEBUG --- http response --- 06:58:09 DEBUG --- http response --- 06:58:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:58:09 DEBUG ----- output ----- 06:58:09 DEBUG { 06:58:09 DEBUG "status": "success", 06:58:09 DEBUG "data": { 06:58:09 DEBUG "resultType": "vector", 06:58:09 DEBUG "result": [ 06:58:09 DEBUG { 06:58:09 DEBUG "metric": { 06:58:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:58:09 DEBUG "pod": "ds-cts-0" 06:58:09 DEBUG ----- output ----- 06:58:09 DEBUG }, 06:58:09 DEBUG { 06:58:09 DEBUG "value": [ 06:58:09 DEBUG "status": "success", 06:58:09 DEBUG 06:58:09 DEBUG "data": { 06:58:09 DEBUG 06:58:09 DEBUG "resultType": "vector", 06:58:09 DEBUG 06:58:09 DEBUG "result": [ 06:58:09 DEBUG [...] 06:58:09 DEBUG { 06:58:09 DEBUG 06:58:09 DEBUG "metric": { 06:58:09 DEBUG 06:58:09 DEBUG "pod": "am-767f6b67b5-9zw22" 06:58:09 DEBUG 06:58:09 DEBUG }, 06:58:09 DEBUG "pod": "ds-cts-2" 06:58:09 DEBUG "value": [ 06:58:09 DEBUG }, 06:58:09 DEBUG 06:58:09 DEBUG "value": [ 06:58:09 DEBUG 06:58:09 DEBUG 1713592689, 06:58:09 DEBUG 06:58:09 DEBUG "0" 06:58:09 DEBUG [...] 06:58:09 DEBUG ] 06:58:09 DEBUG 06:58:09 DEBUG } 06:58:09 DEBUG 06:58:09 DEBUG ] 06:58:09 INFO [http_cmd]: http status code OK 06:58:09 DEBUG 06:58:09 DEBUG } 06:58:09 DEBUG --- status code --- 06:58:09 DEBUG "pod": "am-767f6b67b5-vbltz" 06:58:09 DEBUG } 06:58:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:58:09 DEBUG }, 06:58:09 DEBUG --- http response --- 06:58:09 DEBUG "value": [ 06:58:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:58:09 DEBUG 1713592689, 06:58:09 DEBUG ----- output ----- 06:58:09 DEBUG "2648.5" 06:58:09 DEBUG { 06:58:09 DEBUG ] 06:58:09 DEBUG "status": "success", 06:58:09 DEBUG } 06:58:09 DEBUG "data": { 06:58:09 DEBUG ] 06:58:09 DEBUG "resultType": "vector", 06:58:09 DEBUG } 06:58:09 DEBUG "result": [ 06:58:09 DEBUG } 06:58:09 DEBUG { 06:58:09 DEBUG "metric": { 06:58:09 DEBUG "instance": "10.142.0.107:9100" 06:58:09 DEBUG }, 06:58:09 DEBUG "value": [ 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG [...] 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG "instance": "10.142.0.43:9100" 06:58:09 DEBUG }, 06:58:09 DEBUG "value": [ 06:58:09 DEBUG 1713592689, 06:58:09 DEBUG "0.20136666666665423" 06:58:09 DEBUG ] 06:58:09 DEBUG } 06:58:09 DEBUG ] 06:58:09 DEBUG } 06:58:09 DEBUG } 06:58:09 INFO [http_cmd]: http status code OK 06:58:09 DEBUG --- status code --- 06:58:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:58:09 DEBUG --- http response --- 06:58:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:58:09 DEBUG ----- output ----- 06:58:09 DEBUG { 06:58:09 DEBUG "status": "success", 06:58:09 DEBUG "data": { 06:58:09 DEBUG "resultType": "vector", 06:58:09 DEBUG "result": [ 06:58:09 DEBUG { 06:58:09 DEBUG "metric": { 06:58:09 DEBUG "pod": "ds-idrepo-0" 06:58:09 DEBUG }, 06:58:09 DEBUG "value": [ 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG [...] 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG "pod": "ds-idrepo-2" 06:58:09 DEBUG }, 06:58:09 DEBUG "value": [ 06:58:09 DEBUG 1713592689, 06:58:09 DEBUG "0" 06:58:09 DEBUG ] 06:58:09 DEBUG } 06:58:09 DEBUG ] 06:58:09 DEBUG } 06:58:09 DEBUG } 06:58:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592689" 06:58:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592689" 06:58:09 INFO [http_cmd]: http status code OK 06:58:09 DEBUG --- status code --- 06:58:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:58:09 DEBUG --- http response --- 06:58:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 06:58:09 DEBUG ----- output ----- 06:58:09 DEBUG { 06:58:09 DEBUG "status": "success", 06:58:09 DEBUG "data": { 06:58:09 DEBUG "resultType": "vector", 06:58:09 DEBUG "result": [ 06:58:09 DEBUG { 06:58:09 DEBUG "metric": { 06:58:09 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:58:09 DEBUG }, 06:58:09 DEBUG "value": [ 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG [...] 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG "pod": "ds-idrepo-2" 06:58:09 DEBUG }, 06:58:09 DEBUG "value": [ 06:58:09 DEBUG 1713592689, 06:58:09 DEBUG "8100.085180319829" 06:58:09 DEBUG ] 06:58:09 DEBUG } 06:58:09 DEBUG ] 06:58:09 DEBUG } 06:58:09 DEBUG } 06:58:09 INFO [http_cmd]: http status code OK 06:58:09 DEBUG --- status code --- 06:58:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:58:09 DEBUG --- http response --- 06:58:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:58:09 DEBUG ----- output ----- 06:58:09 DEBUG { 06:58:09 DEBUG "status": "success", 06:58:09 DEBUG "data": { 06:58:09 DEBUG "resultType": "vector", 06:58:09 DEBUG "result": [ 06:58:09 DEBUG { 06:58:09 DEBUG "metric": { 06:58:09 DEBUG "pod": "am-767f6b67b5-9zw22" 06:58:09 DEBUG }, 06:58:09 DEBUG "value": [ 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG [...] 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG "pod": "am-767f6b67b5-vbltz" 06:58:09 DEBUG }, 06:58:09 DEBUG "value": [ 06:58:09 DEBUG 1713592689, 06:58:09 DEBUG "0" 06:58:09 DEBUG ] 06:58:09 DEBUG } 06:58:09 DEBUG ] 06:58:09 DEBUG } 06:58:09 DEBUG } 06:58:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592689" 06:58:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592689" 06:58:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713592689" 06:58:09 INFO [http_cmd]: http status code OK 06:58:09 DEBUG --- status code --- 06:58:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:58:09 DEBUG --- http response --- 06:58:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:58:09 DEBUG ----- output ----- 06:58:09 DEBUG { 06:58:09 DEBUG "status": "success", 06:58:09 DEBUG "data": { 06:58:09 DEBUG "resultType": "vector", 06:58:09 DEBUG "result": [ 06:58:09 DEBUG { 06:58:09 DEBUG "metric": { 06:58:09 DEBUG "pod": "ds-cts-0" 06:58:09 DEBUG }, 06:58:09 DEBUG "value": [ 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG [...] 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG "pod": "ds-cts-2" 06:58:09 DEBUG }, 06:58:09 DEBUG "value": [ 06:58:09 DEBUG 1713592689, 06:58:09 DEBUG "0.000499108888888889" 06:58:09 DEBUG ] 06:58:09 DEBUG } 06:58:09 DEBUG ] 06:58:09 DEBUG } 06:58:09 DEBUG } 06:58:09 INFO [http_cmd]: http status code OK 06:58:09 DEBUG --- status code --- 06:58:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:58:09 DEBUG --- http response --- 06:58:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:58:09 DEBUG ----- output ----- 06:58:09 DEBUG { 06:58:09 DEBUG "status": "success", 06:58:09 DEBUG "data": { 06:58:09 DEBUG "resultType": "vector", 06:58:09 DEBUG "result": [ 06:58:09 DEBUG { 06:58:09 DEBUG "metric": { 06:58:09 DEBUG "pod": "am-767f6b67b5-9zw22" 06:58:09 DEBUG }, 06:58:09 DEBUG "value": [ 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG [...] 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG "pod": "am-767f6b67b5-vbltz" 06:58:09 DEBUG }, 06:58:09 DEBUG "value": [ 06:58:09 DEBUG 1713592689, 06:58:09 DEBUG "2671.5666666666666" 06:58:09 DEBUG ] 06:58:09 DEBUG } 06:58:09 DEBUG ] 06:58:09 DEBUG } 06:58:09 DEBUG } 06:58:09 INFO [http_cmd]: http status code OK 06:58:09 DEBUG --- status code --- 06:58:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:58:09 DEBUG --- http response --- 06:58:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:58:09 DEBUG ----- output ----- 06:58:09 DEBUG { 06:58:09 DEBUG "status": "success", 06:58:09 DEBUG "data": { 06:58:09 DEBUG "resultType": "vector", 06:58:09 DEBUG "result": [ 06:58:09 DEBUG { 06:58:09 DEBUG "metric": { 06:58:09 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 06:58:09 DEBUG }, 06:58:09 DEBUG "value": [ 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG [...] 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 06:58:09 DEBUG }, 06:58:09 DEBUG "value": [ 06:58:09 DEBUG 1713592689, 06:58:09 DEBUG "8.037061021669567" 06:58:09 DEBUG ] 06:58:09 DEBUG } 06:58:09 DEBUG ] 06:58:09 DEBUG } 06:58:09 DEBUG } 06:58:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713592689" 06:58:09 INFO [http_cmd]: http status code OK 06:58:09 DEBUG --- status code --- 06:58:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:58:09 DEBUG --- http response --- 06:58:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:58:09 DEBUG ----- output ----- 06:58:09 DEBUG { 06:58:09 DEBUG "status": "success", 06:58:09 DEBUG "data": { 06:58:09 DEBUG "resultType": "vector", 06:58:09 DEBUG "result": [ 06:58:09 DEBUG { 06:58:09 DEBUG "metric": { 06:58:09 DEBUG "pod": "am-767f6b67b5-9zw22" 06:58:09 DEBUG }, 06:58:09 DEBUG "value": [ 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG [...] 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG "pod": "am-767f6b67b5-vbltz" 06:58:09 DEBUG }, 06:58:09 DEBUG "value": [ 06:58:09 DEBUG 1713592689, 06:58:09 DEBUG "0.097" 06:58:09 DEBUG ] 06:58:09 DEBUG } 06:58:09 DEBUG ] 06:58:09 DEBUG } 06:58:09 DEBUG } 06:58:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592689" 06:58:09 INFO [http_cmd]: http status code OK 06:58:09 DEBUG --- status code --- 06:58:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:58:09 DEBUG --- http response --- 06:58:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:58:09 DEBUG ----- output ----- 06:58:09 DEBUG { 06:58:09 DEBUG "status": "success", 06:58:09 DEBUG "data": { 06:58:09 DEBUG "resultType": "vector", 06:58:09 DEBUG "result": [ 06:58:09 DEBUG { 06:58:09 DEBUG "metric": { 06:58:09 DEBUG "pod": "am-767f6b67b5-9zw22" 06:58:09 DEBUG }, 06:58:09 DEBUG "value": [ 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG [...] 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG "pod": "am-767f6b67b5-vbltz" 06:58:09 DEBUG }, 06:58:09 DEBUG "value": [ 06:58:09 DEBUG 1713592689, 06:58:09 DEBUG "2669.0333333333333" 06:58:09 DEBUG ] 06:58:09 DEBUG } 06:58:09 DEBUG ] 06:58:09 DEBUG } 06:58:09 DEBUG } 06:58:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592689" 06:58:09 INFO [http_cmd]: http status code OK 06:58:09 DEBUG --- status code --- 06:58:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:58:09 DEBUG --- http response --- 06:58:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 06:58:09 DEBUG ----- output ----- 06:58:09 DEBUG { 06:58:09 DEBUG "status": "success", 06:58:09 DEBUG "data": { 06:58:09 DEBUG "resultType": "vector", 06:58:09 DEBUG "result": [ 06:58:09 DEBUG { 06:58:09 DEBUG "metric": { 06:58:09 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:58:09 DEBUG }, 06:58:09 DEBUG "value": [ 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG [...] 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG "pod": "ds-idrepo-2" 06:58:09 DEBUG }, 06:58:09 DEBUG "value": [ 06:58:09 DEBUG 1713592689, 06:58:09 DEBUG "3.6861055907853046" 06:58:09 DEBUG ] 06:58:09 DEBUG } 06:58:09 DEBUG ] 06:58:09 DEBUG } 06:58:09 DEBUG } 06:58:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592689" 06:58:09 INFO [http_cmd]: http status code OK 06:58:09 DEBUG --- status code --- 06:58:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:58:09 DEBUG --- http response --- 06:58:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:58:09 DEBUG ----- output ----- 06:58:09 DEBUG { 06:58:09 DEBUG "status": "success", 06:58:09 DEBUG "data": { 06:58:09 DEBUG "resultType": "vector", 06:58:09 DEBUG "result": [ 06:58:09 DEBUG { 06:58:09 DEBUG "metric": { 06:58:09 DEBUG "instance": "10.142.0.107:9100" 06:58:09 DEBUG }, 06:58:09 DEBUG "value": [ 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG [...] 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG "instance": "10.142.0.43:9100" 06:58:09 DEBUG }, 06:58:09 DEBUG "value": [ 06:58:09 DEBUG 1713592689, 06:58:09 DEBUG "80771565.83333333" 06:58:09 DEBUG ] 06:58:09 DEBUG } 06:58:09 DEBUG ] 06:58:09 DEBUG } 06:58:09 DEBUG } 06:58:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592689" 06:58:09 INFO [http_cmd]: http status code OK 06:58:09 DEBUG --- status code --- 06:58:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:58:09 DEBUG --- http response --- 06:58:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:58:09 DEBUG ----- output ----- 06:58:09 DEBUG { 06:58:09 DEBUG "status": "success", 06:58:09 DEBUG "data": { 06:58:09 DEBUG "resultType": "vector", 06:58:09 DEBUG "result": [ 06:58:09 DEBUG { 06:58:09 DEBUG "metric": { 06:58:09 DEBUG "instance": "10.142.0.107:9100" 06:58:09 DEBUG }, 06:58:09 DEBUG "value": [ 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG [...] 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG 06:58:09 DEBUG "instance": "10.142.0.43:9100" 06:58:09 DEBUG }, 06:58:09 DEBUG "value": [ 06:58:09 DEBUG 1713592689, 06:58:09 DEBUG "0.0796666666666643" 06:58:09 DEBUG ] 06:58:09 DEBUG } 06:58:09 DEBUG ] 06:58:09 DEBUG } 06:58:09 DEBUG } 06:58:10 INFO 06:58:10 INFO [loop_until]: kubectl --namespace=xlou top node 06:58:10 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:58:10 INFO [loop_until]: OK (rc = 0) 06:58:10 DEBUG --- stdout --- 06:58:10 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 70m 0% 6277Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 8046m 26% 7452Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 17110m 57% 11360Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 83m 0% 3380Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 19109m 64% 13600Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 158m 0% 5194Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 17334m 58% 11614Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 18971m 63% 3020Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 19490m 65% 10865Mi 9% gke-xlou-cdm-ds-0f601751-j488 8194m 27% 7038Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 17156m 57% 15609Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 8052m 26% 7684Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 18923m 63% 12891Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 8165m 51% 8494Mi 14% 06:58:10 DEBUG --- stderr --- 06:58:10 DEBUG 06:58:10 INFO 06:58:10 INFO [loop_until]: kubectl --namespace=xlou top pods 06:58:10 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:58:10 INFO [loop_until]: OK (rc = 0) 06:58:10 DEBUG --- stdout --- 06:58:10 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 17272m 8651Mi am-767f6b67b5-qcqz2 17234m 8716Mi am-767f6b67b5-vbltz 18929m 8652Mi ds-cts-0 19594m 10448Mi ds-cts-1 17587m 10684Mi ds-cts-2 19106m 10283Mi ds-idrepo-0 7731m 3787Mi ds-idrepo-1 7569m 3802Mi ds-idrepo-2 7688m 3712Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1177Mi idm-b85f9d978-wnhxb 3m 1237Mi lodemon-7cb5b9456d-dskfn 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7741m 4680Mi 06:58:10 DEBUG --- stderr --- 06:58:10 DEBUG 06:59:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592749" 06:59:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592749" 06:59:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713592749" 06:59:09 INFO [http_cmd]: http status code OK 06:59:09 INFO [http_cmd]: http status code OK 06:59:09 DEBUG --- status code --- 06:59:09 DEBUG --- status code --- 06:59:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:59:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:59:09 DEBUG --- http response --- 06:59:09 DEBUG --- http response --- 06:59:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:59:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 06:59:09 DEBUG ----- output ----- 06:59:09 DEBUG ----- output ----- 06:59:09 DEBUG { 06:59:09 DEBUG { 06:59:09 DEBUG "status": "success", 06:59:09 DEBUG "status": "success", 06:59:09 DEBUG "data": { 06:59:09 DEBUG "data": { 06:59:09 DEBUG "resultType": "vector", 06:59:09 DEBUG "resultType": "vector", 06:59:09 DEBUG "result": [ 06:59:09 DEBUG "result": [ 06:59:09 DEBUG { 06:59:09 DEBUG { 06:59:09 DEBUG "metric": { 06:59:09 DEBUG "metric": { 06:59:09 DEBUG "instance": "10.142.0.107:9100" 06:59:09 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:59:09 DEBUG }, 06:59:09 DEBUG }, 06:59:09 DEBUG "value": [ 06:59:09 DEBUG "value": [ 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG [...] 06:59:09 DEBUG [...] 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG "instance": "10.142.0.43:9100" 06:59:09 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 06:59:09 DEBUG }, 06:59:09 DEBUG }, 06:59:09 DEBUG "value": [ 06:59:09 DEBUG "value": [ 06:59:09 DEBUG 1713592749, 06:59:09 DEBUG 1713592749, 06:59:09 DEBUG "3262361861120" 06:59:09 DEBUG "0" 06:59:09 DEBUG ] 06:59:09 DEBUG ] 06:59:09 DEBUG } 06:59:09 DEBUG } 06:59:09 DEBUG ] 06:59:09 DEBUG ] 06:59:09 DEBUG } 06:59:09 DEBUG } 06:59:09 DEBUG } 06:59:09 DEBUG } 06:59:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592749" 06:59:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592749" 06:59:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592749" 06:59:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592749" 06:59:09 ERROR [http_cmd]: ERROR 06:59:09 ERROR --- status code --- 06:59:09 ERROR http status code is 400 (expected [200, 401, 403, 404]) 06:59:09 ERROR --- http response --- 06:59:09 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 06:59:09 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 06:59:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713592749" 06:59:09 INFO [http_cmd]: http status code OK 06:59:09 DEBUG --- status code --- 06:59:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:59:09 DEBUG --- http response --- 06:59:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:59:09 DEBUG ----- output ----- 06:59:09 DEBUG { 06:59:09 DEBUG "status": "success", 06:59:09 DEBUG "data": { 06:59:09 DEBUG "resultType": "vector", 06:59:09 DEBUG "result": [ 06:59:09 DEBUG { 06:59:09 DEBUG "metric": { 06:59:09 DEBUG "pod": "ds-idrepo-0" 06:59:09 DEBUG }, 06:59:09 DEBUG "value": [ 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG [...] 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG "pod": "ds-idrepo-2" 06:59:09 DEBUG }, 06:59:09 DEBUG "value": [ 06:59:09 DEBUG 1713592749, 06:59:09 DEBUG "0" 06:59:09 DEBUG ] 06:59:09 DEBUG } 06:59:09 DEBUG ] 06:59:09 DEBUG } 06:59:09 DEBUG } 06:59:09 INFO [http_cmd]: http status code OK 06:59:09 DEBUG --- status code --- 06:59:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:59:09 DEBUG --- http response --- 06:59:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:59:09 DEBUG ----- output ----- 06:59:09 DEBUG { 06:59:09 DEBUG "status": "success", 06:59:09 DEBUG "data": { 06:59:09 DEBUG "resultType": "vector", 06:59:09 DEBUG "result": [ 06:59:09 DEBUG { 06:59:09 DEBUG "metric": { 06:59:09 DEBUG "pod": "ds-idrepo-0" 06:59:09 DEBUG }, 06:59:09 DEBUG "value": [ 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG [...] 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG "pod": "ds-idrepo-2" 06:59:09 DEBUG }, 06:59:09 DEBUG "value": [ 06:59:09 DEBUG 1713592749, 06:59:09 DEBUG "0" 06:59:09 DEBUG ] 06:59:09 DEBUG } 06:59:09 DEBUG ] 06:59:09 DEBUG } 06:59:09 DEBUG } 06:59:09 INFO [http_cmd]: http status code OK 06:59:09 INFO [http_cmd]: http status code OK 06:59:09 DEBUG --- status code --- 06:59:09 DEBUG --- status code --- 06:59:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:59:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:59:09 DEBUG --- http response --- 06:59:09 DEBUG --- http response --- 06:59:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:59:09 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 06:59:09 DEBUG ----- output ----- 06:59:09 DEBUG { 06:59:09 DEBUG "status": "success", 06:59:09 DEBUG "data": { 06:59:09 DEBUG "resultType": "vector", 06:59:09 DEBUG "result": [ 06:59:09 DEBUG { 06:59:09 DEBUG "metric": { 06:59:09 DEBUG "pod": "am-767f6b67b5-9zw22" 06:59:09 DEBUG }, 06:59:09 DEBUG "value": [ 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG [...] 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG "pod": "am-767f6b67b5-vbltz" 06:59:09 DEBUG }, 06:59:09 DEBUG "value": [ 06:59:09 DEBUG 1713592749, 06:59:09 DEBUG "480682.5" 06:59:09 DEBUG ] 06:59:09 DEBUG } 06:59:09 DEBUG ] 06:59:09 DEBUG } 06:59:09 DEBUG } 06:59:09 INFO [http_cmd]: http status code OK 06:59:09 DEBUG --- status code --- 06:59:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:59:09 DEBUG --- http response --- 06:59:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:59:09 DEBUG ----- output ----- 06:59:09 DEBUG { 06:59:09 DEBUG "status": "success", 06:59:09 DEBUG "data": { 06:59:09 DEBUG "resultType": "vector", 06:59:09 DEBUG "result": [ 06:59:09 DEBUG { 06:59:09 DEBUG "metric": { 06:59:09 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 06:59:09 DEBUG }, 06:59:09 DEBUG "value": [ 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG [...] 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 06:59:09 DEBUG }, 06:59:09 DEBUG "value": [ 06:59:09 DEBUG 1713592749, 06:59:09 DEBUG "5003608064" 06:59:09 DEBUG ] 06:59:09 DEBUG } 06:59:09 DEBUG ] 06:59:09 DEBUG } 06:59:09 DEBUG } 06:59:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592749" 06:59:09 INFO [http_cmd]: http status code OK 06:59:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592749" 06:59:09 DEBUG --- status code --- 06:59:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:59:09 DEBUG --- http response --- 06:59:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:59:09 DEBUG ----- output ----- 06:59:09 DEBUG { 06:59:09 DEBUG "status": "success", 06:59:09 DEBUG "data": { 06:59:09 DEBUG "resultType": "vector", 06:59:09 DEBUG "result": [ 06:59:09 DEBUG { 06:59:09 DEBUG "metric": { 06:59:09 DEBUG "pod": "ds-cts-0" 06:59:09 DEBUG }, 06:59:09 DEBUG "value": [ 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG [...] 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG "pod": "ds-cts-2" 06:59:09 DEBUG }, 06:59:09 DEBUG "value": [ 06:59:09 DEBUG 1713592749, 06:59:09 DEBUG "29.9" 06:59:09 DEBUG ] 06:59:09 DEBUG } 06:59:09 DEBUG ] 06:59:09 DEBUG } 06:59:09 DEBUG } 06:59:09 INFO [http_cmd]: http status code OK 06:59:09 DEBUG --- status code --- 06:59:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:59:09 DEBUG --- http response --- 06:59:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 06:59:09 DEBUG ----- output ----- 06:59:09 DEBUG { 06:59:09 DEBUG "status": "success", 06:59:09 DEBUG "data": { 06:59:09 DEBUG "resultType": "vector", 06:59:09 DEBUG "result": [ 06:59:09 DEBUG { 06:59:09 DEBUG "metric": { 06:59:09 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:59:09 DEBUG }, 06:59:09 DEBUG "value": [ 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG [...] 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 06:59:09 DEBUG }, 06:59:09 DEBUG "value": [ 06:59:09 DEBUG 1713592749, 06:59:09 DEBUG "0" 06:59:09 DEBUG ] 06:59:09 DEBUG } 06:59:09 DEBUG ] 06:59:09 DEBUG } 06:59:09 DEBUG } 06:59:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592749" 06:59:09 INFO [http_cmd]: http status code OK 06:59:09 DEBUG --- status code --- 06:59:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:59:09 DEBUG --- http response --- 06:59:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:59:09 DEBUG ----- output ----- 06:59:09 DEBUG { 06:59:09 DEBUG "status": "success", 06:59:09 DEBUG "data": { 06:59:09 DEBUG "resultType": "vector", 06:59:09 DEBUG "result": [ 06:59:09 DEBUG { 06:59:09 DEBUG "metric": { 06:59:09 DEBUG "pod": "ds-idrepo-0" 06:59:09 DEBUG }, 06:59:09 DEBUG "value": [ 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG [...] 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG "pod": "ds-idrepo-2" 06:59:09 DEBUG }, 06:59:09 DEBUG "value": [ 06:59:09 DEBUG 1713592749, 06:59:09 DEBUG "0" 06:59:09 DEBUG ] 06:59:09 DEBUG } 06:59:09 DEBUG ] 06:59:09 DEBUG } 06:59:09 DEBUG } 06:59:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592749" 06:59:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592749" 06:59:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713592749" 06:59:09 INFO [http_cmd]: http status code OK 06:59:09 DEBUG --- status code --- 06:59:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:59:09 DEBUG --- http response --- 06:59:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:59:09 DEBUG ----- output ----- 06:59:09 DEBUG { 06:59:09 DEBUG "status": "success", 06:59:09 DEBUG "data": { 06:59:09 DEBUG "resultType": "vector", 06:59:09 DEBUG "result": [ 06:59:09 DEBUG { 06:59:09 DEBUG "metric": { 06:59:09 DEBUG "pod": "am-767f6b67b5-9zw22" 06:59:09 DEBUG }, 06:59:09 DEBUG "value": [ 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG [...] 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG "pod": "am-767f6b67b5-vbltz" 06:59:09 DEBUG }, 06:59:09 DEBUG "value": [ 06:59:09 DEBUG 1713592749, 06:59:09 DEBUG "2672.233333333333" 06:59:09 DEBUG ] 06:59:09 DEBUG } 06:59:09 DEBUG ] 06:59:09 DEBUG } 06:59:09 DEBUG } 06:59:09 INFO [http_cmd]: http status code OK 06:59:09 INFO [http_cmd]: http status code OK 06:59:09 DEBUG --- status code --- 06:59:09 DEBUG --- status code --- 06:59:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:59:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:59:09 DEBUG --- http response --- 06:59:09 DEBUG --- http response --- 06:59:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:59:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:59:09 DEBUG ----- output ----- 06:59:09 DEBUG ----- output ----- 06:59:09 DEBUG { 06:59:09 DEBUG { 06:59:09 DEBUG "status": "success", 06:59:09 DEBUG "status": "success", 06:59:09 DEBUG "data": { 06:59:09 DEBUG "data": { 06:59:09 DEBUG "resultType": "vector", 06:59:09 DEBUG "resultType": "vector", 06:59:09 DEBUG "result": [ 06:59:09 DEBUG "result": [ 06:59:09 DEBUG { 06:59:09 DEBUG { 06:59:09 DEBUG "metric": { 06:59:09 DEBUG "metric": { 06:59:09 DEBUG "pod": "ds-cts-0" 06:59:09 DEBUG "pod": "ds-idrepo-0" 06:59:09 DEBUG }, 06:59:09 DEBUG }, 06:59:09 DEBUG "value": [ 06:59:09 DEBUG "value": [ 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG [...] 06:59:09 DEBUG [...] 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG "pod": "ds-cts-2" 06:59:09 DEBUG "pod": "ds-cts-2" 06:59:09 DEBUG }, 06:59:09 DEBUG }, 06:59:09 DEBUG "value": [ 06:59:09 DEBUG "value": [ 06:59:09 DEBUG 1713592749, 06:59:09 DEBUG 1713592749, 06:59:09 DEBUG "0" 06:59:09 DEBUG "0" 06:59:09 DEBUG ] 06:59:09 DEBUG ] 06:59:09 DEBUG } 06:59:09 DEBUG } 06:59:09 DEBUG ] 06:59:09 DEBUG ] 06:59:09 DEBUG } 06:59:09 DEBUG } 06:59:09 DEBUG } 06:59:09 DEBUG } 06:59:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592749" 06:59:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592749" 06:59:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592749" 06:59:09 INFO [http_cmd]: http status code OK 06:59:09 DEBUG --- status code --- 06:59:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:59:09 DEBUG --- http response --- 06:59:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:59:09 DEBUG ----- output ----- 06:59:09 DEBUG { 06:59:09 DEBUG "status": "success", 06:59:09 DEBUG "data": { 06:59:09 DEBUG "resultType": "vector", 06:59:09 DEBUG "result": [ 06:59:09 DEBUG { 06:59:09 DEBUG "metric": { 06:59:09 DEBUG "instance": "10.142.0.107:9100" 06:59:09 DEBUG }, 06:59:09 DEBUG "value": [ 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG [...] 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG "instance": "10.142.0.43:9100" 06:59:09 DEBUG }, 06:59:09 DEBUG "value": [ 06:59:09 DEBUG 1713592749, 06:59:09 DEBUG "80903299.33333334" 06:59:09 DEBUG ] 06:59:09 DEBUG } 06:59:09 DEBUG ] 06:59:09 DEBUG } 06:59:09 DEBUG } 06:59:09 INFO [http_cmd]: http status code OK 06:59:09 DEBUG --- status code --- 06:59:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:59:09 DEBUG --- http response --- 06:59:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:59:09 DEBUG ----- output ----- 06:59:09 DEBUG { 06:59:09 DEBUG "status": "success", 06:59:09 DEBUG "data": { 06:59:09 DEBUG "resultType": "vector", 06:59:09 DEBUG "result": [ 06:59:09 DEBUG { 06:59:09 DEBUG "metric": { 06:59:09 DEBUG "pod": "am-767f6b67b5-9zw22" 06:59:09 DEBUG }, 06:59:09 DEBUG "value": [ 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG [...] 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG "pod": "am-767f6b67b5-vbltz" 06:59:09 DEBUG }, 06:59:09 DEBUG "value": [ 06:59:09 DEBUG 1713592749, 06:59:09 DEBUG "2660.7" 06:59:09 DEBUG ] 06:59:09 DEBUG } 06:59:09 DEBUG ] 06:59:09 DEBUG } 06:59:09 DEBUG } 06:59:09 INFO [http_cmd]: http status code OK 06:59:09 DEBUG --- status code --- 06:59:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:59:09 DEBUG --- http response --- 06:59:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:59:09 DEBUG ----- output ----- 06:59:09 DEBUG { 06:59:09 DEBUG "status": "success", 06:59:09 DEBUG "data": { 06:59:09 DEBUG "resultType": "vector", 06:59:09 DEBUG "result": [ 06:59:09 DEBUG { 06:59:09 DEBUG "metric": { 06:59:09 DEBUG "instance": "10.142.0.107:9100" 06:59:09 DEBUG }, 06:59:09 DEBUG "value": [ 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG [...] 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG "instance": "10.142.0.43:9100" 06:59:09 DEBUG }, 06:59:09 DEBUG "value": [ 06:59:09 DEBUG 1713592749, 06:59:09 DEBUG "0.2306333333333015" 06:59:09 DEBUG ] 06:59:09 DEBUG } 06:59:09 DEBUG ] 06:59:09 DEBUG } 06:59:09 DEBUG } 06:59:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592749" 06:59:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592749" 06:59:09 INFO [http_cmd]: http status code OK 06:59:09 DEBUG --- status code --- 06:59:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:59:09 DEBUG --- http response --- 06:59:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:59:09 DEBUG ----- output ----- 06:59:09 DEBUG { 06:59:09 DEBUG "status": "success", 06:59:09 DEBUG "data": { 06:59:09 DEBUG "resultType": "vector", 06:59:09 DEBUG "result": [ 06:59:09 DEBUG { 06:59:09 DEBUG "metric": { 06:59:09 DEBUG "pod": "am-767f6b67b5-9zw22" 06:59:09 DEBUG }, 06:59:09 DEBUG "value": [ 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG [...] 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG "pod": "am-767f6b67b5-vbltz" 06:59:09 DEBUG }, 06:59:09 DEBUG "value": [ 06:59:09 DEBUG 1713592749, 06:59:09 DEBUG "0" 06:59:09 DEBUG ] 06:59:09 DEBUG } 06:59:09 DEBUG ] 06:59:09 DEBUG } 06:59:09 DEBUG } 06:59:09 INFO [http_cmd]: http status code OK 06:59:09 DEBUG --- status code --- 06:59:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:59:09 DEBUG --- http response --- 06:59:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:59:09 DEBUG ----- output ----- 06:59:09 DEBUG { 06:59:09 DEBUG "status": "success", 06:59:09 DEBUG "data": { 06:59:09 DEBUG "resultType": "vector", 06:59:09 DEBUG "result": [ 06:59:09 DEBUG { 06:59:09 DEBUG "metric": { 06:59:09 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:59:09 DEBUG }, 06:59:09 DEBUG "value": [ 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG [...] 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG "pod": "ds-idrepo-1" 06:59:09 DEBUG }, 06:59:09 DEBUG "value": [ 06:59:09 DEBUG 1713592749, 06:59:09 DEBUG "0" 06:59:09 DEBUG ] 06:59:09 DEBUG } 06:59:09 DEBUG ] 06:59:09 DEBUG } 06:59:09 DEBUG } 06:59:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592749" 06:59:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592749" 06:59:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713592749" 06:59:09 INFO [http_cmd]: http status code OK 06:59:09 INFO [http_cmd]: http status code OK 06:59:09 DEBUG --- status code --- 06:59:09 DEBUG --- status code --- 06:59:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:59:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:59:09 DEBUG --- http response --- 06:59:09 DEBUG --- http response --- 06:59:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 06:59:09 DEBUG ----- output ----- 06:59:09 DEBUG { 06:59:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:59:09 DEBUG "status": "success", 06:59:09 DEBUG ----- output ----- 06:59:09 DEBUG "data": { 06:59:09 DEBUG { 06:59:09 DEBUG "resultType": "vector", 06:59:09 DEBUG "status": "success", 06:59:09 DEBUG "result": [ 06:59:09 DEBUG "data": { 06:59:09 DEBUG { 06:59:09 DEBUG "resultType": "vector", 06:59:09 DEBUG "metric": { 06:59:09 DEBUG "result": [ 06:59:09 DEBUG "pod": "ds-cts-0" 06:59:09 DEBUG { 06:59:09 DEBUG }, 06:59:09 DEBUG "metric": { 06:59:09 DEBUG "value": [ 06:59:09 DEBUG "pod": "am-767f6b67b5-9zw22" 06:59:09 DEBUG 06:59:09 DEBUG }, 06:59:09 DEBUG 06:59:09 DEBUG "value": [ 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG [...] 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG [...] 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG "pod": "ds-cts-2" 06:59:09 DEBUG 06:59:09 DEBUG }, 06:59:09 DEBUG 06:59:09 DEBUG "value": [ 06:59:09 DEBUG "pod": "am-767f6b67b5-vbltz" 06:59:09 DEBUG 1713592749, 06:59:09 DEBUG }, 06:59:09 DEBUG "0.0009666666666666666" 06:59:09 DEBUG "value": [ 06:59:09 DEBUG ] 06:59:09 DEBUG 1713592749, 06:59:09 DEBUG } 06:59:09 DEBUG "2672.733333333333" 06:59:09 DEBUG ] 06:59:09 DEBUG ] 06:59:09 DEBUG } 06:59:09 DEBUG } 06:59:09 DEBUG } 06:59:09 DEBUG ] 06:59:09 DEBUG } 06:59:09 DEBUG } 06:59:09 INFO [http_cmd]: http status code OK 06:59:09 DEBUG --- status code --- 06:59:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:59:09 DEBUG --- http response --- 06:59:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:59:09 DEBUG ----- output ----- 06:59:09 DEBUG { 06:59:09 DEBUG "status": "success", 06:59:09 DEBUG "data": { 06:59:09 DEBUG "resultType": "vector", 06:59:09 DEBUG "result": [ 06:59:09 DEBUG { 06:59:09 DEBUG "metric": { 06:59:09 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 06:59:09 DEBUG }, 06:59:09 DEBUG "value": [ 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG [...] 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 06:59:09 DEBUG }, 06:59:09 DEBUG "value": [ 06:59:09 DEBUG 1713592749, 06:59:09 DEBUG "8.037938417310013" 06:59:09 DEBUG ] 06:59:09 DEBUG } 06:59:09 DEBUG ] 06:59:09 DEBUG } 06:59:09 DEBUG } 06:59:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713592749" 06:59:09 INFO [http_cmd]: http status code OK 06:59:09 DEBUG --- status code --- 06:59:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:59:09 DEBUG --- http response --- 06:59:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:59:09 DEBUG ----- output ----- 06:59:09 DEBUG { 06:59:09 DEBUG "status": "success", 06:59:09 DEBUG "data": { 06:59:09 DEBUG "resultType": "vector", 06:59:09 DEBUG "result": [ 06:59:09 DEBUG { 06:59:09 DEBUG "metric": { 06:59:09 DEBUG "pod": "am-767f6b67b5-9zw22" 06:59:09 DEBUG }, 06:59:09 DEBUG "value": [ 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG [...] 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG "pod": "am-767f6b67b5-vbltz" 06:59:09 DEBUG }, 06:59:09 DEBUG "value": [ 06:59:09 DEBUG 1713592749, 06:59:09 DEBUG "0.097" 06:59:09 DEBUG ] 06:59:09 DEBUG } 06:59:09 DEBUG ] 06:59:09 DEBUG } 06:59:09 DEBUG } 06:59:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592749" 06:59:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592749" 06:59:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592749" 06:59:09 INFO [http_cmd]: http status code OK 06:59:09 DEBUG --- status code --- 06:59:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:59:09 DEBUG --- http response --- 06:59:09 INFO [http_cmd]: http status code OK 06:59:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 06:59:09 DEBUG --- status code --- 06:59:09 DEBUG ----- output ----- 06:59:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:59:09 DEBUG { 06:59:09 DEBUG --- http response --- 06:59:09 DEBUG "status": "success", 06:59:09 DEBUG "data": { 06:59:09 DEBUG "resultType": "vector", 06:59:09 DEBUG "result": [ 06:59:09 DEBUG { 06:59:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:59:09 DEBUG "metric": { 06:59:09 DEBUG ----- output ----- 06:59:09 DEBUG "pod": "am-767f6b67b5-9zw22" 06:59:09 DEBUG { 06:59:09 DEBUG }, 06:59:09 DEBUG "status": "success", 06:59:09 DEBUG "value": [ 06:59:09 DEBUG "data": { 06:59:09 DEBUG 06:59:09 DEBUG "resultType": "vector", 06:59:09 DEBUG 06:59:09 DEBUG "result": [ 06:59:09 DEBUG 06:59:09 DEBUG { 06:59:09 DEBUG [...] 06:59:09 DEBUG "metric": { 06:59:09 DEBUG 06:59:09 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 06:59:09 DEBUG 06:59:09 DEBUG }, 06:59:09 DEBUG 06:59:09 DEBUG "value": [ 06:59:09 DEBUG "pod": "am-767f6b67b5-vbltz" 06:59:09 DEBUG 06:59:09 DEBUG }, 06:59:09 DEBUG 06:59:09 DEBUG "value": [ 06:59:09 DEBUG 06:59:09 DEBUG 1713592749, 06:59:09 DEBUG [...] 06:59:09 DEBUG "2672.4" 06:59:09 DEBUG 06:59:09 DEBUG ] 06:59:09 DEBUG 06:59:09 DEBUG } 06:59:09 DEBUG 06:59:09 DEBUG ] 06:59:09 DEBUG "pod": "ds-idrepo-1" 06:59:09 DEBUG } 06:59:09 DEBUG }, 06:59:09 DEBUG } 06:59:09 DEBUG "value": [ 06:59:09 DEBUG 1713592749, 06:59:09 DEBUG "0" 06:59:09 DEBUG ] 06:59:09 DEBUG } 06:59:09 DEBUG ] 06:59:09 DEBUG } 06:59:09 DEBUG } 06:59:09 INFO [http_cmd]: http status code OK 06:59:09 DEBUG --- status code --- 06:59:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:59:09 DEBUG --- http response --- 06:59:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:59:09 DEBUG ----- output ----- 06:59:09 DEBUG { 06:59:09 DEBUG "status": "success", 06:59:09 DEBUG "data": { 06:59:09 DEBUG "resultType": "vector", 06:59:09 DEBUG "result": [ 06:59:09 DEBUG { 06:59:09 DEBUG "metric": { 06:59:09 DEBUG "instance": "10.142.0.107:9100" 06:59:09 DEBUG }, 06:59:09 DEBUG "value": [ 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG [...] 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG "instance": "10.142.0.43:9100" 06:59:09 DEBUG }, 06:59:09 DEBUG "value": [ 06:59:09 DEBUG 1713592749, 06:59:09 DEBUG "80903882.19999999" 06:59:09 DEBUG ] 06:59:09 DEBUG } 06:59:09 DEBUG ] 06:59:09 DEBUG } 06:59:09 DEBUG } 06:59:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592749" 06:59:09 INFO [http_cmd]: http status code OK 06:59:09 DEBUG --- status code --- 06:59:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 06:59:09 DEBUG --- http response --- 06:59:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 06:59:09 DEBUG ----- output ----- 06:59:09 DEBUG { 06:59:09 DEBUG "status": "success", 06:59:09 DEBUG "data": { 06:59:09 DEBUG "resultType": "vector", 06:59:09 DEBUG "result": [ 06:59:09 DEBUG { 06:59:09 DEBUG "metric": { 06:59:09 DEBUG "instance": "10.142.0.107:9100" 06:59:09 DEBUG }, 06:59:09 DEBUG "value": [ 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG [...] 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG 06:59:09 DEBUG "instance": "10.142.0.43:9100" 06:59:09 DEBUG }, 06:59:09 DEBUG "value": [ 06:59:09 DEBUG 1713592749, 06:59:09 DEBUG "0.09199999999999588" 06:59:09 DEBUG ] 06:59:09 DEBUG } 06:59:09 DEBUG ] 06:59:09 DEBUG } 06:59:09 DEBUG } 06:59:10 INFO 06:59:10 INFO [loop_until]: kubectl --namespace=xlou top node 06:59:10 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:59:10 INFO [loop_until]: OK (rc = 0) 06:59:10 DEBUG --- stdout --- 06:59:10 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 69m 0% 6275Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 8012m 26% 7457Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 17093m 57% 11357Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 85m 0% 3366Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 18403m 61% 13605Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 153m 0% 5196Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 17492m 58% 11588Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 19080m 63% 3002Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 19422m 65% 10868Mi 9% gke-xlou-cdm-ds-0f601751-j488 8173m 27% 7043Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 17247m 57% 15601Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 8106m 27% 7693Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 19294m 64% 12875Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 8176m 51% 8494Mi 14% 06:59:10 DEBUG --- stderr --- 06:59:10 DEBUG 06:59:10 INFO 06:59:10 INFO [loop_until]: kubectl --namespace=xlou top pods 06:59:10 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 06:59:10 INFO [loop_until]: OK (rc = 0) 06:59:10 DEBUG --- stdout --- 06:59:10 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 18064m 8652Mi am-767f6b67b5-qcqz2 17316m 8713Mi am-767f6b67b5-vbltz 18424m 8653Mi ds-cts-0 18998m 10462Mi ds-cts-1 17174m 10673Mi ds-cts-2 19385m 10273Mi ds-idrepo-0 7655m 3786Mi ds-idrepo-1 7534m 3802Mi ds-idrepo-2 7849m 3711Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1177Mi idm-b85f9d978-wnhxb 4m 1236Mi lodemon-7cb5b9456d-dskfn 5m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 8181m 4683Mi 06:59:10 DEBUG --- stderr --- 06:59:10 DEBUG 07:00:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592809" 07:00:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713592809" 07:00:09 INFO [http_cmd]: http status code OK 07:00:09 DEBUG --- status code --- 07:00:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:00:09 DEBUG --- http response --- 07:00:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 07:00:09 DEBUG ----- output ----- 07:00:09 DEBUG { 07:00:09 DEBUG "status": "success", 07:00:09 DEBUG "data": { 07:00:09 DEBUG "resultType": "vector", 07:00:09 DEBUG "result": [ 07:00:09 DEBUG { 07:00:09 DEBUG "metric": { 07:00:09 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 07:00:09 DEBUG }, 07:00:09 DEBUG "value": [ 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG [...] 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 07:00:09 DEBUG }, 07:00:09 DEBUG "value": [ 07:00:09 DEBUG 1713592809, 07:00:09 DEBUG "0" 07:00:09 DEBUG ] 07:00:09 DEBUG } 07:00:09 DEBUG ] 07:00:09 DEBUG } 07:00:09 DEBUG } 07:00:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592809" 07:00:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592809" 07:00:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592809" 07:00:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592809" 07:00:09 ERROR [http_cmd]: ERROR 07:00:09 INFO [http_cmd]: http status code OK 07:00:09 INFO [http_cmd]: http status code OK 07:00:09 ERROR --- status code --- 07:00:09 DEBUG --- status code --- 07:00:09 DEBUG --- status code --- 07:00:09 ERROR http status code is 400 (expected [200, 401, 403, 404]) 07:00:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:00:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:00:09 ERROR --- http response --- 07:00:09 DEBUG --- http response --- 07:00:09 DEBUG --- http response --- 07:00:09 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 07:00:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:00:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:00:09 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 07:00:09 DEBUG ----- output ----- 07:00:09 DEBUG ----- output ----- 07:00:09 DEBUG { 07:00:09 DEBUG { 07:00:09 DEBUG "status": "success", 07:00:09 DEBUG "status": "success", 07:00:09 DEBUG "data": { 07:00:09 DEBUG "data": { 07:00:09 DEBUG "resultType": "vector", 07:00:09 DEBUG "resultType": "vector", 07:00:09 DEBUG "result": [ 07:00:09 DEBUG "result": [ 07:00:09 DEBUG { 07:00:09 DEBUG { 07:00:09 DEBUG "metric": { 07:00:09 DEBUG "metric": { 07:00:09 DEBUG "instance": "10.142.0.107:9100" 07:00:09 DEBUG "pod": "ds-idrepo-0" 07:00:09 DEBUG }, 07:00:09 DEBUG }, 07:00:09 DEBUG "value": [ 07:00:09 DEBUG "value": [ 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG [...] 07:00:09 DEBUG [...] 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592809" 07:00:09 DEBUG 07:00:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592809" 07:00:09 DEBUG "instance": "10.142.0.43:9100" 07:00:09 DEBUG "pod": "ds-idrepo-2" 07:00:09 DEBUG }, 07:00:09 DEBUG }, 07:00:09 DEBUG "value": [ 07:00:09 DEBUG "value": [ 07:00:09 DEBUG 1713592809, 07:00:09 DEBUG 1713592809, 07:00:09 DEBUG "3266573544448" 07:00:09 DEBUG "0" 07:00:09 DEBUG ] 07:00:09 DEBUG ] 07:00:09 DEBUG } 07:00:09 DEBUG } 07:00:09 DEBUG ] 07:00:09 DEBUG ] 07:00:09 DEBUG } 07:00:09 DEBUG } 07:00:09 DEBUG } 07:00:09 DEBUG } 07:00:09 INFO [http_cmd]: http status code OK 07:00:09 DEBUG --- status code --- 07:00:09 INFO [http_cmd]: http status code OK 07:00:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:00:09 DEBUG --- status code --- 07:00:09 DEBUG --- http response --- 07:00:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713592809" 07:00:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:00:09 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 07:00:09 DEBUG --- http response --- 07:00:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:00:09 DEBUG ----- output ----- 07:00:09 DEBUG { 07:00:09 DEBUG "status": "success", 07:00:09 DEBUG "data": { 07:00:09 DEBUG "resultType": "vector", 07:00:09 DEBUG "result": [ 07:00:09 DEBUG { 07:00:09 DEBUG "metric": { 07:00:09 DEBUG "pod": "ds-idrepo-0" 07:00:09 DEBUG }, 07:00:09 DEBUG "value": [ 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG [...] 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG "pod": "ds-idrepo-2" 07:00:09 DEBUG }, 07:00:09 DEBUG "value": [ 07:00:09 DEBUG 1713592809, 07:00:09 DEBUG "0" 07:00:09 DEBUG ] 07:00:09 DEBUG } 07:00:09 DEBUG ] 07:00:09 DEBUG } 07:00:09 DEBUG } 07:00:09 INFO [http_cmd]: http status code OK 07:00:09 DEBUG --- status code --- 07:00:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:00:09 DEBUG --- http response --- 07:00:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:00:09 DEBUG ----- output ----- 07:00:09 DEBUG { 07:00:09 DEBUG "status": "success", 07:00:09 DEBUG "data": { 07:00:09 DEBUG "resultType": "vector", 07:00:09 DEBUG "result": [ 07:00:09 DEBUG { 07:00:09 DEBUG "metric": { 07:00:09 DEBUG "pod": "am-767f6b67b5-9zw22" 07:00:09 DEBUG }, 07:00:09 DEBUG "value": [ 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG [...] 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG "pod": "am-767f6b67b5-vbltz" 07:00:09 DEBUG }, 07:00:09 DEBUG "value": [ 07:00:09 DEBUG 1713592809, 07:00:09 DEBUG "480681.5" 07:00:09 DEBUG ] 07:00:09 DEBUG } 07:00:09 DEBUG ] 07:00:09 DEBUG } 07:00:09 DEBUG } 07:00:09 INFO [http_cmd]: http status code OK 07:00:09 DEBUG --- status code --- 07:00:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:00:09 DEBUG --- http response --- 07:00:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 07:00:09 DEBUG ----- output ----- 07:00:09 DEBUG { 07:00:09 DEBUG "status": "success", 07:00:09 DEBUG "data": { 07:00:09 DEBUG "resultType": "vector", 07:00:09 DEBUG "result": [ 07:00:09 DEBUG { 07:00:09 DEBUG "metric": { 07:00:09 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 07:00:09 DEBUG }, 07:00:09 DEBUG "value": [ 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG [...] 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 07:00:09 DEBUG }, 07:00:09 DEBUG "value": [ 07:00:09 DEBUG 1713592809, 07:00:09 DEBUG "0" 07:00:09 DEBUG ] 07:00:09 DEBUG } 07:00:09 DEBUG ] 07:00:09 DEBUG } 07:00:09 DEBUG } 07:00:09 INFO [http_cmd]: http status code OK 07:00:09 DEBUG --- status code --- 07:00:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:00:09 DEBUG --- http response --- 07:00:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:00:09 DEBUG ----- output ----- 07:00:09 DEBUG { 07:00:09 DEBUG "status": "success", 07:00:09 DEBUG "data": { 07:00:09 DEBUG "resultType": "vector", 07:00:09 DEBUG "result": [ 07:00:09 DEBUG { 07:00:09 DEBUG "metric": { 07:00:09 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 07:00:09 DEBUG }, 07:00:09 DEBUG "value": [ 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG [...] 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 07:00:09 DEBUG }, 07:00:09 DEBUG "value": [ 07:00:09 DEBUG 1713592809, 07:00:09 DEBUG "5005516800" 07:00:09 DEBUG ] 07:00:09 DEBUG } 07:00:09 DEBUG ] 07:00:09 DEBUG } 07:00:09 DEBUG } 07:00:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592809" 07:00:09 INFO [http_cmd]: http status code OK 07:00:09 DEBUG --- status code --- 07:00:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:00:09 DEBUG --- http response --- 07:00:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 07:00:09 DEBUG ----- output ----- 07:00:09 DEBUG { 07:00:09 DEBUG "status": "success", 07:00:09 DEBUG "data": { 07:00:09 DEBUG "resultType": "vector", 07:00:09 DEBUG "result": [ 07:00:09 DEBUG { 07:00:09 DEBUG "metric": { 07:00:09 DEBUG "pod": "ds-cts-0" 07:00:09 DEBUG }, 07:00:09 DEBUG "value": [ 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG [...] 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG "pod": "ds-cts-2" 07:00:09 DEBUG }, 07:00:09 DEBUG "value": [ 07:00:09 DEBUG 1713592809, 07:00:09 DEBUG "25.666666666666668" 07:00:09 DEBUG ] 07:00:09 DEBUG } 07:00:09 DEBUG ] 07:00:09 DEBUG } 07:00:09 DEBUG } 07:00:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592809" 07:00:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592809" 07:00:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592809" 07:00:09 INFO [http_cmd]: http status code OK 07:00:09 DEBUG --- status code --- 07:00:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:00:09 DEBUG --- http response --- 07:00:09 INFO [http_cmd]: http status code OK 07:00:09 INFO [http_cmd]: http status code OK 07:00:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:00:09 DEBUG --- status code --- 07:00:09 DEBUG --- status code --- 07:00:09 DEBUG ----- output ----- 07:00:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:00:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:00:09 DEBUG { 07:00:09 DEBUG --- http response --- 07:00:09 DEBUG --- http response --- 07:00:09 DEBUG "status": "success", 07:00:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:00:09 DEBUG "data": { 07:00:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:00:09 DEBUG ----- output ----- 07:00:09 DEBUG "resultType": "vector", 07:00:09 DEBUG ----- output ----- 07:00:09 DEBUG { 07:00:09 DEBUG "result": [ 07:00:09 DEBUG { 07:00:09 DEBUG "status": "success", 07:00:09 DEBUG { 07:00:09 DEBUG "status": "success", 07:00:09 DEBUG "data": { 07:00:09 DEBUG "metric": { 07:00:09 DEBUG "data": { 07:00:09 DEBUG "resultType": "vector", 07:00:09 DEBUG "pod": "am-767f6b67b5-9zw22" 07:00:09 DEBUG "resultType": "vector", 07:00:09 DEBUG "result": [ 07:00:09 DEBUG }, 07:00:09 DEBUG "result": [ 07:00:09 DEBUG { 07:00:09 DEBUG "value": [ 07:00:09 DEBUG { 07:00:09 DEBUG "metric": { 07:00:09 DEBUG 07:00:09 DEBUG "metric": { 07:00:09 DEBUG "pod": "am-767f6b67b5-9zw22" 07:00:09 DEBUG 07:00:09 DEBUG "pod": "ds-idrepo-0" 07:00:09 DEBUG }, 07:00:09 DEBUG 07:00:09 DEBUG }, 07:00:09 DEBUG "value": [ 07:00:09 DEBUG [...] 07:00:09 DEBUG "value": [ 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG [...] 07:00:09 DEBUG "pod": "am-767f6b67b5-vbltz" 07:00:09 DEBUG [...] 07:00:09 DEBUG 07:00:09 DEBUG }, 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG "value": [ 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG 1713592809, 07:00:09 DEBUG 07:00:09 DEBUG "pod": "am-767f6b67b5-vbltz" 07:00:09 DEBUG "2669.4" 07:00:09 DEBUG "pod": "ds-idrepo-2" 07:00:09 DEBUG }, 07:00:09 DEBUG ] 07:00:09 DEBUG }, 07:00:09 DEBUG "value": [ 07:00:09 DEBUG } 07:00:09 DEBUG "value": [ 07:00:09 DEBUG 1713592809, 07:00:09 DEBUG ] 07:00:09 DEBUG 1713592809, 07:00:09 DEBUG "2674.5666666666666" 07:00:09 DEBUG } 07:00:09 DEBUG "0" 07:00:09 DEBUG ] 07:00:09 DEBUG } 07:00:09 DEBUG ] 07:00:09 DEBUG } 07:00:09 DEBUG } 07:00:09 DEBUG ] 07:00:09 DEBUG ] 07:00:09 DEBUG } 07:00:09 DEBUG } 07:00:09 DEBUG } 07:00:09 DEBUG } 07:00:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713592809" 07:00:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592809" 07:00:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592809" 07:00:09 INFO [http_cmd]: http status code OK 07:00:09 DEBUG --- status code --- 07:00:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:00:09 DEBUG --- http response --- 07:00:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 07:00:09 DEBUG ----- output ----- 07:00:09 DEBUG { 07:00:09 DEBUG "status": "success", 07:00:09 DEBUG "data": { 07:00:09 DEBUG "resultType": "vector", 07:00:09 DEBUG "result": [ 07:00:09 DEBUG { 07:00:09 DEBUG "metric": { 07:00:09 DEBUG "pod": "ds-idrepo-0" 07:00:09 DEBUG }, 07:00:09 DEBUG "value": [ 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG [...] 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG "pod": "ds-cts-2" 07:00:09 DEBUG }, 07:00:09 DEBUG "value": [ 07:00:09 DEBUG 1713592809, 07:00:09 DEBUG "0" 07:00:09 DEBUG ] 07:00:09 DEBUG } 07:00:09 DEBUG ] 07:00:09 DEBUG } 07:00:09 DEBUG } 07:00:09 INFO [http_cmd]: http status code OK 07:00:09 DEBUG --- status code --- 07:00:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:00:09 DEBUG --- http response --- 07:00:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 07:00:09 DEBUG ----- output ----- 07:00:09 DEBUG { 07:00:09 INFO [http_cmd]: http status code OK 07:00:09 DEBUG "status": "success", 07:00:09 DEBUG --- status code --- 07:00:09 DEBUG "data": { 07:00:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:00:09 DEBUG "resultType": "vector", 07:00:09 DEBUG --- http response --- 07:00:09 DEBUG "result": [ 07:00:09 DEBUG { 07:00:09 DEBUG "metric": { 07:00:09 DEBUG "pod": "ds-cts-0" 07:00:09 DEBUG }, 07:00:09 DEBUG "value": [ 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG [...] 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG "pod": "ds-cts-2" 07:00:09 DEBUG }, 07:00:09 DEBUG "value": [ 07:00:09 DEBUG 1713592809, 07:00:09 DEBUG "0" 07:00:09 DEBUG ] 07:00:09 DEBUG } 07:00:09 DEBUG ] 07:00:09 DEBUG } 07:00:09 DEBUG } 07:00:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:00:09 DEBUG ----- output ----- 07:00:09 DEBUG { 07:00:09 DEBUG "status": "success", 07:00:09 DEBUG "data": { 07:00:09 DEBUG "resultType": "vector", 07:00:09 DEBUG "result": [ 07:00:09 DEBUG { 07:00:09 DEBUG "metric": { 07:00:09 DEBUG "instance": "10.142.0.107:9100" 07:00:09 DEBUG }, 07:00:09 DEBUG "value": [ 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG [...] 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG "instance": "10.142.0.43:9100" 07:00:09 DEBUG }, 07:00:09 DEBUG "value": [ 07:00:09 DEBUG 1713592809, 07:00:09 DEBUG "80654771.63333333" 07:00:09 DEBUG ] 07:00:09 DEBUG } 07:00:09 DEBUG ] 07:00:09 DEBUG } 07:00:09 DEBUG } 07:00:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592809" 07:00:09 INFO [http_cmd]: http status code OK 07:00:09 DEBUG --- status code --- 07:00:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:00:09 DEBUG --- http response --- 07:00:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:00:09 DEBUG ----- output ----- 07:00:09 DEBUG { 07:00:09 DEBUG "status": "success", 07:00:09 DEBUG "data": { 07:00:09 DEBUG "resultType": "vector", 07:00:09 DEBUG "result": [ 07:00:09 DEBUG { 07:00:09 DEBUG "metric": { 07:00:09 DEBUG "instance": "10.142.0.107:9100" 07:00:09 DEBUG }, 07:00:09 DEBUG "value": [ 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG [...] 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG "instance": "10.142.0.43:9100" 07:00:09 DEBUG }, 07:00:09 DEBUG "value": [ 07:00:09 DEBUG 1713592809, 07:00:09 DEBUG "0.17059999999997236" 07:00:09 DEBUG ] 07:00:09 DEBUG } 07:00:09 DEBUG ] 07:00:09 DEBUG } 07:00:09 DEBUG } 07:00:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713592809" 07:00:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592809" 07:00:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592809" 07:00:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713592809" 07:00:09 INFO [http_cmd]: http status code OK 07:00:09 INFO [http_cmd]: http status code OK 07:00:09 DEBUG --- status code --- 07:00:09 DEBUG --- status code --- 07:00:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:00:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:00:09 DEBUG --- http response --- 07:00:09 DEBUG --- http response --- 07:00:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:00:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:00:09 DEBUG ----- output ----- 07:00:09 DEBUG ----- output ----- 07:00:09 DEBUG { 07:00:09 DEBUG { 07:00:09 DEBUG "status": "success", 07:00:09 DEBUG "status": "success", 07:00:09 DEBUG "data": { 07:00:09 DEBUG "data": { 07:00:09 DEBUG "resultType": "vector", 07:00:09 DEBUG "resultType": "vector", 07:00:09 DEBUG "result": [ 07:00:09 DEBUG "result": [ 07:00:09 DEBUG { 07:00:09 DEBUG { 07:00:09 DEBUG "metric": { 07:00:09 INFO [http_cmd]: http status code OK 07:00:09 DEBUG "metric": { 07:00:09 DEBUG "pod": "am-767f6b67b5-9zw22" 07:00:09 DEBUG --- status code --- 07:00:09 DEBUG "pod": "am-767f6b67b5-9zw22" 07:00:09 DEBUG }, 07:00:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:00:09 DEBUG }, 07:00:09 DEBUG "value": [ 07:00:09 DEBUG --- http response --- 07:00:09 DEBUG "value": [ 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG ----- output ----- 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG { 07:00:09 DEBUG [...] 07:00:09 DEBUG [...] 07:00:09 DEBUG "status": "success", 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG "data": { 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG "resultType": "vector", 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG "result": [ 07:00:09 DEBUG "pod": "am-767f6b67b5-vbltz" 07:00:09 DEBUG "pod": "am-767f6b67b5-vbltz" 07:00:09 DEBUG { 07:00:09 DEBUG }, 07:00:09 DEBUG }, 07:00:09 DEBUG "metric": { 07:00:09 DEBUG "value": [ 07:00:09 DEBUG "value": [ 07:00:09 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 07:00:09 DEBUG 1713592809, 07:00:09 DEBUG 1713592809, 07:00:09 DEBUG }, 07:00:09 DEBUG "0" 07:00:09 DEBUG "0.097" 07:00:09 DEBUG "value": [ 07:00:09 DEBUG ] 07:00:09 DEBUG ] 07:00:09 DEBUG 07:00:09 DEBUG } 07:00:09 DEBUG } 07:00:09 DEBUG 07:00:09 DEBUG ] 07:00:09 DEBUG ] 07:00:09 DEBUG 07:00:09 DEBUG } 07:00:09 DEBUG } 07:00:09 DEBUG [...] 07:00:09 DEBUG } 07:00:09 DEBUG } 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG "pod": "lodemon-7cb5b9456d-dskfn" 07:00:09 DEBUG }, 07:00:09 DEBUG "value": [ 07:00:09 DEBUG 1713592809, 07:00:09 DEBUG "0" 07:00:09 DEBUG ] 07:00:09 DEBUG } 07:00:09 DEBUG ] 07:00:09 DEBUG } 07:00:09 DEBUG } 07:00:09 INFO [http_cmd]: http status code OK 07:00:09 DEBUG --- status code --- 07:00:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:00:09 DEBUG --- http response --- 07:00:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:00:09 DEBUG ----- output ----- 07:00:09 DEBUG { 07:00:09 DEBUG "status": "success", 07:00:09 DEBUG "data": { 07:00:09 DEBUG "resultType": "vector", 07:00:09 DEBUG "result": [ 07:00:09 DEBUG { 07:00:09 DEBUG "metric": { 07:00:09 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 07:00:09 DEBUG }, 07:00:09 DEBUG "value": [ 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG [...] 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 07:00:09 DEBUG }, 07:00:09 DEBUG "value": [ 07:00:09 DEBUG 1713592809, 07:00:09 DEBUG "8.030486330993233" 07:00:09 DEBUG ] 07:00:09 DEBUG } 07:00:09 DEBUG ] 07:00:09 DEBUG } 07:00:09 DEBUG } 07:00:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592809" 07:00:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592809" 07:00:09 INFO [http_cmd]: http status code OK 07:00:09 DEBUG --- status code --- 07:00:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:00:09 DEBUG --- http response --- 07:00:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 07:00:09 DEBUG ----- output ----- 07:00:09 DEBUG { 07:00:09 DEBUG "status": "success", 07:00:09 DEBUG "data": { 07:00:09 DEBUG "resultType": "vector", 07:00:09 DEBUG "result": [ 07:00:09 DEBUG { 07:00:09 DEBUG "metric": { 07:00:09 DEBUG "pod": "ds-cts-0" 07:00:09 DEBUG }, 07:00:09 DEBUG "value": [ 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG [...] 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG "pod": "ds-cts-2" 07:00:09 DEBUG }, 07:00:09 DEBUG "value": [ 07:00:09 DEBUG 1713592809, 07:00:09 DEBUG "0.0008280488888888889" 07:00:09 DEBUG ] 07:00:09 DEBUG } 07:00:09 DEBUG ] 07:00:09 DEBUG } 07:00:09 DEBUG } 07:00:09 INFO [http_cmd]: http status code OK 07:00:09 DEBUG --- status code --- 07:00:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:00:09 DEBUG --- http response --- 07:00:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:00:09 DEBUG ----- output ----- 07:00:09 DEBUG { 07:00:09 DEBUG "status": "success", 07:00:09 DEBUG "data": { 07:00:09 DEBUG "resultType": "vector", 07:00:09 DEBUG "result": [ 07:00:09 DEBUG { 07:00:09 DEBUG "metric": { 07:00:09 DEBUG "pod": "am-767f6b67b5-9zw22" 07:00:09 DEBUG }, 07:00:09 DEBUG "value": [ 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG [...] 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG "pod": "am-767f6b67b5-vbltz" 07:00:09 DEBUG }, 07:00:09 DEBUG "value": [ 07:00:09 DEBUG 1713592809, 07:00:09 DEBUG "2668.6" 07:00:09 DEBUG ] 07:00:09 DEBUG } 07:00:09 DEBUG ] 07:00:09 DEBUG } 07:00:09 DEBUG } 07:00:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592809" 07:00:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592809" 07:00:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592809" 07:00:09 INFO [http_cmd]: http status code OK 07:00:09 DEBUG --- status code --- 07:00:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:00:09 DEBUG --- http response --- 07:00:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:00:09 DEBUG ----- output ----- 07:00:09 DEBUG { 07:00:09 DEBUG "status": "success", 07:00:09 DEBUG "data": { 07:00:09 DEBUG "resultType": "vector", 07:00:09 DEBUG "result": [ 07:00:09 DEBUG { 07:00:09 DEBUG "metric": { 07:00:09 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 07:00:09 DEBUG }, 07:00:09 DEBUG "value": [ 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG [...] 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG "pod": "lodemon-7cb5b9456d-dskfn" 07:00:09 DEBUG }, 07:00:09 DEBUG "value": [ 07:00:09 DEBUG 1713592809, 07:00:09 DEBUG "0" 07:00:09 DEBUG ] 07:00:09 DEBUG } 07:00:09 DEBUG ] 07:00:09 DEBUG } 07:00:09 DEBUG } 07:00:09 INFO [http_cmd]: http status code OK 07:00:09 DEBUG --- status code --- 07:00:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:00:09 DEBUG --- http response --- 07:00:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:00:09 DEBUG ----- output ----- 07:00:09 DEBUG { 07:00:09 DEBUG "status": "success", 07:00:09 DEBUG "data": { 07:00:09 DEBUG "resultType": "vector", 07:00:09 DEBUG "result": [ 07:00:09 DEBUG { 07:00:09 DEBUG "metric": { 07:00:09 DEBUG "pod": "am-767f6b67b5-9zw22" 07:00:09 DEBUG }, 07:00:09 DEBUG "value": [ 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG [...] 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG "pod": "am-767f6b67b5-vbltz" 07:00:09 DEBUG }, 07:00:09 DEBUG "value": [ 07:00:09 DEBUG 1713592809, 07:00:09 DEBUG "2669.5666666666666" 07:00:09 DEBUG ] 07:00:09 DEBUG } 07:00:09 DEBUG ] 07:00:09 DEBUG } 07:00:09 DEBUG } 07:00:09 INFO [http_cmd]: http status code OK 07:00:09 DEBUG --- status code --- 07:00:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:00:09 DEBUG --- http response --- 07:00:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:00:09 DEBUG ----- output ----- 07:00:09 DEBUG { 07:00:09 DEBUG "status": "success", 07:00:09 DEBUG "data": { 07:00:09 DEBUG "resultType": "vector", 07:00:09 DEBUG "result": [ 07:00:09 DEBUG { 07:00:09 DEBUG "metric": { 07:00:09 DEBUG "instance": "10.142.0.107:9100" 07:00:09 DEBUG }, 07:00:09 DEBUG "value": [ 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG [...] 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG "instance": "10.142.0.43:9100" 07:00:09 DEBUG }, 07:00:09 DEBUG "value": [ 07:00:09 DEBUG 1713592809, 07:00:09 DEBUG "80655346.36666666" 07:00:09 DEBUG ] 07:00:09 DEBUG } 07:00:09 DEBUG ] 07:00:09 DEBUG } 07:00:09 DEBUG } 07:00:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592809" 07:00:09 INFO [http_cmd]: http status code OK 07:00:09 DEBUG --- status code --- 07:00:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:00:09 DEBUG --- http response --- 07:00:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:00:09 DEBUG ----- output ----- 07:00:09 DEBUG { 07:00:09 DEBUG "status": "success", 07:00:09 DEBUG "data": { 07:00:09 DEBUG "resultType": "vector", 07:00:09 DEBUG "result": [ 07:00:09 DEBUG { 07:00:09 DEBUG "metric": { 07:00:09 DEBUG "instance": "10.142.0.107:9100" 07:00:09 DEBUG }, 07:00:09 DEBUG "value": [ 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG [...] 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG 07:00:09 DEBUG "instance": "10.142.0.43:9100" 07:00:09 DEBUG }, 07:00:09 DEBUG "value": [ 07:00:09 DEBUG 1713592809, 07:00:09 DEBUG "0.07066666666666872" 07:00:09 DEBUG ] 07:00:09 DEBUG } 07:00:09 DEBUG ] 07:00:09 DEBUG } 07:00:09 DEBUG } 07:00:10 INFO 07:00:10 INFO [loop_until]: kubectl --namespace=xlou top node 07:00:10 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 07:00:10 INFO [loop_until]: OK (rc = 0) 07:00:10 DEBUG --- stdout --- 07:00:10 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 71m 0% 6269Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 8021m 26% 7471Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 17247m 57% 11368Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 81m 0% 3376Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 18555m 62% 13604Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 163m 0% 5199Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 16947m 56% 11595Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 19147m 64% 3016Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 19384m 64% 10874Mi 9% gke-xlou-cdm-ds-0f601751-j488 8160m 27% 7030Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 17282m 57% 15597Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 8013m 26% 7694Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 19369m 64% 12881Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 8179m 51% 8490Mi 14% 07:00:10 DEBUG --- stderr --- 07:00:10 DEBUG 07:00:10 INFO 07:00:10 INFO [loop_until]: kubectl --namespace=xlou top pods 07:00:10 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 07:00:11 INFO [loop_until]: OK (rc = 0) 07:00:11 DEBUG --- stdout --- 07:00:11 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 17117m 8652Mi am-767f6b67b5-qcqz2 17250m 8714Mi am-767f6b67b5-vbltz 18568m 8653Mi ds-cts-0 19261m 10457Mi ds-cts-1 17799m 10673Mi ds-cts-2 19760m 10267Mi ds-idrepo-0 7692m 3787Mi ds-idrepo-1 7605m 3802Mi ds-idrepo-2 7658m 3712Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1177Mi idm-b85f9d978-wnhxb 3m 1236Mi lodemon-7cb5b9456d-dskfn 1m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 8044m 4683Mi 07:00:11 DEBUG --- stderr --- 07:00:11 DEBUG 07:01:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592869" 07:01:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592869" 07:01:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592869" 07:01:09 INFO [http_cmd]: http status code OK 07:01:09 INFO [http_cmd]: http status code OK 07:01:09 DEBUG --- status code --- 07:01:09 DEBUG --- status code --- 07:01:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:01:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:01:09 DEBUG --- http response --- 07:01:09 DEBUG --- http response --- 07:01:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:01:09 DEBUG ----- output ----- 07:01:09 DEBUG { 07:01:09 DEBUG "status": "success", 07:01:09 DEBUG "data": { 07:01:09 DEBUG "resultType": "vector", 07:01:09 DEBUG "result": [ 07:01:09 DEBUG { 07:01:09 DEBUG "metric": { 07:01:09 DEBUG "pod": "ds-idrepo-0" 07:01:09 DEBUG }, 07:01:09 DEBUG "value": [ 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG [...] 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG "pod": "ds-idrepo-2" 07:01:09 DEBUG }, 07:01:09 DEBUG "value": [ 07:01:09 DEBUG 1713592869, 07:01:09 DEBUG "0" 07:01:09 DEBUG ] 07:01:09 DEBUG } 07:01:09 DEBUG ] 07:01:09 DEBUG } 07:01:09 DEBUG } 07:01:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 07:01:09 DEBUG ----- output ----- 07:01:09 DEBUG { 07:01:09 DEBUG "status": "success", 07:01:09 DEBUG "data": { 07:01:09 DEBUG "resultType": "vector", 07:01:09 DEBUG "result": [ 07:01:09 DEBUG { 07:01:09 DEBUG "metric": { 07:01:09 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 07:01:09 DEBUG }, 07:01:09 DEBUG "value": [ 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG [...] 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG "pod": "ds-cts-2" 07:01:09 DEBUG }, 07:01:09 DEBUG "value": [ 07:01:09 DEBUG 1713592869, 07:01:09 DEBUG "45163380.14299557" 07:01:09 DEBUG ] 07:01:09 DEBUG } 07:01:09 DEBUG ] 07:01:09 DEBUG } 07:01:09 DEBUG } 07:01:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592869" 07:01:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592869" 07:01:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713592869" 07:01:09 INFO [http_cmd]: http status code OK 07:01:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592869" 07:01:09 DEBUG --- status code --- 07:01:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:01:09 DEBUG --- http response --- 07:01:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 07:01:09 DEBUG ----- output ----- 07:01:09 DEBUG { 07:01:09 DEBUG "status": "success", 07:01:09 DEBUG "data": { 07:01:09 DEBUG "resultType": "vector", 07:01:09 DEBUG "result": [ 07:01:09 DEBUG { 07:01:09 DEBUG "metric": { 07:01:09 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 07:01:09 DEBUG }, 07:01:09 DEBUG "value": [ 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG [...] 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG "pod": "ds-cts-2" 07:01:09 DEBUG }, 07:01:09 DEBUG "value": [ 07:01:09 DEBUG 1713592869, 07:01:09 DEBUG "68236379.95966792" 07:01:09 DEBUG ] 07:01:09 DEBUG } 07:01:09 DEBUG ] 07:01:09 DEBUG } 07:01:09 DEBUG } 07:01:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592869" 07:01:09 INFO [http_cmd]: http status code OK 07:01:09 DEBUG --- status code --- 07:01:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:01:09 DEBUG --- http response --- 07:01:09 INFO [http_cmd]: http status code OK 07:01:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:01:09 DEBUG --- status code --- 07:01:09 DEBUG ----- output ----- 07:01:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:01:09 DEBUG { 07:01:09 DEBUG --- http response --- 07:01:09 DEBUG "status": "success", 07:01:09 DEBUG "data": { 07:01:09 DEBUG "resultType": "vector", 07:01:09 DEBUG "result": [ 07:01:09 DEBUG { 07:01:09 DEBUG "metric": { 07:01:09 DEBUG "pod": "ds-idrepo-0" 07:01:09 DEBUG }, 07:01:09 DEBUG "value": [ 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG [...] 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG "pod": "ds-idrepo-2" 07:01:09 DEBUG }, 07:01:09 DEBUG "value": [ 07:01:09 DEBUG 1713592869, 07:01:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592869" 07:01:09 DEBUG "0" 07:01:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 07:01:09 DEBUG ] 07:01:09 DEBUG ----- output ----- 07:01:09 DEBUG } 07:01:09 DEBUG { 07:01:09 DEBUG ] 07:01:09 DEBUG "status": "success", 07:01:09 DEBUG } 07:01:09 DEBUG "data": { 07:01:09 DEBUG } 07:01:09 DEBUG "resultType": "vector", 07:01:09 DEBUG "result": [ 07:01:09 DEBUG { 07:01:09 DEBUG "metric": { 07:01:09 DEBUG "pod": "ds-cts-0" 07:01:09 DEBUG }, 07:01:09 DEBUG "value": [ 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG [...] 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG "pod": "ds-cts-2" 07:01:09 DEBUG }, 07:01:09 DEBUG "value": [ 07:01:09 DEBUG 1713592869, 07:01:09 DEBUG "31.53333333333333" 07:01:09 DEBUG ] 07:01:09 DEBUG } 07:01:09 DEBUG ] 07:01:09 DEBUG } 07:01:09 DEBUG } 07:01:09 ERROR [http_cmd]: ERROR 07:01:09 ERROR --- status code --- 07:01:09 ERROR http status code is 400 (expected [200, 401, 403, 404]) 07:01:09 ERROR --- http response --- 07:01:09 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 07:01:09 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 07:01:09 INFO [http_cmd]: http status code OK 07:01:09 DEBUG --- status code --- 07:01:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:01:09 DEBUG --- http response --- 07:01:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:01:09 DEBUG ----- output ----- 07:01:09 DEBUG { 07:01:09 DEBUG "status": "success", 07:01:09 DEBUG "data": { 07:01:09 DEBUG "resultType": "vector", 07:01:09 DEBUG "result": [ 07:01:09 DEBUG { 07:01:09 DEBUG "metric": { 07:01:09 DEBUG "instance": "10.142.0.107:9100" 07:01:09 DEBUG }, 07:01:09 DEBUG "value": [ 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG [...] 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG "instance": "10.142.0.43:9100" 07:01:09 DEBUG }, 07:01:09 DEBUG "value": [ 07:01:09 DEBUG 1713592869, 07:01:09 DEBUG "3270583958528" 07:01:09 DEBUG ] 07:01:09 DEBUG } 07:01:09 DEBUG ] 07:01:09 DEBUG } 07:01:09 DEBUG } 07:01:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713592869" 07:01:09 INFO [http_cmd]: http status code OK 07:01:09 DEBUG --- status code --- 07:01:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:01:09 DEBUG --- http response --- 07:01:09 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 07:01:09 INFO [http_cmd]: http status code OK 07:01:09 DEBUG --- status code --- 07:01:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:01:09 DEBUG --- http response --- 07:01:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:01:09 DEBUG ----- output ----- 07:01:09 DEBUG { 07:01:09 DEBUG "status": "success", 07:01:09 DEBUG "data": { 07:01:09 DEBUG "resultType": "vector", 07:01:09 DEBUG "result": [ 07:01:09 DEBUG { 07:01:09 DEBUG "metric": { 07:01:09 DEBUG "pod": "am-767f6b67b5-9zw22" 07:01:09 DEBUG }, 07:01:09 DEBUG "value": [ 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG [...] 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG "pod": "am-767f6b67b5-vbltz" 07:01:09 DEBUG }, 07:01:09 DEBUG "value": [ 07:01:09 DEBUG 1713592869, 07:01:09 DEBUG "480785.5" 07:01:09 DEBUG ] 07:01:09 DEBUG } 07:01:09 DEBUG ] 07:01:09 DEBUG } 07:01:09 DEBUG } 07:01:09 INFO [http_cmd]: http status code OK 07:01:09 DEBUG --- status code --- 07:01:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:01:09 DEBUG --- http response --- 07:01:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:01:09 DEBUG ----- output ----- 07:01:09 DEBUG { 07:01:09 DEBUG "status": "success", 07:01:09 DEBUG "data": { 07:01:09 DEBUG "resultType": "vector", 07:01:09 DEBUG "result": [ 07:01:09 DEBUG { 07:01:09 DEBUG "metric": { 07:01:09 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 07:01:09 DEBUG }, 07:01:09 DEBUG "value": [ 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG [...] 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 07:01:09 DEBUG }, 07:01:09 DEBUG "value": [ 07:01:09 DEBUG 1713592869, 07:01:09 DEBUG "5006938112" 07:01:09 DEBUG ] 07:01:09 DEBUG } 07:01:09 DEBUG ] 07:01:09 DEBUG } 07:01:09 DEBUG } 07:01:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592869" 07:01:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592869" 07:01:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713592869" 07:01:09 INFO [http_cmd]: http status code OK 07:01:09 DEBUG --- status code --- 07:01:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:01:09 DEBUG --- http response --- 07:01:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:01:09 DEBUG ----- output ----- 07:01:09 DEBUG { 07:01:09 DEBUG "status": "success", 07:01:09 DEBUG "data": { 07:01:09 DEBUG "resultType": "vector", 07:01:09 DEBUG "result": [ 07:01:09 DEBUG { 07:01:09 DEBUG "metric": { 07:01:09 DEBUG "pod": "am-767f6b67b5-9zw22" 07:01:09 DEBUG }, 07:01:09 DEBUG "value": [ 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG [...] 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG "pod": "am-767f6b67b5-vbltz" 07:01:09 DEBUG }, 07:01:09 DEBUG "value": [ 07:01:09 DEBUG 1713592869, 07:01:09 DEBUG "2665.733333333333" 07:01:09 DEBUG ] 07:01:09 DEBUG } 07:01:09 DEBUG ] 07:01:09 DEBUG } 07:01:09 DEBUG } 07:01:09 INFO [http_cmd]: http status code OK 07:01:09 INFO [http_cmd]: http status code OK 07:01:09 DEBUG --- status code --- 07:01:09 DEBUG --- status code --- 07:01:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:01:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:01:09 DEBUG --- http response --- 07:01:09 DEBUG --- http response --- 07:01:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:01:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 07:01:09 DEBUG ----- output ----- 07:01:09 DEBUG ----- output ----- 07:01:09 DEBUG { 07:01:09 DEBUG { 07:01:09 DEBUG "status": "success", 07:01:09 DEBUG "status": "success", 07:01:09 DEBUG "data": { 07:01:09 DEBUG "data": { 07:01:09 DEBUG "resultType": "vector", 07:01:09 DEBUG "resultType": "vector", 07:01:09 DEBUG "result": [ 07:01:09 DEBUG "result": [ 07:01:09 DEBUG { 07:01:09 DEBUG { 07:01:09 DEBUG "metric": { 07:01:09 DEBUG "metric": { 07:01:09 DEBUG "pod": "ds-idrepo-0" 07:01:09 DEBUG "pod": "ds-idrepo-0" 07:01:09 DEBUG }, 07:01:09 DEBUG }, 07:01:09 DEBUG "value": [ 07:01:09 DEBUG "value": [ 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592869" 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG [...] 07:01:09 DEBUG [...] 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG "pod": "ds-idrepo-2" 07:01:09 DEBUG "pod": "ds-cts-2" 07:01:09 DEBUG }, 07:01:09 DEBUG }, 07:01:09 DEBUG "value": [ 07:01:09 DEBUG "value": [ 07:01:09 DEBUG 1713592869, 07:01:09 DEBUG 1713592869, 07:01:09 DEBUG "0" 07:01:09 DEBUG "0" 07:01:09 DEBUG ] 07:01:09 DEBUG ] 07:01:09 DEBUG } 07:01:09 DEBUG } 07:01:09 DEBUG ] 07:01:09 DEBUG ] 07:01:09 DEBUG } 07:01:09 DEBUG } 07:01:09 DEBUG } 07:01:09 DEBUG } 07:01:09 INFO [http_cmd]: http status code OK 07:01:09 DEBUG --- status code --- 07:01:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:01:09 DEBUG --- http response --- 07:01:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:01:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592869" 07:01:09 DEBUG ----- output ----- 07:01:09 DEBUG { 07:01:09 DEBUG "status": "success", 07:01:09 DEBUG "data": { 07:01:09 DEBUG "resultType": "vector", 07:01:09 DEBUG "result": [ 07:01:09 DEBUG { 07:01:09 DEBUG "metric": { 07:01:09 DEBUG "pod": "am-767f6b67b5-9zw22" 07:01:09 DEBUG }, 07:01:09 DEBUG "value": [ 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG [...] 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG "pod": "am-767f6b67b5-vbltz" 07:01:09 DEBUG }, 07:01:09 DEBUG "value": [ 07:01:09 DEBUG 1713592869, 07:01:09 DEBUG "2670.7" 07:01:09 DEBUG ] 07:01:09 DEBUG } 07:01:09 DEBUG ] 07:01:09 DEBUG } 07:01:09 DEBUG } 07:01:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592869" 07:01:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592869" 07:01:09 INFO [http_cmd]: http status code OK 07:01:09 INFO [http_cmd]: http status code OK 07:01:09 DEBUG --- status code --- 07:01:09 DEBUG --- status code --- 07:01:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:01:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:01:09 DEBUG --- http response --- 07:01:09 DEBUG --- http response --- 07:01:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:01:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:01:09 DEBUG ----- output ----- 07:01:09 DEBUG ----- output ----- 07:01:09 DEBUG { 07:01:09 DEBUG { 07:01:09 DEBUG "status": "success", 07:01:09 DEBUG "status": "success", 07:01:09 DEBUG "data": { 07:01:09 DEBUG "data": { 07:01:09 DEBUG "resultType": "vector", 07:01:09 DEBUG "resultType": "vector", 07:01:09 DEBUG "result": [ 07:01:09 DEBUG "result": [ 07:01:09 DEBUG { 07:01:09 DEBUG { 07:01:09 DEBUG "metric": { 07:01:09 DEBUG "metric": { 07:01:09 DEBUG "instance": "10.142.0.107:9100" 07:01:09 DEBUG "instance": "10.142.0.107:9100" 07:01:09 DEBUG }, 07:01:09 DEBUG }, 07:01:09 DEBUG "value": [ 07:01:09 DEBUG "value": [ 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG [...] 07:01:09 DEBUG [...] 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG "instance": "10.142.0.43:9100" 07:01:09 DEBUG "instance": "10.142.0.43:9100" 07:01:09 DEBUG }, 07:01:09 DEBUG }, 07:01:09 DEBUG "value": [ 07:01:09 DEBUG "value": [ 07:01:09 DEBUG 1713592869, 07:01:09 DEBUG 1713592869, 07:01:09 DEBUG "0.22046666666665923" 07:01:09 DEBUG "80868835.03333333" 07:01:09 DEBUG ] 07:01:09 DEBUG ] 07:01:09 DEBUG } 07:01:09 DEBUG } 07:01:09 DEBUG ] 07:01:09 DEBUG ] 07:01:09 DEBUG } 07:01:09 DEBUG } 07:01:09 DEBUG } 07:01:09 DEBUG } 07:01:09 INFO [http_cmd]: http status code OK 07:01:09 DEBUG --- status code --- 07:01:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:01:09 DEBUG --- http response --- 07:01:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 07:01:09 DEBUG ----- output ----- 07:01:09 DEBUG { 07:01:09 DEBUG "status": "success", 07:01:09 DEBUG "data": { 07:01:09 DEBUG "resultType": "vector", 07:01:09 DEBUG "result": [ 07:01:09 DEBUG { 07:01:09 DEBUG "metric": { 07:01:09 DEBUG "pod": "ds-cts-0" 07:01:09 DEBUG }, 07:01:09 DEBUG "value": [ 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG [...] 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG "pod": "ds-cts-2" 07:01:09 DEBUG }, 07:01:09 DEBUG "value": [ 07:01:09 DEBUG 1713592869, 07:01:09 DEBUG "0" 07:01:09 DEBUG ] 07:01:09 DEBUG } 07:01:09 DEBUG ] 07:01:09 DEBUG } 07:01:09 DEBUG } 07:01:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592869" 07:01:09 INFO [http_cmd]: http status code OK 07:01:09 DEBUG --- status code --- 07:01:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:01:09 DEBUG --- http response --- 07:01:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:01:09 DEBUG ----- output ----- 07:01:09 DEBUG { 07:01:09 DEBUG "status": "success", 07:01:09 DEBUG "data": { 07:01:09 DEBUG "resultType": "vector", 07:01:09 DEBUG "result": [ 07:01:09 DEBUG { 07:01:09 DEBUG "metric": { 07:01:09 DEBUG "pod": "am-767f6b67b5-9zw22" 07:01:09 DEBUG }, 07:01:09 DEBUG "value": [ 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG [...] 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG "pod": "am-767f6b67b5-vbltz" 07:01:09 DEBUG }, 07:01:09 DEBUG "value": [ 07:01:09 DEBUG 1713592869, 07:01:09 DEBUG "2661.366666666667" 07:01:09 DEBUG ] 07:01:09 DEBUG } 07:01:09 DEBUG ] 07:01:09 DEBUG } 07:01:09 DEBUG } 07:01:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592869" 07:01:09 INFO [http_cmd]: http status code OK 07:01:09 DEBUG --- status code --- 07:01:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:01:09 DEBUG --- http response --- 07:01:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 07:01:09 DEBUG ----- output ----- 07:01:09 DEBUG { 07:01:09 DEBUG "status": "success", 07:01:09 DEBUG "data": { 07:01:09 DEBUG "resultType": "vector", 07:01:09 DEBUG "result": [ 07:01:09 DEBUG { 07:01:09 DEBUG "metric": { 07:01:09 DEBUG "pod": "ds-cts-0" 07:01:09 DEBUG }, 07:01:09 DEBUG "value": [ 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG [...] 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG "pod": "ds-cts-2" 07:01:09 DEBUG }, 07:01:09 DEBUG "value": [ 07:01:09 DEBUG 1713592869, 07:01:09 DEBUG "0.0002666666666666667" 07:01:09 DEBUG ] 07:01:09 DEBUG } 07:01:09 DEBUG ] 07:01:09 DEBUG } 07:01:09 DEBUG } 07:01:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592869" 07:01:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592869" 07:01:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713592869" 07:01:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713592869" 07:01:09 INFO [http_cmd]: http status code OK 07:01:09 DEBUG --- status code --- 07:01:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:01:09 DEBUG --- http response --- 07:01:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:01:09 DEBUG ----- output ----- 07:01:09 DEBUG { 07:01:09 DEBUG "status": "success", 07:01:09 DEBUG "data": { 07:01:09 DEBUG "resultType": "vector", 07:01:09 DEBUG "result": [ 07:01:09 DEBUG { 07:01:09 DEBUG "metric": { 07:01:09 DEBUG "pod": "am-767f6b67b5-9zw22" 07:01:09 DEBUG }, 07:01:09 DEBUG "value": [ 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG [...] 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG "pod": "am-767f6b67b5-vbltz" 07:01:09 DEBUG }, 07:01:09 DEBUG "value": [ 07:01:09 DEBUG 1713592869, 07:01:09 DEBUG "0" 07:01:09 DEBUG ] 07:01:09 DEBUG } 07:01:09 DEBUG ] 07:01:09 DEBUG } 07:01:09 DEBUG } 07:01:09 INFO [http_cmd]: http status code OK 07:01:09 DEBUG --- status code --- 07:01:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:01:09 DEBUG --- http response --- 07:01:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 07:01:09 DEBUG ----- output ----- 07:01:09 DEBUG { 07:01:09 DEBUG "status": "success", 07:01:09 DEBUG "data": { 07:01:09 DEBUG "resultType": "vector", 07:01:09 DEBUG "result": [ 07:01:09 DEBUG { 07:01:09 DEBUG "metric": { 07:01:09 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 07:01:09 DEBUG }, 07:01:09 DEBUG "value": [ 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG [...] 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG "pod": "idm-b85f9d978-wnhxb" 07:01:09 DEBUG }, 07:01:09 DEBUG "value": [ 07:01:09 DEBUG 1713592869, 07:01:09 DEBUG "0" 07:01:09 DEBUG ] 07:01:09 DEBUG } 07:01:09 DEBUG ] 07:01:09 DEBUG } 07:01:09 DEBUG } 07:01:09 INFO [http_cmd]: http status code OK 07:01:09 DEBUG --- status code --- 07:01:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:01:09 DEBUG --- http response --- 07:01:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:01:09 DEBUG ----- output ----- 07:01:09 DEBUG { 07:01:09 DEBUG "status": "success", 07:01:09 DEBUG "data": { 07:01:09 DEBUG "resultType": "vector", 07:01:09 DEBUG "result": [ 07:01:09 DEBUG { 07:01:09 DEBUG "metric": { 07:01:09 DEBUG "pod": "am-767f6b67b5-9zw22" 07:01:09 DEBUG }, 07:01:09 DEBUG "value": [ 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG [...] 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG "pod": "am-767f6b67b5-vbltz" 07:01:09 DEBUG }, 07:01:09 DEBUG "value": [ 07:01:09 DEBUG 1713592869, 07:01:09 DEBUG "0.097" 07:01:09 DEBUG ] 07:01:09 DEBUG } 07:01:09 DEBUG ] 07:01:09 DEBUG } 07:01:09 DEBUG } 07:01:09 INFO [http_cmd]: http status code OK 07:01:09 DEBUG --- status code --- 07:01:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:01:09 DEBUG --- http response --- 07:01:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:01:09 DEBUG ----- output ----- 07:01:09 DEBUG { 07:01:09 DEBUG "status": "success", 07:01:09 DEBUG "data": { 07:01:09 DEBUG "resultType": "vector", 07:01:09 DEBUG "result": [ 07:01:09 DEBUG { 07:01:09 DEBUG "metric": { 07:01:09 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 07:01:09 DEBUG }, 07:01:09 DEBUG "value": [ 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG [...] 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 07:01:09 DEBUG }, 07:01:09 DEBUG "value": [ 07:01:09 DEBUG 1713592869, 07:01:09 DEBUG "8.023198854259528" 07:01:09 DEBUG ] 07:01:09 DEBUG } 07:01:09 DEBUG ] 07:01:09 DEBUG } 07:01:09 DEBUG } 07:01:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592869" 07:01:09 INFO [http_cmd]: http status code OK 07:01:09 DEBUG --- status code --- 07:01:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:01:09 DEBUG --- http response --- 07:01:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 07:01:09 DEBUG ----- output ----- 07:01:09 DEBUG { 07:01:09 DEBUG "status": "success", 07:01:09 DEBUG "data": { 07:01:09 DEBUG "resultType": "vector", 07:01:09 DEBUG "result": [ 07:01:09 DEBUG { 07:01:09 DEBUG "metric": { 07:01:09 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 07:01:09 DEBUG }, 07:01:09 DEBUG "value": [ 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG [...] 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG "pod": "idm-b85f9d978-wnhxb" 07:01:09 DEBUG }, 07:01:09 DEBUG "value": [ 07:01:09 DEBUG 1713592869, 07:01:09 DEBUG "0" 07:01:09 DEBUG ] 07:01:09 DEBUG } 07:01:09 DEBUG ] 07:01:09 DEBUG } 07:01:09 DEBUG } 07:01:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592869" 07:01:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592869" 07:01:09 INFO [http_cmd]: http status code OK 07:01:09 DEBUG --- status code --- 07:01:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:01:09 DEBUG --- http response --- 07:01:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:01:09 DEBUG ----- output ----- 07:01:09 DEBUG { 07:01:09 DEBUG "status": "success", 07:01:09 DEBUG "data": { 07:01:09 DEBUG "resultType": "vector", 07:01:09 DEBUG "result": [ 07:01:09 DEBUG { 07:01:09 DEBUG "metric": { 07:01:09 DEBUG "pod": "am-767f6b67b5-9zw22" 07:01:09 DEBUG }, 07:01:09 DEBUG "value": [ 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG [...] 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG "pod": "am-767f6b67b5-vbltz" 07:01:09 DEBUG }, 07:01:09 DEBUG "value": [ 07:01:09 DEBUG 1713592869, 07:01:09 DEBUG "2667.2666666666664" 07:01:09 DEBUG ] 07:01:09 DEBUG } 07:01:09 DEBUG ] 07:01:09 DEBUG } 07:01:09 DEBUG } 07:01:09 INFO [http_cmd]: http status code OK 07:01:09 DEBUG --- status code --- 07:01:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:01:09 DEBUG --- http response --- 07:01:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:01:09 DEBUG ----- output ----- 07:01:09 DEBUG { 07:01:09 DEBUG "status": "success", 07:01:09 DEBUG "data": { 07:01:09 DEBUG "resultType": "vector", 07:01:09 DEBUG "result": [ 07:01:09 DEBUG { 07:01:09 DEBUG "metric": { 07:01:09 DEBUG "instance": "10.142.0.107:9100" 07:01:09 DEBUG }, 07:01:09 DEBUG "value": [ 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG [...] 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG "instance": "10.142.0.43:9100" 07:01:09 DEBUG }, 07:01:09 DEBUG "value": [ 07:01:09 DEBUG 1713592869, 07:01:09 DEBUG "80868996.46666667" 07:01:09 DEBUG ] 07:01:09 DEBUG } 07:01:09 DEBUG ] 07:01:09 DEBUG } 07:01:09 DEBUG } 07:01:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592869" 07:01:09 INFO [http_cmd]: http status code OK 07:01:09 DEBUG --- status code --- 07:01:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:01:09 DEBUG --- http response --- 07:01:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:01:09 DEBUG ----- output ----- 07:01:09 DEBUG { 07:01:09 DEBUG "status": "success", 07:01:09 DEBUG "data": { 07:01:09 DEBUG "resultType": "vector", 07:01:09 DEBUG "result": [ 07:01:09 DEBUG { 07:01:09 DEBUG "metric": { 07:01:09 DEBUG "instance": "10.142.0.107:9100" 07:01:09 DEBUG }, 07:01:09 DEBUG "value": [ 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG [...] 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG 07:01:09 DEBUG "instance": "10.142.0.43:9100" 07:01:09 DEBUG }, 07:01:09 DEBUG "value": [ 07:01:09 DEBUG 1713592869, 07:01:09 DEBUG "0.0849999999999947" 07:01:09 DEBUG ] 07:01:09 DEBUG } 07:01:09 DEBUG ] 07:01:09 DEBUG } 07:01:09 DEBUG } 07:01:10 INFO 07:01:10 INFO [loop_until]: kubectl --namespace=xlou top node 07:01:10 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 07:01:10 INFO [loop_until]: OK (rc = 0) 07:01:10 DEBUG --- stdout --- 07:01:10 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 71m 0% 6281Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 8037m 26% 7472Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 17249m 57% 11376Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 86m 0% 3376Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 18585m 62% 13627Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 152m 0% 5169Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 17480m 58% 11604Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 19163m 64% 2999Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 19473m 65% 10862Mi 9% gke-xlou-cdm-ds-0f601751-j488 8114m 27% 7034Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 17426m 58% 15604Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 7998m 26% 7708Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 19283m 64% 12884Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 8176m 51% 8494Mi 14% 07:01:10 DEBUG --- stderr --- 07:01:10 DEBUG 07:01:11 INFO 07:01:11 INFO [loop_until]: kubectl --namespace=xlou top pods 07:01:11 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 07:01:11 INFO [loop_until]: OK (rc = 0) 07:01:11 DEBUG --- stdout --- 07:01:11 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 17277m 8653Mi am-767f6b67b5-qcqz2 17190m 8711Mi am-767f6b67b5-vbltz 18708m 8652Mi ds-cts-0 18549m 10485Mi ds-cts-1 17506m 10674Mi ds-cts-2 19332m 10268Mi ds-idrepo-0 7720m 3787Mi ds-idrepo-1 7576m 3803Mi ds-idrepo-2 7568m 3712Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1177Mi idm-b85f9d978-wnhxb 3m 1236Mi lodemon-7cb5b9456d-dskfn 5m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 8016m 4685Mi 07:01:11 DEBUG --- stderr --- 07:01:11 DEBUG 07:02:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592929" 07:02:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592929" 07:02:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592929" 07:02:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592929" 07:02:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592929" 07:02:09 INFO [http_cmd]: http status code OK 07:02:09 INFO [http_cmd]: http status code OK 07:02:09 INFO [http_cmd]: http status code OK 07:02:09 DEBUG --- status code --- 07:02:09 DEBUG --- status code --- 07:02:09 DEBUG --- status code --- 07:02:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:02:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:02:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:02:09 DEBUG --- http response --- 07:02:09 DEBUG --- http response --- 07:02:09 DEBUG --- http response --- 07:02:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:02:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:02:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:02:09 DEBUG ----- output ----- 07:02:09 DEBUG ----- output ----- 07:02:09 DEBUG ----- output ----- 07:02:09 DEBUG { 07:02:09 DEBUG { 07:02:09 DEBUG { 07:02:09 DEBUG "status": "success", 07:02:09 DEBUG "status": "success", 07:02:09 DEBUG "status": "success", 07:02:09 DEBUG "data": { 07:02:09 DEBUG "data": { 07:02:09 DEBUG "data": { 07:02:09 DEBUG "resultType": "vector", 07:02:09 DEBUG "resultType": "vector", 07:02:09 DEBUG "resultType": "vector", 07:02:09 DEBUG "result": [ 07:02:09 DEBUG "result": [ 07:02:09 DEBUG "result": [ 07:02:09 DEBUG { 07:02:09 DEBUG { 07:02:09 DEBUG { 07:02:09 DEBUG "metric": { 07:02:09 DEBUG "metric": { 07:02:09 DEBUG "metric": { 07:02:09 DEBUG "pod": "am-767f6b67b5-9zw22" 07:02:09 DEBUG "pod": "ds-idrepo-0" 07:02:09 DEBUG "instance": "10.142.0.107:9100" 07:02:09 DEBUG }, 07:02:09 DEBUG }, 07:02:09 DEBUG }, 07:02:09 DEBUG "value": [ 07:02:09 DEBUG "value": [ 07:02:09 DEBUG "value": [ 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG [...] 07:02:09 DEBUG [...] 07:02:09 DEBUG [...] 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG "pod": "am-767f6b67b5-vbltz" 07:02:09 DEBUG "pod": "ds-idrepo-2" 07:02:09 DEBUG "instance": "10.142.0.43:9100" 07:02:09 DEBUG }, 07:02:09 DEBUG }, 07:02:09 DEBUG }, 07:02:09 DEBUG "value": [ 07:02:09 DEBUG "value": [ 07:02:09 DEBUG "value": [ 07:02:09 DEBUG 1713592929, 07:02:09 DEBUG 1713592929, 07:02:09 DEBUG 1713592929, 07:02:09 DEBUG "480281.5" 07:02:09 DEBUG "0" 07:02:09 DEBUG "3275081198592" 07:02:09 DEBUG ] 07:02:09 DEBUG ] 07:02:09 DEBUG ] 07:02:09 DEBUG } 07:02:09 DEBUG } 07:02:09 DEBUG } 07:02:09 DEBUG ] 07:02:09 DEBUG ] 07:02:09 DEBUG ] 07:02:09 DEBUG } 07:02:09 DEBUG } 07:02:09 DEBUG } 07:02:09 DEBUG } 07:02:09 DEBUG } 07:02:09 DEBUG } 07:02:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592929" 07:02:09 INFO [http_cmd]: http status code OK 07:02:09 DEBUG --- status code --- 07:02:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:02:09 DEBUG --- http response --- 07:02:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 07:02:09 DEBUG ----- output ----- 07:02:09 DEBUG { 07:02:09 DEBUG "status": "success", 07:02:09 DEBUG "data": { 07:02:09 DEBUG "resultType": "vector", 07:02:09 DEBUG "result": [ 07:02:09 DEBUG { 07:02:09 DEBUG "metric": { 07:02:09 DEBUG "pod": "ds-cts-0" 07:02:09 DEBUG }, 07:02:09 DEBUG "value": [ 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG [...] 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG "pod": "ds-cts-2" 07:02:09 DEBUG }, 07:02:09 DEBUG "value": [ 07:02:09 DEBUG 1713592929, 07:02:09 DEBUG "30" 07:02:09 DEBUG ] 07:02:09 DEBUG } 07:02:09 DEBUG ] 07:02:09 DEBUG } 07:02:09 DEBUG } 07:02:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592929" 07:02:09 INFO [http_cmd]: http status code OK 07:02:09 DEBUG --- status code --- 07:02:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:02:09 DEBUG --- http response --- 07:02:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 07:02:09 DEBUG ----- output ----- 07:02:09 DEBUG { 07:02:09 DEBUG "status": "success", 07:02:09 DEBUG "data": { 07:02:09 DEBUG "resultType": "vector", 07:02:09 DEBUG "result": [ 07:02:09 DEBUG { 07:02:09 DEBUG "metric": { 07:02:09 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 07:02:09 DEBUG }, 07:02:09 DEBUG "value": [ 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG [...] 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG "pod": "ds-idrepo-0" 07:02:09 DEBUG }, 07:02:09 DEBUG "value": [ 07:02:09 DEBUG 1713592929, 07:02:09 DEBUG "0" 07:02:09 DEBUG ] 07:02:09 DEBUG } 07:02:09 DEBUG ] 07:02:09 DEBUG } 07:02:09 DEBUG } 07:02:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592929" 07:02:09 INFO [http_cmd]: http status code OK 07:02:09 DEBUG --- status code --- 07:02:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:02:09 DEBUG --- http response --- 07:02:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:02:09 DEBUG ----- output ----- 07:02:09 DEBUG { 07:02:09 DEBUG "status": "success", 07:02:09 DEBUG "data": { 07:02:09 DEBUG "resultType": "vector", 07:02:09 DEBUG "result": [ 07:02:09 DEBUG { 07:02:09 DEBUG "metric": { 07:02:09 DEBUG "pod": "ds-idrepo-0" 07:02:09 DEBUG }, 07:02:09 DEBUG "value": [ 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG [...] 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG "pod": "ds-idrepo-2" 07:02:09 DEBUG }, 07:02:09 DEBUG "value": [ 07:02:09 DEBUG 1713592929, 07:02:09 DEBUG "0" 07:02:09 DEBUG ] 07:02:09 DEBUG } 07:02:09 DEBUG ] 07:02:09 DEBUG } 07:02:09 DEBUG } 07:02:09 INFO [http_cmd]: http status code OK 07:02:09 DEBUG --- status code --- 07:02:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:02:09 DEBUG --- http response --- 07:02:09 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 07:02:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713592929" 07:02:09 INFO [http_cmd]: http status code OK 07:02:09 DEBUG --- status code --- 07:02:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:02:09 DEBUG --- http response --- 07:02:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 07:02:09 DEBUG ----- output ----- 07:02:09 DEBUG { 07:02:09 DEBUG "status": "success", 07:02:09 DEBUG "data": { 07:02:09 DEBUG "resultType": "vector", 07:02:09 DEBUG "result": [ 07:02:09 DEBUG { 07:02:09 DEBUG "metric": { 07:02:09 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 07:02:09 DEBUG }, 07:02:09 DEBUG "value": [ 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG [...] 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG "pod": "ds-idrepo-0" 07:02:09 DEBUG }, 07:02:09 DEBUG "value": [ 07:02:09 DEBUG 1713592929, 07:02:09 DEBUG "4060.5749837623493" 07:02:09 DEBUG ] 07:02:09 DEBUG } 07:02:09 DEBUG ] 07:02:09 DEBUG } 07:02:09 DEBUG } 07:02:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713592929" 07:02:09 ERROR [http_cmd]: ERROR 07:02:09 ERROR --- status code --- 07:02:09 ERROR http status code is 400 (expected [200, 401, 403, 404]) 07:02:09 ERROR --- http response --- 07:02:09 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 07:02:09 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 07:02:09 INFO [http_cmd]: http status code OK 07:02:09 DEBUG --- status code --- 07:02:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:02:09 DEBUG --- http response --- 07:02:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:02:09 DEBUG ----- output ----- 07:02:09 DEBUG { 07:02:09 DEBUG "status": "success", 07:02:09 DEBUG "data": { 07:02:09 DEBUG "resultType": "vector", 07:02:09 DEBUG "result": [ 07:02:09 DEBUG { 07:02:09 DEBUG "metric": { 07:02:09 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 07:02:09 DEBUG }, 07:02:09 DEBUG "value": [ 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG [...] 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 07:02:09 DEBUG }, 07:02:09 DEBUG "value": [ 07:02:09 DEBUG 1713592929, 07:02:09 DEBUG "5011136512" 07:02:09 DEBUG ] 07:02:09 DEBUG } 07:02:09 DEBUG ] 07:02:09 DEBUG } 07:02:09 DEBUG } 07:02:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592929" 07:02:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592929" 07:02:09 INFO [http_cmd]: http status code OK 07:02:09 DEBUG --- status code --- 07:02:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:02:09 DEBUG --- http response --- 07:02:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:02:09 DEBUG ----- output ----- 07:02:09 DEBUG { 07:02:09 DEBUG "status": "success", 07:02:09 DEBUG "data": { 07:02:09 DEBUG "resultType": "vector", 07:02:09 DEBUG "result": [ 07:02:09 DEBUG { 07:02:09 DEBUG "metric": { 07:02:09 DEBUG "pod": "am-767f6b67b5-9zw22" 07:02:09 DEBUG }, 07:02:09 DEBUG "value": [ 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG [...] 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG "pod": "am-767f6b67b5-vbltz" 07:02:09 DEBUG }, 07:02:09 DEBUG "value": [ 07:02:09 DEBUG 1713592929, 07:02:09 DEBUG "2668.3333333333335" 07:02:09 DEBUG ] 07:02:09 DEBUG } 07:02:09 DEBUG ] 07:02:09 DEBUG } 07:02:09 DEBUG } 07:02:09 INFO [http_cmd]: http status code OK 07:02:09 DEBUG --- status code --- 07:02:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:02:09 DEBUG --- http response --- 07:02:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713592929" 07:02:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:02:09 DEBUG ----- output ----- 07:02:09 DEBUG { 07:02:09 DEBUG "status": "success", 07:02:09 DEBUG "data": { 07:02:09 DEBUG "resultType": "vector", 07:02:09 DEBUG "result": [ 07:02:09 DEBUG { 07:02:09 DEBUG "metric": { 07:02:09 DEBUG "instance": "10.142.0.107:9100" 07:02:09 DEBUG }, 07:02:09 DEBUG "value": [ 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG [...] 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG "instance": "10.142.0.43:9100" 07:02:09 DEBUG }, 07:02:09 DEBUG "value": [ 07:02:09 DEBUG 1713592929, 07:02:09 DEBUG "0.2269666666665813" 07:02:09 DEBUG ] 07:02:09 DEBUG } 07:02:09 DEBUG ] 07:02:09 DEBUG } 07:02:09 DEBUG } 07:02:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592929" 07:02:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592929" 07:02:09 INFO [http_cmd]: http status code OK 07:02:09 DEBUG --- status code --- 07:02:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:02:09 DEBUG --- http response --- 07:02:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 07:02:09 DEBUG ----- output ----- 07:02:09 DEBUG { 07:02:09 DEBUG "status": "success", 07:02:09 DEBUG "data": { 07:02:09 DEBUG "resultType": "vector", 07:02:09 DEBUG "result": [ 07:02:09 DEBUG { 07:02:09 DEBUG "metric": { 07:02:09 DEBUG "pod": "ds-idrepo-0" 07:02:09 DEBUG }, 07:02:09 DEBUG "value": [ 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG [...] 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG "pod": "ds-cts-2" 07:02:09 DEBUG }, 07:02:09 DEBUG "value": [ 07:02:09 DEBUG 1713592929, 07:02:09 DEBUG "0" 07:02:09 DEBUG ] 07:02:09 DEBUG } 07:02:09 DEBUG ] 07:02:09 DEBUG } 07:02:09 DEBUG } 07:02:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592929" 07:02:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592929" 07:02:09 INFO [http_cmd]: http status code OK 07:02:09 DEBUG --- status code --- 07:02:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:02:09 DEBUG --- http response --- 07:02:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:02:09 DEBUG ----- output ----- 07:02:09 DEBUG { 07:02:09 DEBUG "status": "success", 07:02:09 DEBUG "data": { 07:02:09 DEBUG "resultType": "vector", 07:02:09 DEBUG "result": [ 07:02:09 DEBUG { 07:02:09 DEBUG "metric": { 07:02:09 DEBUG "pod": "ds-idrepo-0" 07:02:09 DEBUG }, 07:02:09 DEBUG "value": [ 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG [...] 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG "pod": "ds-idrepo-2" 07:02:09 DEBUG }, 07:02:09 DEBUG "value": [ 07:02:09 DEBUG 1713592929, 07:02:09 DEBUG "0" 07:02:09 DEBUG ] 07:02:09 DEBUG } 07:02:09 DEBUG ] 07:02:09 DEBUG } 07:02:09 DEBUG } 07:02:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592929" 07:02:09 INFO [http_cmd]: http status code OK 07:02:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592929" 07:02:09 INFO [http_cmd]: http status code OK 07:02:09 DEBUG --- status code --- 07:02:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:02:09 INFO [http_cmd]: http status code OK 07:02:09 DEBUG --- http response --- 07:02:09 DEBUG --- status code --- 07:02:09 DEBUG --- status code --- 07:02:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:02:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:02:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:02:09 DEBUG --- http response --- 07:02:09 DEBUG ----- output ----- 07:02:09 DEBUG --- http response --- 07:02:09 DEBUG { 07:02:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:02:09 DEBUG "status": "success", 07:02:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:02:09 DEBUG ----- output ----- 07:02:09 DEBUG "data": { 07:02:09 DEBUG ----- output ----- 07:02:09 DEBUG { 07:02:09 DEBUG "resultType": "vector", 07:02:09 DEBUG { 07:02:09 DEBUG "status": "success", 07:02:09 DEBUG "result": [ 07:02:09 DEBUG "status": "success", 07:02:09 DEBUG "data": { 07:02:09 DEBUG { 07:02:09 DEBUG "data": { 07:02:09 DEBUG "resultType": "vector", 07:02:09 DEBUG "metric": { 07:02:09 DEBUG "resultType": "vector", 07:02:09 DEBUG "result": [ 07:02:09 DEBUG "pod": "am-767f6b67b5-9zw22" 07:02:09 DEBUG "result": [ 07:02:09 DEBUG { 07:02:09 DEBUG }, 07:02:09 DEBUG { 07:02:09 DEBUG "metric": { 07:02:09 DEBUG "value": [ 07:02:09 DEBUG "metric": { 07:02:09 DEBUG "pod": "am-767f6b67b5-9zw22" 07:02:09 DEBUG 07:02:09 DEBUG "instance": "10.142.0.107:9100" 07:02:09 DEBUG }, 07:02:09 DEBUG 07:02:09 DEBUG }, 07:02:09 DEBUG "value": [ 07:02:09 DEBUG 07:02:09 DEBUG "value": [ 07:02:09 DEBUG 07:02:09 DEBUG [...] 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG [...] 07:02:09 DEBUG 07:02:09 DEBUG [...] 07:02:09 DEBUG 07:02:09 DEBUG "pod": "am-767f6b67b5-vbltz" 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG }, 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG "value": [ 07:02:09 DEBUG 07:02:09 DEBUG "pod": "am-767f6b67b5-vbltz" 07:02:09 DEBUG 1713592929, 07:02:09 DEBUG "instance": "10.142.0.43:9100" 07:02:09 DEBUG }, 07:02:09 DEBUG "2666.8333333333335" 07:02:09 DEBUG }, 07:02:09 DEBUG "value": [ 07:02:09 DEBUG ] 07:02:09 DEBUG "value": [ 07:02:09 DEBUG 1713592929, 07:02:09 DEBUG } 07:02:09 DEBUG 1713592929, 07:02:09 DEBUG "2675.633333333333" 07:02:09 DEBUG ] 07:02:09 DEBUG "80823608.13333333" 07:02:09 DEBUG ] 07:02:09 DEBUG } 07:02:09 DEBUG ] 07:02:09 DEBUG } 07:02:09 DEBUG } 07:02:09 DEBUG } 07:02:09 DEBUG ] 07:02:09 DEBUG ] 07:02:09 DEBUG } 07:02:09 DEBUG } 07:02:09 DEBUG } 07:02:09 DEBUG } 07:02:09 INFO [http_cmd]: http status code OK 07:02:09 INFO [http_cmd]: http status code OK 07:02:09 DEBUG --- status code --- 07:02:09 DEBUG --- status code --- 07:02:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:02:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:02:09 DEBUG --- http response --- 07:02:09 DEBUG --- http response --- 07:02:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 07:02:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 07:02:09 DEBUG ----- output ----- 07:02:09 DEBUG ----- output ----- 07:02:09 DEBUG { 07:02:09 DEBUG { 07:02:09 DEBUG "status": "success", 07:02:09 DEBUG "status": "success", 07:02:09 DEBUG "data": { 07:02:09 DEBUG "data": { 07:02:09 DEBUG "resultType": "vector", 07:02:09 DEBUG "resultType": "vector", 07:02:09 DEBUG "result": [ 07:02:09 DEBUG "result": [ 07:02:09 DEBUG { 07:02:09 DEBUG { 07:02:09 DEBUG "metric": { 07:02:09 DEBUG "metric": { 07:02:09 DEBUG "pod": "ds-cts-0" 07:02:09 DEBUG "pod": "ds-cts-0" 07:02:09 DEBUG }, 07:02:09 DEBUG }, 07:02:09 DEBUG "value": [ 07:02:09 DEBUG "value": [ 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG [...] 07:02:09 DEBUG [...] 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG "pod": "ds-cts-2" 07:02:09 DEBUG "pod": "ds-cts-2" 07:02:09 DEBUG }, 07:02:09 DEBUG }, 07:02:09 DEBUG "value": [ 07:02:09 DEBUG "value": [ 07:02:09 DEBUG 1713592929, 07:02:09 DEBUG 1713592929, 07:02:09 DEBUG "0" 07:02:09 DEBUG "0.00006666666666666667" 07:02:09 DEBUG ] 07:02:09 DEBUG ] 07:02:09 DEBUG } 07:02:09 DEBUG } 07:02:09 DEBUG ] 07:02:09 DEBUG ] 07:02:09 DEBUG } 07:02:09 DEBUG } 07:02:09 DEBUG } 07:02:09 DEBUG } 07:02:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592929" 07:02:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592929" 07:02:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713592929" 07:02:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713592929" 07:02:09 INFO [http_cmd]: http status code OK 07:02:09 DEBUG --- status code --- 07:02:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:02:09 DEBUG --- http response --- 07:02:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:02:09 DEBUG ----- output ----- 07:02:09 DEBUG { 07:02:09 DEBUG "status": "success", 07:02:09 DEBUG "data": { 07:02:09 DEBUG "resultType": "vector", 07:02:09 DEBUG "result": [ 07:02:09 DEBUG { 07:02:09 DEBUG "metric": { 07:02:09 DEBUG "pod": "am-767f6b67b5-9zw22" 07:02:09 DEBUG }, 07:02:09 DEBUG "value": [ 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG [...] 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG "pod": "am-767f6b67b5-vbltz" 07:02:09 DEBUG }, 07:02:09 DEBUG "value": [ 07:02:09 DEBUG 1713592929, 07:02:09 DEBUG "0.097" 07:02:09 DEBUG ] 07:02:09 DEBUG } 07:02:09 DEBUG ] 07:02:09 DEBUG } 07:02:09 DEBUG } 07:02:09 INFO [http_cmd]: http status code OK 07:02:09 DEBUG --- status code --- 07:02:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:02:09 DEBUG --- http response --- 07:02:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:02:09 DEBUG ----- output ----- 07:02:09 DEBUG { 07:02:09 DEBUG "status": "success", 07:02:09 DEBUG "data": { 07:02:09 DEBUG "resultType": "vector", 07:02:09 DEBUG "result": [ 07:02:09 DEBUG { 07:02:09 DEBUG "metric": { 07:02:09 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 07:02:09 DEBUG }, 07:02:09 DEBUG "value": [ 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG [...] 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG "pod": "ds-idrepo-2" 07:02:09 DEBUG }, 07:02:09 DEBUG "value": [ 07:02:09 DEBUG 1713592929, 07:02:09 DEBUG "0" 07:02:09 DEBUG ] 07:02:09 DEBUG } 07:02:09 DEBUG ] 07:02:09 DEBUG } 07:02:09 DEBUG } 07:02:09 INFO [http_cmd]: http status code OK 07:02:09 DEBUG --- status code --- 07:02:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:02:09 DEBUG --- http response --- 07:02:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:02:09 DEBUG ----- output ----- 07:02:09 DEBUG { 07:02:09 DEBUG "status": "success", 07:02:09 DEBUG "data": { 07:02:09 DEBUG "resultType": "vector", 07:02:09 DEBUG "result": [ 07:02:09 DEBUG { 07:02:09 DEBUG "metric": { 07:02:09 DEBUG "pod": "am-767f6b67b5-9zw22" 07:02:09 DEBUG }, 07:02:09 DEBUG "value": [ 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG [...] 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG "pod": "am-767f6b67b5-vbltz" 07:02:09 DEBUG }, 07:02:09 DEBUG "value": [ 07:02:09 DEBUG 1713592929, 07:02:09 DEBUG "0" 07:02:09 DEBUG ] 07:02:09 DEBUG } 07:02:09 DEBUG ] 07:02:09 DEBUG } 07:02:09 DEBUG } 07:02:09 INFO [http_cmd]: http status code OK 07:02:09 DEBUG --- status code --- 07:02:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:02:09 DEBUG --- http response --- 07:02:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:02:09 DEBUG ----- output ----- 07:02:09 DEBUG { 07:02:09 DEBUG "status": "success", 07:02:09 DEBUG "data": { 07:02:09 DEBUG "resultType": "vector", 07:02:09 DEBUG "result": [ 07:02:09 DEBUG { 07:02:09 DEBUG "metric": { 07:02:09 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 07:02:09 DEBUG }, 07:02:09 DEBUG "value": [ 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG [...] 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 07:02:09 DEBUG }, 07:02:09 DEBUG "value": [ 07:02:09 DEBUG 1713592929, 07:02:09 DEBUG "8.028153483153336" 07:02:09 DEBUG ] 07:02:09 DEBUG } 07:02:09 DEBUG ] 07:02:09 DEBUG } 07:02:09 DEBUG } 07:02:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592929" 07:02:09 INFO [http_cmd]: http status code OK 07:02:09 DEBUG --- status code --- 07:02:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:02:09 DEBUG --- http response --- 07:02:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:02:09 DEBUG ----- output ----- 07:02:09 DEBUG { 07:02:09 DEBUG "status": "success", 07:02:09 DEBUG "data": { 07:02:09 DEBUG "resultType": "vector", 07:02:09 DEBUG "result": [ 07:02:09 DEBUG { 07:02:09 DEBUG "metric": { 07:02:09 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 07:02:09 DEBUG }, 07:02:09 DEBUG "value": [ 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG [...] 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG "pod": "ds-idrepo-2" 07:02:09 DEBUG }, 07:02:09 DEBUG "value": [ 07:02:09 DEBUG 1713592929, 07:02:09 DEBUG "0" 07:02:09 DEBUG ] 07:02:09 DEBUG } 07:02:09 DEBUG ] 07:02:09 DEBUG } 07:02:09 DEBUG } 07:02:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592929" 07:02:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592929" 07:02:09 INFO [http_cmd]: http status code OK 07:02:09 DEBUG --- status code --- 07:02:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:02:09 DEBUG --- http response --- 07:02:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:02:09 DEBUG ----- output ----- 07:02:09 DEBUG { 07:02:09 DEBUG "status": "success", 07:02:09 DEBUG "data": { 07:02:09 DEBUG "resultType": "vector", 07:02:09 DEBUG "result": [ 07:02:09 DEBUG { 07:02:09 DEBUG "metric": { 07:02:09 DEBUG "pod": "am-767f6b67b5-9zw22" 07:02:09 DEBUG }, 07:02:09 DEBUG "value": [ 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG [...] 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG "pod": "am-767f6b67b5-vbltz" 07:02:09 DEBUG }, 07:02:09 DEBUG "value": [ 07:02:09 DEBUG 1713592929, 07:02:09 DEBUG "2669.4666666666667" 07:02:09 DEBUG ] 07:02:09 DEBUG } 07:02:09 DEBUG ] 07:02:09 DEBUG } 07:02:09 DEBUG } 07:02:09 INFO [http_cmd]: http status code OK 07:02:09 DEBUG --- status code --- 07:02:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:02:09 DEBUG --- http response --- 07:02:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:02:09 DEBUG ----- output ----- 07:02:09 DEBUG { 07:02:09 DEBUG "status": "success", 07:02:09 DEBUG "data": { 07:02:09 DEBUG "resultType": "vector", 07:02:09 DEBUG "result": [ 07:02:09 DEBUG { 07:02:09 DEBUG "metric": { 07:02:09 DEBUG "instance": "10.142.0.107:9100" 07:02:09 DEBUG }, 07:02:09 DEBUG "value": [ 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG [...] 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG "instance": "10.142.0.43:9100" 07:02:09 DEBUG }, 07:02:09 DEBUG "value": [ 07:02:09 DEBUG 1713592929, 07:02:09 DEBUG "80824375.16666666" 07:02:09 DEBUG ] 07:02:09 DEBUG } 07:02:09 DEBUG ] 07:02:09 DEBUG } 07:02:09 DEBUG } 07:02:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592929" 07:02:09 INFO [http_cmd]: http status code OK 07:02:09 DEBUG --- status code --- 07:02:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:02:09 DEBUG --- http response --- 07:02:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:02:09 DEBUG ----- output ----- 07:02:09 DEBUG { 07:02:09 DEBUG "status": "success", 07:02:09 DEBUG "data": { 07:02:09 DEBUG "resultType": "vector", 07:02:09 DEBUG "result": [ 07:02:09 DEBUG { 07:02:09 DEBUG "metric": { 07:02:09 DEBUG "instance": "10.142.0.107:9100" 07:02:09 DEBUG }, 07:02:09 DEBUG "value": [ 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG [...] 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG 07:02:09 DEBUG "instance": "10.142.0.43:9100" 07:02:09 DEBUG }, 07:02:09 DEBUG "value": [ 07:02:09 DEBUG 1713592929, 07:02:09 DEBUG "0.07999999999999356" 07:02:09 DEBUG ] 07:02:09 DEBUG } 07:02:09 DEBUG ] 07:02:09 DEBUG } 07:02:09 DEBUG } 07:02:11 INFO 07:02:11 INFO [loop_until]: kubectl --namespace=xlou top node 07:02:11 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 07:02:11 INFO [loop_until]: OK (rc = 0) 07:02:11 DEBUG --- stdout --- 07:02:11 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 83m 0% 6279Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7978m 26% 7469Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 17226m 57% 11359Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 81m 0% 3374Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 18875m 63% 13623Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 151m 0% 5197Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 17486m 58% 11597Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 19077m 63% 3002Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 19471m 65% 10860Mi 9% gke-xlou-cdm-ds-0f601751-j488 8176m 27% 7030Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 17556m 58% 15601Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 8022m 26% 7703Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 19501m 65% 12884Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 8161m 51% 8494Mi 14% 07:02:11 DEBUG --- stderr --- 07:02:11 DEBUG 07:02:11 INFO 07:02:11 INFO [loop_until]: kubectl --namespace=xlou top pods 07:02:11 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 07:02:11 INFO [loop_until]: OK (rc = 0) 07:02:11 DEBUG --- stdout --- 07:02:11 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 17176m 8650Mi am-767f6b67b5-qcqz2 17042m 8717Mi am-767f6b67b5-vbltz 19134m 8652Mi ds-cts-0 18919m 10467Mi ds-cts-1 17582m 10673Mi ds-cts-2 19121m 10277Mi ds-idrepo-0 7744m 3787Mi ds-idrepo-1 7333m 3803Mi ds-idrepo-2 7597m 3712Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1177Mi idm-b85f9d978-wnhxb 3m 1236Mi lodemon-7cb5b9456d-dskfn 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 8105m 4688Mi 07:02:11 DEBUG --- stderr --- 07:02:11 DEBUG 07:03:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592989" 07:03:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592989" 07:03:09 INFO [http_cmd]: http status code OK 07:03:09 INFO [http_cmd]: http status code OK 07:03:09 DEBUG --- status code --- 07:03:09 DEBUG --- status code --- 07:03:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:03:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:03:09 DEBUG --- http response --- 07:03:09 DEBUG --- http response --- 07:03:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:03:09 DEBUG ----- output ----- 07:03:09 DEBUG { 07:03:09 DEBUG "status": "success", 07:03:09 DEBUG "data": { 07:03:09 DEBUG "resultType": "vector", 07:03:09 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 07:03:09 DEBUG "result": [ 07:03:09 DEBUG { 07:03:09 DEBUG "metric": { 07:03:09 DEBUG "pod": "ds-idrepo-0" 07:03:09 DEBUG }, 07:03:09 DEBUG "value": [ 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG [...] 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG "pod": "ds-idrepo-2" 07:03:09 DEBUG }, 07:03:09 DEBUG "value": [ 07:03:09 DEBUG 1713592989, 07:03:09 DEBUG "0" 07:03:09 DEBUG ] 07:03:09 DEBUG } 07:03:09 DEBUG ] 07:03:09 DEBUG } 07:03:09 DEBUG } 07:03:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592989" 07:03:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592989" 07:03:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592989" 07:03:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592989" 07:03:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592989" 07:03:09 INFO [http_cmd]: http status code OK 07:03:09 INFO [http_cmd]: http status code OK 07:03:09 DEBUG --- status code --- 07:03:09 DEBUG --- status code --- 07:03:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:03:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:03:09 DEBUG --- http response --- 07:03:09 DEBUG --- http response --- 07:03:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:03:09 DEBUG ----- output ----- 07:03:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:03:09 DEBUG { 07:03:09 DEBUG ----- output ----- 07:03:09 DEBUG "status": "success", 07:03:09 DEBUG { 07:03:09 DEBUG "data": { 07:03:09 DEBUG "status": "success", 07:03:09 DEBUG "resultType": "vector", 07:03:09 DEBUG "data": { 07:03:09 DEBUG "result": [ 07:03:09 DEBUG "resultType": "vector", 07:03:09 DEBUG { 07:03:09 DEBUG "result": [ 07:03:09 DEBUG "metric": { 07:03:09 DEBUG { 07:03:09 DEBUG "pod": "am-767f6b67b5-9zw22" 07:03:09 DEBUG "metric": { 07:03:09 DEBUG }, 07:03:09 DEBUG "instance": "10.142.0.107:9100" 07:03:09 DEBUG "value": [ 07:03:09 DEBUG }, 07:03:09 DEBUG 07:03:09 DEBUG "value": [ 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG [...] 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG [...] 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG "pod": "am-767f6b67b5-vbltz" 07:03:09 DEBUG 07:03:09 DEBUG }, 07:03:09 DEBUG "instance": "10.142.0.43:9100" 07:03:09 DEBUG "value": [ 07:03:09 DEBUG }, 07:03:09 DEBUG 1713592989, 07:03:09 DEBUG "value": [ 07:03:09 DEBUG "480530.5" 07:03:09 DEBUG 1713592989, 07:03:09 DEBUG ] 07:03:09 DEBUG "3279555894272" 07:03:09 DEBUG } 07:03:09 DEBUG ] 07:03:09 DEBUG ] 07:03:09 DEBUG } 07:03:09 DEBUG } 07:03:09 DEBUG ] 07:03:09 DEBUG } 07:03:09 DEBUG } 07:03:09 DEBUG } 07:03:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592989" 07:03:09 INFO [http_cmd]: http status code OK 07:03:09 DEBUG --- status code --- 07:03:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:03:09 DEBUG --- http response --- 07:03:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:03:09 DEBUG ----- output ----- 07:03:09 DEBUG { 07:03:09 DEBUG "status": "success", 07:03:09 DEBUG "data": { 07:03:09 DEBUG "resultType": "vector", 07:03:09 DEBUG "result": [ 07:03:09 DEBUG { 07:03:09 DEBUG "metric": { 07:03:09 DEBUG "pod": "ds-idrepo-0" 07:03:09 DEBUG }, 07:03:09 DEBUG "value": [ 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG [...] 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG "pod": "ds-idrepo-2" 07:03:09 DEBUG }, 07:03:09 DEBUG "value": [ 07:03:09 DEBUG 1713592989, 07:03:09 DEBUG "0" 07:03:09 DEBUG ] 07:03:09 DEBUG } 07:03:09 DEBUG ] 07:03:09 DEBUG } 07:03:09 DEBUG } 07:03:09 INFO [http_cmd]: http status code OK 07:03:09 DEBUG --- status code --- 07:03:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:03:09 DEBUG --- http response --- 07:03:09 INFO [http_cmd]: http status code OK 07:03:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 07:03:09 DEBUG --- status code --- 07:03:09 DEBUG ----- output ----- 07:03:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:03:09 DEBUG { 07:03:09 DEBUG --- http response --- 07:03:09 DEBUG "status": "success", 07:03:09 DEBUG "data": { 07:03:09 DEBUG "resultType": "vector", 07:03:09 DEBUG "result": [ 07:03:09 DEBUG { 07:03:09 DEBUG "metric": { 07:03:09 DEBUG "pod": "ds-cts-0" 07:03:09 DEBUG }, 07:03:09 DEBUG "value": [ 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG [...] 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG "pod": "ds-cts-2" 07:03:09 DEBUG }, 07:03:09 DEBUG "value": [ 07:03:09 DEBUG 1713592989, 07:03:09 DEBUG "24.5" 07:03:09 DEBUG ] 07:03:09 DEBUG } 07:03:09 DEBUG ] 07:03:09 DEBUG } 07:03:09 DEBUG } 07:03:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 07:03:09 DEBUG ----- output ----- 07:03:09 DEBUG { 07:03:09 DEBUG "status": "success", 07:03:09 DEBUG "data": { 07:03:09 DEBUG "resultType": "vector", 07:03:09 DEBUG "result": [ 07:03:09 DEBUG { 07:03:09 DEBUG "metric": { 07:03:09 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 07:03:09 DEBUG }, 07:03:09 DEBUG "value": [ 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG [...] 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG "pod": "ds-cts-1" 07:03:09 DEBUG }, 07:03:09 DEBUG "value": [ 07:03:09 DEBUG 1713592989, 07:03:09 DEBUG "61199770.82464192" 07:03:09 DEBUG ] 07:03:09 DEBUG } 07:03:09 DEBUG ] 07:03:09 DEBUG } 07:03:09 DEBUG } 07:03:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713592989" 07:03:09 INFO [http_cmd]: http status code OK 07:03:09 DEBUG --- status code --- 07:03:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:03:09 DEBUG --- http response --- 07:03:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 07:03:09 DEBUG ----- output ----- 07:03:09 DEBUG { 07:03:09 DEBUG "status": "success", 07:03:09 DEBUG "data": { 07:03:09 DEBUG "resultType": "vector", 07:03:09 DEBUG "result": [ 07:03:09 DEBUG { 07:03:09 DEBUG "metric": { 07:03:09 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 07:03:09 DEBUG }, 07:03:09 DEBUG "value": [ 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG [...] 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG "pod": "ds-cts-1" 07:03:09 DEBUG }, 07:03:09 DEBUG "value": [ 07:03:09 DEBUG 1713592989, 07:03:09 DEBUG "78160918.92643583" 07:03:09 DEBUG ] 07:03:09 DEBUG } 07:03:09 DEBUG ] 07:03:09 DEBUG } 07:03:09 DEBUG } 07:03:09 ERROR [http_cmd]: ERROR 07:03:09 ERROR --- status code --- 07:03:09 ERROR http status code is 400 (expected [200, 401, 403, 404]) 07:03:09 ERROR --- http response --- 07:03:09 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 07:03:09 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 07:03:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713592989" 07:03:09 INFO [http_cmd]: http status code OK 07:03:09 DEBUG --- status code --- 07:03:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:03:09 DEBUG --- http response --- 07:03:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:03:09 DEBUG ----- output ----- 07:03:09 DEBUG { 07:03:09 DEBUG "status": "success", 07:03:09 DEBUG "data": { 07:03:09 DEBUG "resultType": "vector", 07:03:09 DEBUG "result": [ 07:03:09 DEBUG { 07:03:09 DEBUG "metric": { 07:03:09 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 07:03:09 DEBUG }, 07:03:09 DEBUG "value": [ 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG [...] 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 07:03:09 DEBUG }, 07:03:09 DEBUG "value": [ 07:03:09 DEBUG 1713592989, 07:03:09 DEBUG "5014519808" 07:03:09 DEBUG ] 07:03:09 DEBUG } 07:03:09 DEBUG ] 07:03:09 DEBUG } 07:03:09 DEBUG } 07:03:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592989" 07:03:09 INFO [http_cmd]: http status code OK 07:03:09 DEBUG --- status code --- 07:03:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:03:09 DEBUG --- http response --- 07:03:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:03:09 DEBUG ----- output ----- 07:03:09 DEBUG { 07:03:09 DEBUG "status": "success", 07:03:09 DEBUG "data": { 07:03:09 DEBUG "resultType": "vector", 07:03:09 DEBUG "result": [ 07:03:09 DEBUG { 07:03:09 DEBUG "metric": { 07:03:09 DEBUG "instance": "10.142.0.107:9100" 07:03:09 DEBUG }, 07:03:09 DEBUG "value": [ 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG [...] 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG "instance": "10.142.0.43:9100" 07:03:09 DEBUG }, 07:03:09 DEBUG "value": [ 07:03:09 DEBUG 1713592989, 07:03:09 DEBUG "0.2071999999999775" 07:03:09 DEBUG ] 07:03:09 DEBUG } 07:03:09 DEBUG ] 07:03:09 DEBUG } 07:03:09 DEBUG } 07:03:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592989" 07:03:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592989" 07:03:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592989" 07:03:09 INFO [http_cmd]: http status code OK 07:03:09 DEBUG --- status code --- 07:03:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:03:09 DEBUG --- http response --- 07:03:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:03:09 DEBUG ----- output ----- 07:03:09 DEBUG { 07:03:09 DEBUG "status": "success", 07:03:09 DEBUG "data": { 07:03:09 DEBUG "resultType": "vector", 07:03:09 DEBUG "result": [ 07:03:09 DEBUG { 07:03:09 DEBUG "metric": { 07:03:09 DEBUG "pod": "am-767f6b67b5-9zw22" 07:03:09 DEBUG }, 07:03:09 DEBUG "value": [ 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG [...] 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG "pod": "am-767f6b67b5-vbltz" 07:03:09 DEBUG }, 07:03:09 DEBUG "value": [ 07:03:09 DEBUG 1713592989, 07:03:09 DEBUG "2662.9" 07:03:09 DEBUG ] 07:03:09 DEBUG } 07:03:09 DEBUG ] 07:03:09 DEBUG } 07:03:09 DEBUG } 07:03:09 INFO [http_cmd]: http status code OK 07:03:09 DEBUG --- status code --- 07:03:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:03:09 DEBUG --- http response --- 07:03:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 07:03:09 DEBUG ----- output ----- 07:03:09 DEBUG { 07:03:09 DEBUG "status": "success", 07:03:09 DEBUG "data": { 07:03:09 DEBUG "resultType": "vector", 07:03:09 DEBUG "result": [ 07:03:09 DEBUG { 07:03:09 DEBUG "metric": { 07:03:09 DEBUG "pod": "ds-cts-0" 07:03:09 DEBUG }, 07:03:09 DEBUG "value": [ 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG [...] 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG "pod": "ds-cts-2" 07:03:09 DEBUG }, 07:03:09 DEBUG "value": [ 07:03:09 DEBUG 1713592989, 07:03:09 DEBUG "0" 07:03:09 DEBUG ] 07:03:09 DEBUG } 07:03:09 DEBUG ] 07:03:09 DEBUG } 07:03:09 DEBUG } 07:03:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713592989" 07:03:09 INFO [http_cmd]: http status code OK 07:03:09 DEBUG --- status code --- 07:03:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:03:09 DEBUG --- http response --- 07:03:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:03:09 DEBUG ----- output ----- 07:03:09 DEBUG { 07:03:09 DEBUG "status": "success", 07:03:09 DEBUG "data": { 07:03:09 DEBUG "resultType": "vector", 07:03:09 DEBUG "result": [ 07:03:09 DEBUG { 07:03:09 DEBUG "metric": { 07:03:09 DEBUG "pod": "am-767f6b67b5-9zw22" 07:03:09 DEBUG }, 07:03:09 DEBUG "value": [ 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG [...] 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG "pod": "am-767f6b67b5-vbltz" 07:03:09 DEBUG }, 07:03:09 DEBUG "value": [ 07:03:09 DEBUG 1713592989, 07:03:09 DEBUG "2661.5" 07:03:09 DEBUG ] 07:03:09 DEBUG } 07:03:09 DEBUG ] 07:03:09 DEBUG } 07:03:09 DEBUG } 07:03:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592989" 07:03:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592989" 07:03:09 INFO [http_cmd]: http status code OK 07:03:09 DEBUG --- status code --- 07:03:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:03:09 DEBUG --- http response --- 07:03:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 07:03:09 DEBUG ----- output ----- 07:03:09 DEBUG { 07:03:09 DEBUG "status": "success", 07:03:09 DEBUG "data": { 07:03:09 DEBUG "resultType": "vector", 07:03:09 DEBUG "result": [ 07:03:09 DEBUG { 07:03:09 DEBUG "metric": { 07:03:09 DEBUG "pod": "ds-idrepo-0" 07:03:09 DEBUG }, 07:03:09 DEBUG "value": [ 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG [...] 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG "pod": "ds-cts-2" 07:03:09 DEBUG }, 07:03:09 DEBUG "value": [ 07:03:09 DEBUG 1713592989, 07:03:09 DEBUG "0" 07:03:09 DEBUG ] 07:03:09 DEBUG } 07:03:09 DEBUG ] 07:03:09 DEBUG } 07:03:09 DEBUG } 07:03:09 INFO [http_cmd]: http status code OK 07:03:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592989" 07:03:09 DEBUG --- status code --- 07:03:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:03:09 DEBUG --- http response --- 07:03:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:03:09 DEBUG ----- output ----- 07:03:09 DEBUG { 07:03:09 DEBUG "status": "success", 07:03:09 DEBUG "data": { 07:03:09 DEBUG "resultType": "vector", 07:03:09 DEBUG "result": [ 07:03:09 DEBUG { 07:03:09 DEBUG "metric": { 07:03:09 DEBUG "pod": "am-767f6b67b5-9zw22" 07:03:09 DEBUG }, 07:03:09 DEBUG "value": [ 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG [...] 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG "pod": "am-767f6b67b5-vbltz" 07:03:09 DEBUG }, 07:03:09 DEBUG "value": [ 07:03:09 DEBUG 1713592989, 07:03:09 DEBUG "2670.366666666667" 07:03:09 DEBUG ] 07:03:09 DEBUG } 07:03:09 DEBUG ] 07:03:09 DEBUG } 07:03:09 DEBUG } 07:03:09 INFO [http_cmd]: http status code OK 07:03:09 DEBUG --- status code --- 07:03:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:03:09 DEBUG --- http response --- 07:03:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:03:09 DEBUG ----- output ----- 07:03:09 DEBUG { 07:03:09 DEBUG "status": "success", 07:03:09 DEBUG "data": { 07:03:09 DEBUG "resultType": "vector", 07:03:09 DEBUG "result": [ 07:03:09 DEBUG { 07:03:09 DEBUG "metric": { 07:03:09 DEBUG "pod": "ds-idrepo-0" 07:03:09 DEBUG }, 07:03:09 DEBUG "value": [ 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG [...] 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG "pod": "ds-idrepo-2" 07:03:09 DEBUG }, 07:03:09 DEBUG "value": [ 07:03:09 DEBUG 1713592989, 07:03:09 DEBUG "0" 07:03:09 DEBUG ] 07:03:09 DEBUG } 07:03:09 DEBUG ] 07:03:09 DEBUG } 07:03:09 DEBUG } 07:03:09 INFO [http_cmd]: http status code OK 07:03:09 DEBUG --- status code --- 07:03:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:03:09 DEBUG --- http response --- 07:03:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 07:03:09 DEBUG ----- output ----- 07:03:09 DEBUG { 07:03:09 DEBUG "status": "success", 07:03:09 DEBUG "data": { 07:03:09 DEBUG "resultType": "vector", 07:03:09 DEBUG "result": [ 07:03:09 DEBUG { 07:03:09 DEBUG "metric": { 07:03:09 DEBUG "pod": "ds-cts-0" 07:03:09 DEBUG }, 07:03:09 DEBUG "value": [ 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG [...] 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG "pod": "ds-cts-2" 07:03:09 DEBUG }, 07:03:09 DEBUG "value": [ 07:03:09 DEBUG 1713592989, 07:03:09 DEBUG "0.00016666666666666663" 07:03:09 DEBUG ] 07:03:09 DEBUG } 07:03:09 DEBUG ] 07:03:09 DEBUG } 07:03:09 DEBUG } 07:03:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592989" 07:03:09 INFO [http_cmd]: http status code OK 07:03:09 DEBUG --- status code --- 07:03:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:03:09 DEBUG --- http response --- 07:03:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:03:09 DEBUG ----- output ----- 07:03:09 DEBUG { 07:03:09 DEBUG "status": "success", 07:03:09 DEBUG "data": { 07:03:09 DEBUG "resultType": "vector", 07:03:09 DEBUG "result": [ 07:03:09 DEBUG { 07:03:09 DEBUG "metric": { 07:03:09 DEBUG "instance": "10.142.0.107:9100" 07:03:09 DEBUG }, 07:03:09 DEBUG "value": [ 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG [...] 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG "instance": "10.142.0.43:9100" 07:03:09 DEBUG }, 07:03:09 DEBUG "value": [ 07:03:09 DEBUG 1713592989, 07:03:09 DEBUG "80537223.16666666" 07:03:09 DEBUG ] 07:03:09 DEBUG } 07:03:09 DEBUG ] 07:03:09 DEBUG } 07:03:09 DEBUG } 07:03:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713592989" 07:03:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592989" 07:03:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592989" 07:03:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713592989" 07:03:09 INFO [http_cmd]: http status code OK 07:03:09 INFO [http_cmd]: http status code OK 07:03:09 DEBUG --- status code --- 07:03:09 DEBUG --- status code --- 07:03:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:03:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:03:09 DEBUG --- http response --- 07:03:09 DEBUG --- http response --- 07:03:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:03:09 DEBUG ----- output ----- 07:03:09 DEBUG { 07:03:09 DEBUG "status": "success", 07:03:09 DEBUG "data": { 07:03:09 DEBUG "resultType": "vector", 07:03:09 DEBUG "result": [ 07:03:09 DEBUG { 07:03:09 DEBUG "metric": { 07:03:09 DEBUG "pod": "am-767f6b67b5-9zw22" 07:03:09 DEBUG }, 07:03:09 DEBUG "value": [ 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG [...] 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG "pod": "am-767f6b67b5-vbltz" 07:03:09 DEBUG }, 07:03:09 DEBUG "value": [ 07:03:09 DEBUG 1713592989, 07:03:09 DEBUG "0.097" 07:03:09 DEBUG ] 07:03:09 DEBUG } 07:03:09 DEBUG ] 07:03:09 DEBUG } 07:03:09 DEBUG } 07:03:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 07:03:09 DEBUG ----- output ----- 07:03:09 DEBUG { 07:03:09 DEBUG "status": "success", 07:03:09 DEBUG "data": { 07:03:09 DEBUG "resultType": "vector", 07:03:09 DEBUG "result": [ 07:03:09 DEBUG { 07:03:09 DEBUG "metric": { 07:03:09 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 07:03:09 DEBUG }, 07:03:09 DEBUG "value": [ 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG [...] 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG "pod": "ds-cts-1" 07:03:09 DEBUG }, 07:03:09 DEBUG "value": [ 07:03:09 DEBUG 1713592989, 07:03:09 DEBUG "31755.24961757753" 07:03:09 DEBUG ] 07:03:09 DEBUG } 07:03:09 DEBUG ] 07:03:09 DEBUG } 07:03:09 DEBUG } 07:03:09 INFO [http_cmd]: http status code OK 07:03:09 DEBUG --- status code --- 07:03:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:03:09 DEBUG --- http response --- 07:03:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:03:09 DEBUG ----- output ----- 07:03:09 DEBUG { 07:03:09 DEBUG "status": "success", 07:03:09 DEBUG "data": { 07:03:09 DEBUG "resultType": "vector", 07:03:09 DEBUG "result": [ 07:03:09 DEBUG { 07:03:09 DEBUG "metric": { 07:03:09 DEBUG "pod": "am-767f6b67b5-9zw22" 07:03:09 DEBUG }, 07:03:09 DEBUG "value": [ 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG [...] 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG "pod": "am-767f6b67b5-vbltz" 07:03:09 DEBUG }, 07:03:09 DEBUG "value": [ 07:03:09 DEBUG 1713592989, 07:03:09 DEBUG "0" 07:03:09 DEBUG ] 07:03:09 DEBUG } 07:03:09 DEBUG ] 07:03:09 DEBUG } 07:03:09 DEBUG } 07:03:09 INFO [http_cmd]: http status code OK 07:03:09 DEBUG --- status code --- 07:03:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:03:09 DEBUG --- http response --- 07:03:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:03:09 DEBUG ----- output ----- 07:03:09 DEBUG { 07:03:09 DEBUG "status": "success", 07:03:09 DEBUG "data": { 07:03:09 DEBUG "resultType": "vector", 07:03:09 DEBUG "result": [ 07:03:09 DEBUG { 07:03:09 DEBUG "metric": { 07:03:09 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 07:03:09 DEBUG }, 07:03:09 DEBUG "value": [ 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG [...] 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 07:03:09 DEBUG }, 07:03:09 DEBUG "value": [ 07:03:09 DEBUG 1713592989, 07:03:09 DEBUG "8.019012299044343" 07:03:09 DEBUG ] 07:03:09 DEBUG } 07:03:09 DEBUG ] 07:03:09 DEBUG } 07:03:09 DEBUG } 07:03:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592989" 07:03:09 INFO [http_cmd]: http status code OK 07:03:09 DEBUG --- status code --- 07:03:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:03:09 DEBUG --- http response --- 07:03:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:03:09 DEBUG ----- output ----- 07:03:09 DEBUG { 07:03:09 DEBUG "status": "success", 07:03:09 DEBUG "data": { 07:03:09 DEBUG "resultType": "vector", 07:03:09 DEBUG "result": [ 07:03:09 DEBUG { 07:03:09 DEBUG "metric": { 07:03:09 DEBUG "pod": "am-767f6b67b5-9zw22" 07:03:09 DEBUG }, 07:03:09 DEBUG "value": [ 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG [...] 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG "pod": "am-767f6b67b5-vbltz" 07:03:09 DEBUG }, 07:03:09 DEBUG "value": [ 07:03:09 DEBUG 1713592989, 07:03:09 DEBUG "2664.366666666667" 07:03:09 DEBUG ] 07:03:09 DEBUG } 07:03:09 DEBUG ] 07:03:09 DEBUG } 07:03:09 DEBUG } 07:03:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713592989" 07:03:09 INFO [http_cmd]: http status code OK 07:03:09 DEBUG --- status code --- 07:03:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:03:09 DEBUG --- http response --- 07:03:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 07:03:09 DEBUG ----- output ----- 07:03:09 DEBUG { 07:03:09 DEBUG "status": "success", 07:03:09 DEBUG "data": { 07:03:09 DEBUG "resultType": "vector", 07:03:09 DEBUG "result": [ 07:03:09 DEBUG { 07:03:09 DEBUG "metric": { 07:03:09 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 07:03:09 DEBUG }, 07:03:09 DEBUG "value": [ 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG [...] 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG "pod": "ds-cts-1" 07:03:09 DEBUG }, 07:03:09 DEBUG "value": [ 07:03:09 DEBUG 1713592989, 07:03:09 DEBUG "17.626781150050583" 07:03:09 DEBUG ] 07:03:09 DEBUG } 07:03:09 DEBUG ] 07:03:09 DEBUG } 07:03:09 DEBUG } 07:03:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592989" 07:03:09 INFO [http_cmd]: http status code OK 07:03:09 DEBUG --- status code --- 07:03:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:03:09 DEBUG --- http response --- 07:03:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:03:09 DEBUG ----- output ----- 07:03:09 DEBUG { 07:03:09 DEBUG "status": "success", 07:03:09 DEBUG "data": { 07:03:09 DEBUG "resultType": "vector", 07:03:09 DEBUG "result": [ 07:03:09 DEBUG { 07:03:09 DEBUG "metric": { 07:03:09 DEBUG "instance": "10.142.0.107:9100" 07:03:09 DEBUG }, 07:03:09 DEBUG "value": [ 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG [...] 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG "instance": "10.142.0.43:9100" 07:03:09 DEBUG }, 07:03:09 DEBUG "value": [ 07:03:09 DEBUG 1713592989, 07:03:09 DEBUG "80537754.63333333" 07:03:09 DEBUG ] 07:03:09 DEBUG } 07:03:09 DEBUG ] 07:03:09 DEBUG } 07:03:09 DEBUG } 07:03:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713592989" 07:03:09 INFO [http_cmd]: http status code OK 07:03:09 DEBUG --- status code --- 07:03:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:03:09 DEBUG --- http response --- 07:03:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:03:09 DEBUG ----- output ----- 07:03:09 DEBUG { 07:03:09 DEBUG "status": "success", 07:03:09 DEBUG "data": { 07:03:09 DEBUG "resultType": "vector", 07:03:09 DEBUG "result": [ 07:03:09 DEBUG { 07:03:09 DEBUG "metric": { 07:03:09 DEBUG "instance": "10.142.0.107:9100" 07:03:09 DEBUG }, 07:03:09 DEBUG "value": [ 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG [...] 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG 07:03:09 DEBUG "instance": "10.142.0.43:9100" 07:03:09 DEBUG }, 07:03:09 DEBUG "value": [ 07:03:09 DEBUG 1713592989, 07:03:09 DEBUG "0.08033333333333229" 07:03:09 DEBUG ] 07:03:09 DEBUG } 07:03:09 DEBUG ] 07:03:09 DEBUG } 07:03:09 DEBUG } 07:03:11 INFO 07:03:11 INFO [loop_until]: kubectl --namespace=xlou top node 07:03:11 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 07:03:11 INFO [loop_until]: OK (rc = 0) 07:03:11 DEBUG --- stdout --- 07:03:11 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 69m 0% 6281Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7687m 25% 7475Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 17091m 57% 11371Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 81m 0% 3365Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 18740m 62% 13626Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 159m 0% 5208Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 17375m 58% 11600Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 19050m 63% 2998Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 19317m 64% 10866Mi 9% gke-xlou-cdm-ds-0f601751-j488 8164m 27% 7040Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 17225m 57% 15592Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 8045m 26% 7700Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 18845m 63% 12896Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 8179m 51% 8493Mi 14% 07:03:11 DEBUG --- stderr --- 07:03:11 DEBUG 07:03:11 INFO 07:03:11 INFO [loop_until]: kubectl --namespace=xlou top pods 07:03:11 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 07:03:11 INFO [loop_until]: OK (rc = 0) 07:03:11 DEBUG --- stdout --- 07:03:11 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 16779m 8652Mi am-767f6b67b5-qcqz2 17339m 8713Mi am-767f6b67b5-vbltz 17978m 8655Mi ds-cts-0 18236m 10477Mi ds-cts-1 17355m 10674Mi ds-cts-2 18702m 10300Mi ds-idrepo-0 7570m 3787Mi ds-idrepo-1 7660m 3803Mi ds-idrepo-2 7981m 3711Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1177Mi idm-b85f9d978-wnhxb 3m 1236Mi lodemon-7cb5b9456d-dskfn 1m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 8160m 4691Mi 07:03:11 DEBUG --- stderr --- 07:03:11 DEBUG 07:04:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593049" 07:04:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593049" 07:04:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593049" 07:04:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593049" 07:04:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593049" 07:04:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593049" 07:04:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713593049" 07:04:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593049" 07:04:09 INFO [http_cmd]: http status code OK 07:04:09 INFO [http_cmd]: http status code OK 07:04:09 INFO [http_cmd]: http status code OK 07:04:09 INFO [http_cmd]: http status code OK 07:04:09 DEBUG --- status code --- 07:04:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713593049" 07:04:09 INFO [http_cmd]: http status code OK 07:04:09 INFO [http_cmd]: http status code OK 07:04:09 DEBUG --- status code --- 07:04:09 DEBUG --- status code --- 07:04:09 DEBUG --- status code --- 07:04:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:04:09 DEBUG --- status code --- 07:04:09 DEBUG --- status code --- 07:04:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:04:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:04:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:04:09 DEBUG --- http response --- 07:04:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:04:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:04:09 DEBUG --- http response --- 07:04:09 DEBUG --- http response --- 07:04:09 DEBUG --- http response --- 07:04:09 DEBUG --- http response --- 07:04:09 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 07:04:09 DEBUG --- http response --- 07:04:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 07:04:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 07:04:09 DEBUG ----- output ----- 07:04:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 07:04:09 DEBUG ----- output ----- 07:04:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:04:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:04:09 DEBUG { 07:04:09 DEBUG ----- output ----- 07:04:09 DEBUG { 07:04:09 DEBUG ----- output ----- 07:04:09 DEBUG ----- output ----- 07:04:09 DEBUG "status": "success", 07:04:09 DEBUG { 07:04:09 DEBUG "status": "success", 07:04:09 DEBUG { 07:04:09 DEBUG { 07:04:09 DEBUG "data": { 07:04:09 DEBUG "status": "success", 07:04:09 DEBUG "data": { 07:04:09 DEBUG "status": "success", 07:04:09 DEBUG "status": "success", 07:04:09 DEBUG "resultType": "vector", 07:04:09 DEBUG "data": { 07:04:09 DEBUG "resultType": "vector", 07:04:09 DEBUG "data": { 07:04:09 DEBUG "data": { 07:04:09 DEBUG "result": [ 07:04:09 DEBUG "resultType": "vector", 07:04:09 DEBUG "result": [ 07:04:09 DEBUG "resultType": "vector", 07:04:09 DEBUG "resultType": "vector", 07:04:09 DEBUG { 07:04:09 DEBUG "result": [ 07:04:09 DEBUG { 07:04:09 DEBUG "result": [ 07:04:09 DEBUG "result": [ 07:04:09 DEBUG "metric": { 07:04:09 DEBUG { 07:04:09 DEBUG "metric": { 07:04:09 DEBUG { 07:04:09 DEBUG { 07:04:09 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 07:04:09 DEBUG "metric": { 07:04:09 DEBUG "pod": "ds-cts-0" 07:04:09 DEBUG "metric": { 07:04:09 DEBUG "metric": { 07:04:09 DEBUG }, 07:04:09 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 07:04:09 DEBUG }, 07:04:09 DEBUG "pod": "ds-idrepo-0" 07:04:09 DEBUG "pod": "ds-idrepo-0" 07:04:09 DEBUG "value": [ 07:04:09 DEBUG }, 07:04:09 DEBUG "value": [ 07:04:09 DEBUG }, 07:04:09 DEBUG }, 07:04:09 DEBUG 07:04:09 DEBUG "value": [ 07:04:09 DEBUG 07:04:09 DEBUG "value": [ 07:04:09 DEBUG "value": [ 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG [...] 07:04:09 DEBUG 07:04:09 DEBUG [...] 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG [...] 07:04:09 DEBUG 07:04:09 DEBUG [...] 07:04:09 DEBUG [...] 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 07:04:09 DEBUG 07:04:09 DEBUG "pod": "ds-cts-2" 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG }, 07:04:09 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 07:04:09 DEBUG }, 07:04:09 DEBUG "pod": "ds-idrepo-2" 07:04:09 DEBUG "pod": "ds-idrepo-2" 07:04:09 DEBUG "value": [ 07:04:09 DEBUG }, 07:04:09 DEBUG "value": [ 07:04:09 DEBUG }, 07:04:09 DEBUG }, 07:04:09 DEBUG 1713593049, 07:04:09 DEBUG "value": [ 07:04:09 DEBUG 1713593049, 07:04:09 DEBUG "value": [ 07:04:09 DEBUG "value": [ 07:04:09 DEBUG "247519.70071803537" 07:04:09 DEBUG 1713593049, 07:04:09 DEBUG "30" 07:04:09 DEBUG 1713593049, 07:04:09 DEBUG 1713593049, 07:04:09 DEBUG ] 07:04:09 DEBUG "0" 07:04:09 DEBUG ] 07:04:09 DEBUG "0" 07:04:09 DEBUG "0" 07:04:09 DEBUG } 07:04:09 DEBUG ] 07:04:09 DEBUG } 07:04:09 DEBUG ] 07:04:09 DEBUG ] 07:04:09 DEBUG ] 07:04:09 DEBUG } 07:04:09 DEBUG ] 07:04:09 DEBUG } 07:04:09 DEBUG } 07:04:09 DEBUG } 07:04:09 DEBUG ] 07:04:09 DEBUG } 07:04:09 DEBUG ] 07:04:09 DEBUG ] 07:04:09 DEBUG } 07:04:09 DEBUG } 07:04:09 DEBUG } 07:04:09 DEBUG } 07:04:09 DEBUG } 07:04:09 DEBUG } 07:04:09 DEBUG } 07:04:09 DEBUG } 07:04:09 INFO [http_cmd]: http status code OK 07:04:09 ERROR [http_cmd]: ERROR 07:04:09 DEBUG --- status code --- 07:04:09 INFO [http_cmd]: http status code OK 07:04:09 ERROR --- status code --- 07:04:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:04:09 DEBUG --- status code --- 07:04:09 ERROR http status code is 400 (expected [200, 401, 403, 404]) 07:04:09 DEBUG --- http response --- 07:04:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:04:09 ERROR --- http response --- 07:04:09 DEBUG --- http response --- 07:04:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:04:09 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 07:04:09 DEBUG ----- output ----- 07:04:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:04:09 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 07:04:09 DEBUG { 07:04:09 DEBUG ----- output ----- 07:04:09 DEBUG "status": "success", 07:04:09 DEBUG { 07:04:09 DEBUG "data": { 07:04:09 DEBUG "status": "success", 07:04:09 DEBUG "resultType": "vector", 07:04:09 DEBUG "data": { 07:04:09 DEBUG "result": [ 07:04:09 DEBUG "resultType": "vector", 07:04:09 DEBUG { 07:04:09 DEBUG "result": [ 07:04:09 DEBUG "metric": { 07:04:09 DEBUG { 07:04:09 DEBUG "pod": "am-767f6b67b5-9zw22" 07:04:09 DEBUG "metric": { 07:04:09 DEBUG }, 07:04:09 DEBUG "instance": "10.142.0.107:9100" 07:04:09 DEBUG "value": [ 07:04:09 DEBUG }, 07:04:09 DEBUG 07:04:09 DEBUG "value": [ 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG [...] 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG [...] 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG "pod": "am-767f6b67b5-vbltz" 07:04:09 DEBUG 07:04:09 DEBUG }, 07:04:09 DEBUG "instance": "10.142.0.43:9100" 07:04:09 DEBUG "value": [ 07:04:09 DEBUG }, 07:04:09 DEBUG 1713593049, 07:04:09 DEBUG "value": [ 07:04:09 DEBUG "480131.5" 07:04:09 DEBUG 1713593049, 07:04:09 DEBUG ] 07:04:09 DEBUG "3282962568192" 07:04:09 DEBUG } 07:04:09 DEBUG ] 07:04:09 DEBUG ] 07:04:09 DEBUG } 07:04:09 DEBUG } 07:04:09 DEBUG ] 07:04:09 DEBUG } 07:04:09 DEBUG } 07:04:09 DEBUG } 07:04:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713593049" 07:04:09 INFO [http_cmd]: http status code OK 07:04:09 DEBUG --- status code --- 07:04:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:04:09 DEBUG --- http response --- 07:04:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:04:09 DEBUG ----- output ----- 07:04:09 DEBUG { 07:04:09 DEBUG "status": "success", 07:04:09 DEBUG "data": { 07:04:09 DEBUG "resultType": "vector", 07:04:09 DEBUG "result": [ 07:04:09 DEBUG { 07:04:09 DEBUG "metric": { 07:04:09 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 07:04:09 DEBUG }, 07:04:09 DEBUG "value": [ 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG [...] 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 07:04:09 DEBUG }, 07:04:09 DEBUG "value": [ 07:04:09 DEBUG 1713593049, 07:04:09 DEBUG "5015240704" 07:04:09 DEBUG ] 07:04:09 DEBUG } 07:04:09 DEBUG ] 07:04:09 DEBUG } 07:04:09 DEBUG } 07:04:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713593049" 07:04:09 INFO [http_cmd]: http status code OK 07:04:09 DEBUG --- status code --- 07:04:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:04:09 DEBUG --- http response --- 07:04:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:04:09 DEBUG ----- output ----- 07:04:09 DEBUG { 07:04:09 DEBUG "status": "success", 07:04:09 DEBUG "data": { 07:04:09 DEBUG "resultType": "vector", 07:04:09 DEBUG "result": [ 07:04:09 DEBUG { 07:04:09 DEBUG "metric": { 07:04:09 DEBUG "instance": "10.142.0.107:9100" 07:04:09 DEBUG }, 07:04:09 DEBUG "value": [ 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG [...] 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG "instance": "10.142.0.43:9100" 07:04:09 DEBUG }, 07:04:09 DEBUG "value": [ 07:04:09 DEBUG 1713593049, 07:04:09 DEBUG "0.15136666666664855" 07:04:09 DEBUG ] 07:04:09 DEBUG } 07:04:09 DEBUG ] 07:04:09 DEBUG } 07:04:09 DEBUG } 07:04:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593049" 07:04:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713593049" 07:04:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593049" 07:04:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593049" 07:04:09 INFO [http_cmd]: http status code OK 07:04:09 DEBUG --- status code --- 07:04:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:04:09 DEBUG --- http response --- 07:04:09 INFO [http_cmd]: http status code OK 07:04:09 INFO [http_cmd]: http status code OK 07:04:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:04:09 DEBUG --- status code --- 07:04:09 DEBUG --- status code --- 07:04:09 DEBUG ----- output ----- 07:04:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:04:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:04:09 DEBUG { 07:04:09 DEBUG --- http response --- 07:04:09 DEBUG --- http response --- 07:04:09 DEBUG "status": "success", 07:04:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 07:04:09 DEBUG "data": { 07:04:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 07:04:09 DEBUG ----- output ----- 07:04:09 DEBUG "resultType": "vector", 07:04:09 DEBUG ----- output ----- 07:04:09 DEBUG { 07:04:09 DEBUG "result": [ 07:04:09 DEBUG { 07:04:09 DEBUG "status": "success", 07:04:09 DEBUG { 07:04:09 DEBUG "status": "success", 07:04:09 DEBUG "data": { 07:04:09 DEBUG "metric": { 07:04:09 DEBUG "data": { 07:04:09 DEBUG "resultType": "vector", 07:04:09 DEBUG "pod": "am-767f6b67b5-9zw22" 07:04:09 DEBUG "resultType": "vector", 07:04:09 DEBUG "result": [ 07:04:09 DEBUG }, 07:04:09 DEBUG "result": [ 07:04:09 DEBUG { 07:04:09 DEBUG "value": [ 07:04:09 DEBUG { 07:04:09 DEBUG "metric": { 07:04:09 DEBUG 07:04:09 DEBUG "metric": { 07:04:09 DEBUG "pod": "ds-cts-0" 07:04:09 DEBUG 07:04:09 DEBUG "pod": "ds-idrepo-0" 07:04:09 DEBUG }, 07:04:09 DEBUG 07:04:09 DEBUG }, 07:04:09 DEBUG "value": [ 07:04:09 DEBUG [...] 07:04:09 DEBUG "value": [ 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG [...] 07:04:09 DEBUG "pod": "am-767f6b67b5-vbltz" 07:04:09 DEBUG [...] 07:04:09 DEBUG 07:04:09 DEBUG }, 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG "value": [ 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG 1713593049, 07:04:09 DEBUG 07:04:09 DEBUG "pod": "ds-cts-2" 07:04:09 DEBUG "2664.4666666666667" 07:04:09 DEBUG "pod": "ds-cts-2" 07:04:09 DEBUG }, 07:04:09 DEBUG ] 07:04:09 DEBUG }, 07:04:09 DEBUG "value": [ 07:04:09 DEBUG } 07:04:09 DEBUG "value": [ 07:04:09 DEBUG 1713593049, 07:04:09 DEBUG ] 07:04:09 DEBUG 1713593049, 07:04:09 DEBUG "0.0004991088888888889" 07:04:09 DEBUG } 07:04:09 DEBUG "0" 07:04:09 DEBUG ] 07:04:09 DEBUG } 07:04:09 DEBUG ] 07:04:09 DEBUG } 07:04:09 INFO [http_cmd]: http status code OK 07:04:09 DEBUG } 07:04:09 DEBUG ] 07:04:09 DEBUG --- status code --- 07:04:09 DEBUG ] 07:04:09 DEBUG } 07:04:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:04:09 DEBUG } 07:04:09 DEBUG } 07:04:09 DEBUG --- http response --- 07:04:09 DEBUG } 07:04:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 07:04:09 DEBUG ----- output ----- 07:04:09 DEBUG { 07:04:09 DEBUG "status": "success", 07:04:09 DEBUG "data": { 07:04:09 DEBUG "resultType": "vector", 07:04:09 DEBUG "result": [ 07:04:09 DEBUG { 07:04:09 DEBUG "metric": { 07:04:09 DEBUG "pod": "ds-cts-0" 07:04:09 DEBUG }, 07:04:09 DEBUG "value": [ 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG [...] 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG "pod": "ds-cts-2" 07:04:09 DEBUG }, 07:04:09 DEBUG "value": [ 07:04:09 DEBUG 1713593049, 07:04:09 DEBUG "0" 07:04:09 DEBUG ] 07:04:09 DEBUG } 07:04:09 DEBUG ] 07:04:09 DEBUG } 07:04:09 DEBUG } 07:04:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593049" 07:04:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593049" 07:04:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593049" 07:04:09 INFO [http_cmd]: http status code OK 07:04:09 INFO [http_cmd]: http status code OK 07:04:09 DEBUG --- status code --- 07:04:09 DEBUG --- status code --- 07:04:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:04:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:04:09 DEBUG --- http response --- 07:04:09 DEBUG --- http response --- 07:04:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:04:09 DEBUG ----- output ----- 07:04:09 DEBUG { 07:04:09 DEBUG "status": "success", 07:04:09 DEBUG "data": { 07:04:09 DEBUG "resultType": "vector", 07:04:09 DEBUG "result": [ 07:04:09 DEBUG { 07:04:09 DEBUG "metric": { 07:04:09 DEBUG "pod": "am-767f6b67b5-9zw22" 07:04:09 DEBUG }, 07:04:09 DEBUG "value": [ 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG [...] 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG "pod": "am-767f6b67b5-vbltz" 07:04:09 DEBUG }, 07:04:09 DEBUG "value": [ 07:04:09 DEBUG 1713593049, 07:04:09 DEBUG "2669.3" 07:04:09 DEBUG ] 07:04:09 DEBUG } 07:04:09 DEBUG ] 07:04:09 DEBUG } 07:04:09 DEBUG } 07:04:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:04:09 DEBUG ----- output ----- 07:04:09 DEBUG { 07:04:09 DEBUG "status": "success", 07:04:09 DEBUG "data": { 07:04:09 DEBUG "resultType": "vector", 07:04:09 DEBUG "result": [ 07:04:09 DEBUG { 07:04:09 DEBUG "metric": { 07:04:09 DEBUG "pod": "ds-idrepo-0" 07:04:09 DEBUG }, 07:04:09 DEBUG "value": [ 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG [...] 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG "pod": "ds-idrepo-2" 07:04:09 DEBUG }, 07:04:09 DEBUG "value": [ 07:04:09 DEBUG 1713593049, 07:04:09 DEBUG "0" 07:04:09 DEBUG ] 07:04:09 DEBUG } 07:04:09 DEBUG ] 07:04:09 DEBUG } 07:04:09 DEBUG } 07:04:09 INFO [http_cmd]: http status code OK 07:04:09 DEBUG --- status code --- 07:04:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:04:09 DEBUG --- http response --- 07:04:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:04:09 DEBUG ----- output ----- 07:04:09 DEBUG { 07:04:09 DEBUG "status": "success", 07:04:09 DEBUG "data": { 07:04:09 DEBUG "resultType": "vector", 07:04:09 DEBUG "result": [ 07:04:09 DEBUG { 07:04:09 DEBUG "metric": { 07:04:09 DEBUG "pod": "am-767f6b67b5-9zw22" 07:04:09 DEBUG }, 07:04:09 DEBUG "value": [ 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG [...] 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG "pod": "am-767f6b67b5-vbltz" 07:04:09 DEBUG }, 07:04:09 DEBUG "value": [ 07:04:09 DEBUG 1713593049, 07:04:09 DEBUG "2663.866666666667" 07:04:09 DEBUG ] 07:04:09 DEBUG } 07:04:09 DEBUG ] 07:04:09 DEBUG } 07:04:09 DEBUG } 07:04:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713593049" 07:04:09 INFO [http_cmd]: http status code OK 07:04:09 DEBUG --- status code --- 07:04:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:04:09 DEBUG --- http response --- 07:04:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:04:09 DEBUG ----- output ----- 07:04:09 DEBUG { 07:04:09 DEBUG "status": "success", 07:04:09 DEBUG "data": { 07:04:09 DEBUG "resultType": "vector", 07:04:09 DEBUG "result": [ 07:04:09 DEBUG { 07:04:09 DEBUG "metric": { 07:04:09 DEBUG "instance": "10.142.0.107:9100" 07:04:09 DEBUG }, 07:04:09 DEBUG "value": [ 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG [...] 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG "instance": "10.142.0.43:9100" 07:04:09 DEBUG }, 07:04:09 DEBUG "value": [ 07:04:09 DEBUG 1713593049, 07:04:09 DEBUG "61591773.166666664" 07:04:09 DEBUG ] 07:04:09 DEBUG } 07:04:09 DEBUG ] 07:04:09 DEBUG } 07:04:09 DEBUG } 07:04:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713593049" 07:04:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593049" 07:04:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593049" 07:04:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593049" 07:04:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713593049" 07:04:09 INFO [http_cmd]: http status code OK 07:04:09 DEBUG --- status code --- 07:04:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:04:09 DEBUG --- http response --- 07:04:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:04:09 DEBUG ----- output ----- 07:04:09 DEBUG { 07:04:09 DEBUG "status": "success", 07:04:09 DEBUG "data": { 07:04:09 DEBUG "resultType": "vector", 07:04:09 DEBUG "result": [ 07:04:09 DEBUG { 07:04:09 DEBUG "metric": { 07:04:09 DEBUG "pod": "am-767f6b67b5-9zw22" 07:04:09 DEBUG }, 07:04:09 DEBUG "value": [ 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG [...] 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG "pod": "am-767f6b67b5-vbltz" 07:04:09 DEBUG }, 07:04:09 DEBUG "value": [ 07:04:09 DEBUG 1713593049, 07:04:09 DEBUG "0.097" 07:04:09 DEBUG ] 07:04:09 DEBUG } 07:04:09 DEBUG ] 07:04:09 DEBUG } 07:04:09 DEBUG } 07:04:09 INFO [http_cmd]: http status code OK 07:04:09 DEBUG --- status code --- 07:04:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:04:09 DEBUG --- http response --- 07:04:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 07:04:09 DEBUG ----- output ----- 07:04:09 DEBUG { 07:04:09 DEBUG "status": "success", 07:04:09 DEBUG "data": { 07:04:09 DEBUG "resultType": "vector", 07:04:09 DEBUG "result": [ 07:04:09 DEBUG { 07:04:09 DEBUG "metric": { 07:04:09 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 07:04:09 DEBUG }, 07:04:09 DEBUG "value": [ 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG [...] 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 07:04:09 DEBUG }, 07:04:09 DEBUG "value": [ 07:04:09 DEBUG 1713593049, 07:04:09 DEBUG "61752216.470590636" 07:04:09 DEBUG ] 07:04:09 DEBUG } 07:04:09 DEBUG ] 07:04:09 DEBUG } 07:04:09 DEBUG } 07:04:09 INFO [http_cmd]: http status code OK 07:04:09 DEBUG --- status code --- 07:04:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:04:09 DEBUG --- http response --- 07:04:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 07:04:09 DEBUG ----- output ----- 07:04:09 DEBUG { 07:04:09 DEBUG "status": "success", 07:04:09 DEBUG "data": { 07:04:09 DEBUG "resultType": "vector", 07:04:09 DEBUG "result": [ 07:04:09 DEBUG { 07:04:09 DEBUG "metric": { 07:04:09 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 07:04:09 DEBUG }, 07:04:09 DEBUG "value": [ 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG [...] 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 07:04:09 DEBUG }, 07:04:09 DEBUG "value": [ 07:04:09 DEBUG 1713593049, 07:04:09 DEBUG "13.644467872700982" 07:04:09 DEBUG ] 07:04:09 DEBUG } 07:04:09 DEBUG ] 07:04:09 DEBUG } 07:04:09 DEBUG } 07:04:09 INFO [http_cmd]: http status code OK 07:04:09 DEBUG --- status code --- 07:04:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:04:09 DEBUG --- http response --- 07:04:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:04:09 DEBUG ----- output ----- 07:04:09 DEBUG { 07:04:09 DEBUG "status": "success", 07:04:09 DEBUG "data": { 07:04:09 DEBUG "resultType": "vector", 07:04:09 DEBUG "result": [ 07:04:09 DEBUG { 07:04:09 DEBUG "metric": { 07:04:09 DEBUG "pod": "am-767f6b67b5-9zw22" 07:04:09 DEBUG }, 07:04:09 DEBUG "value": [ 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG [...] 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG "pod": "am-767f6b67b5-vbltz" 07:04:09 DEBUG }, 07:04:09 DEBUG "value": [ 07:04:09 DEBUG 1713593049, 07:04:09 DEBUG "0" 07:04:09 DEBUG ] 07:04:09 DEBUG } 07:04:09 DEBUG ] 07:04:09 DEBUG } 07:04:09 DEBUG } 07:04:09 INFO [http_cmd]: http status code OK 07:04:09 DEBUG --- status code --- 07:04:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:04:09 DEBUG --- http response --- 07:04:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:04:09 DEBUG ----- output ----- 07:04:09 DEBUG { 07:04:09 DEBUG "status": "success", 07:04:09 DEBUG "data": { 07:04:09 DEBUG "resultType": "vector", 07:04:09 DEBUG "result": [ 07:04:09 DEBUG { 07:04:09 DEBUG "metric": { 07:04:09 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 07:04:09 DEBUG }, 07:04:09 DEBUG "value": [ 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG [...] 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 07:04:09 DEBUG }, 07:04:09 DEBUG "value": [ 07:04:09 DEBUG 1713593049, 07:04:09 DEBUG "8.039308744722256" 07:04:09 DEBUG ] 07:04:09 DEBUG } 07:04:09 DEBUG ] 07:04:09 DEBUG } 07:04:09 DEBUG } 07:04:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593049" 07:04:09 INFO [http_cmd]: http status code OK 07:04:09 DEBUG --- status code --- 07:04:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:04:09 DEBUG --- http response --- 07:04:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:04:09 DEBUG ----- output ----- 07:04:09 DEBUG { 07:04:09 DEBUG "status": "success", 07:04:09 DEBUG "data": { 07:04:09 DEBUG "resultType": "vector", 07:04:09 DEBUG "result": [ 07:04:09 DEBUG { 07:04:09 DEBUG "metric": { 07:04:09 DEBUG "pod": "am-767f6b67b5-9zw22" 07:04:09 DEBUG }, 07:04:09 DEBUG "value": [ 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG [...] 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG "pod": "am-767f6b67b5-vbltz" 07:04:09 DEBUG }, 07:04:09 DEBUG "value": [ 07:04:09 DEBUG 1713593049, 07:04:09 DEBUG "2665.133333333333" 07:04:09 DEBUG ] 07:04:09 DEBUG } 07:04:09 DEBUG ] 07:04:09 DEBUG } 07:04:09 DEBUG } 07:04:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713593049" 07:04:09 INFO [http_cmd]: http status code OK 07:04:09 DEBUG --- status code --- 07:04:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:04:09 DEBUG --- http response --- 07:04:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:04:09 DEBUG ----- output ----- 07:04:09 DEBUG { 07:04:09 DEBUG "status": "success", 07:04:09 DEBUG "data": { 07:04:09 DEBUG "resultType": "vector", 07:04:09 DEBUG "result": [ 07:04:09 DEBUG { 07:04:09 DEBUG "metric": { 07:04:09 DEBUG "instance": "10.142.0.107:9100" 07:04:09 DEBUG }, 07:04:09 DEBUG "value": [ 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG [...] 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG "instance": "10.142.0.43:9100" 07:04:09 DEBUG }, 07:04:09 DEBUG "value": [ 07:04:09 DEBUG 1713593049, 07:04:09 DEBUG "61592489.03333333" 07:04:09 DEBUG ] 07:04:09 DEBUG } 07:04:09 DEBUG ] 07:04:09 DEBUG } 07:04:09 DEBUG } 07:04:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713593049" 07:04:09 INFO [http_cmd]: http status code OK 07:04:09 DEBUG --- status code --- 07:04:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:04:09 DEBUG --- http response --- 07:04:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:04:09 DEBUG ----- output ----- 07:04:09 DEBUG { 07:04:09 DEBUG "status": "success", 07:04:09 DEBUG "data": { 07:04:09 DEBUG "resultType": "vector", 07:04:09 DEBUG "result": [ 07:04:09 DEBUG { 07:04:09 DEBUG "metric": { 07:04:09 DEBUG "instance": "10.142.0.107:9100" 07:04:09 DEBUG }, 07:04:09 DEBUG "value": [ 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG [...] 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG 07:04:09 DEBUG "instance": "10.142.0.43:9100" 07:04:09 DEBUG }, 07:04:09 DEBUG "value": [ 07:04:09 DEBUG 1713593049, 07:04:09 DEBUG "0.06466666666665902" 07:04:09 DEBUG ] 07:04:09 DEBUG } 07:04:09 DEBUG ] 07:04:09 DEBUG } 07:04:09 DEBUG } 07:04:11 INFO 07:04:11 INFO [loop_until]: kubectl --namespace=xlou top node 07:04:11 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 07:04:11 INFO [loop_until]: OK (rc = 0) 07:04:11 DEBUG --- stdout --- 07:04:11 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 69m 0% 6273Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7976m 26% 7460Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 17090m 57% 11367Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 78m 0% 3353Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 18683m 62% 13626Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 139m 0% 5199Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 17344m 58% 11595Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 19073m 63% 3009Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 19421m 65% 10861Mi 9% gke-xlou-cdm-ds-0f601751-j488 8137m 27% 7041Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 17050m 57% 15600Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 8024m 26% 7701Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 19649m 65% 12896Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 8183m 51% 8495Mi 14% 07:04:11 DEBUG --- stderr --- 07:04:11 DEBUG 07:04:11 INFO 07:04:11 INFO [loop_until]: kubectl --namespace=xlou top pods 07:04:11 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 07:04:11 INFO [loop_until]: OK (rc = 0) 07:04:11 DEBUG --- stdout --- 07:04:11 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 17302m 8651Mi am-767f6b67b5-qcqz2 17120m 8717Mi am-767f6b67b5-vbltz 18964m 8654Mi ds-cts-0 18633m 10478Mi ds-cts-1 17014m 10682Mi ds-cts-2 19574m 10259Mi ds-idrepo-0 7763m 3787Mi ds-idrepo-1 7533m 3803Mi ds-idrepo-2 7770m 3712Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1177Mi idm-b85f9d978-wnhxb 3m 1236Mi lodemon-7cb5b9456d-dskfn 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7790m 4692Mi 07:04:11 DEBUG --- stderr --- 07:04:11 DEBUG 07:04:22 WARNING File /tmp/lodemon_data-20240420_063158/node_disk_read_bytes_total.json does not exist. Skipping average value computation... 127.0.0.1 - - [20/Apr/2024 07:04:22] "GET /monitoring/average?start_time=20240420_053510&stop_time=20240420_060221 HTTP/1.1" 200 - 07:05:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593109" 07:05:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593109" 07:05:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593109" 07:05:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593109" 07:05:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593109" 07:05:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593109" 07:05:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593109" 07:05:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713593109" 07:05:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713593109" 07:05:09 INFO [http_cmd]: http status code OK 07:05:09 DEBUG --- status code --- 07:05:09 INFO [http_cmd]: http status code OK 07:05:09 INFO [http_cmd]: http status code OK 07:05:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:05:09 DEBUG --- status code --- 07:05:09 DEBUG --- status code --- 07:05:09 DEBUG --- http response --- 07:05:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:05:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:05:09 DEBUG --- http response --- 07:05:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 07:05:09 DEBUG --- http response --- 07:05:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:05:09 DEBUG ----- output ----- 07:05:09 DEBUG ----- output ----- 07:05:09 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 07:05:09 DEBUG { 07:05:09 DEBUG { 07:05:09 DEBUG "status": "success", 07:05:09 DEBUG "status": "success", 07:05:09 DEBUG "data": { 07:05:09 DEBUG "data": { 07:05:09 DEBUG "resultType": "vector", 07:05:09 DEBUG "resultType": "vector", 07:05:09 DEBUG "result": [ 07:05:09 DEBUG "result": [ 07:05:09 DEBUG { 07:05:09 DEBUG { 07:05:09 DEBUG "metric": { 07:05:09 DEBUG "metric": { 07:05:09 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 07:05:09 ERROR [http_cmd]: ERROR 07:05:09 INFO [http_cmd]: http status code OK 07:05:09 DEBUG "pod": "am-767f6b67b5-9zw22" 07:05:09 INFO [http_cmd]: http status code OK 07:05:09 DEBUG }, 07:05:09 ERROR --- status code --- 07:05:09 DEBUG --- status code --- 07:05:09 DEBUG }, 07:05:09 DEBUG --- status code --- 07:05:09 DEBUG "value": [ 07:05:09 ERROR http status code is 400 (expected [200, 401, 403, 404]) 07:05:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:05:09 DEBUG "value": [ 07:05:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:05:09 DEBUG 07:05:09 ERROR --- http response --- 07:05:09 DEBUG --- http response --- 07:05:09 DEBUG 07:05:09 DEBUG --- http response --- 07:05:09 DEBUG 07:05:09 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 07:05:09 DEBUG 07:05:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 07:05:09 DEBUG 07:05:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 07:05:09 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 07:05:09 DEBUG 07:05:09 DEBUG ----- output ----- 07:05:09 DEBUG [...] 07:05:09 DEBUG ----- output ----- 07:05:09 DEBUG [...] 07:05:09 DEBUG { 07:05:09 DEBUG 07:05:09 DEBUG { 07:05:09 DEBUG 07:05:09 DEBUG "status": "success", 07:05:09 DEBUG 07:05:09 DEBUG "status": "success", 07:05:09 DEBUG 07:05:09 DEBUG "data": { 07:05:09 DEBUG 07:05:09 DEBUG "data": { 07:05:09 DEBUG 07:05:09 DEBUG "resultType": "vector", 07:05:09 DEBUG "pod": "idm-b85f9d978-wnhxb" 07:05:09 DEBUG "resultType": "vector", 07:05:09 DEBUG "pod": "am-767f6b67b5-vbltz" 07:05:09 DEBUG "result": [ 07:05:09 DEBUG }, 07:05:09 DEBUG "result": [ 07:05:09 DEBUG }, 07:05:09 DEBUG { 07:05:09 DEBUG "value": [ 07:05:09 INFO [http_cmd]: http status code OK 07:05:09 INFO [http_cmd]: http status code OK 07:05:09 INFO [http_cmd]: http status code OK 07:05:09 DEBUG { 07:05:09 DEBUG "value": [ 07:05:09 DEBUG "metric": { 07:05:09 DEBUG 1713593109, 07:05:09 DEBUG --- status code --- 07:05:09 DEBUG --- status code --- 07:05:09 DEBUG --- status code --- 07:05:09 DEBUG "metric": { 07:05:09 DEBUG 1713593109, 07:05:09 DEBUG "pod": "ds-cts-0" 07:05:09 DEBUG "0" 07:05:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:05:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:05:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:05:09 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 07:05:09 DEBUG "384312.5" 07:05:09 DEBUG }, 07:05:09 DEBUG ] 07:05:09 DEBUG --- http response --- 07:05:09 DEBUG --- http response --- 07:05:09 DEBUG --- http response --- 07:05:09 DEBUG }, 07:05:09 DEBUG ] 07:05:09 DEBUG "value": [ 07:05:09 DEBUG } 07:05:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:05:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:05:09 DEBUG "value": [ 07:05:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:05:09 DEBUG } 07:05:09 DEBUG 07:05:09 DEBUG ] 07:05:09 DEBUG ----- output ----- 07:05:09 DEBUG ----- output ----- 07:05:09 DEBUG 07:05:09 DEBUG ----- output ----- 07:05:09 DEBUG ] 07:05:09 DEBUG 07:05:09 DEBUG } 07:05:09 DEBUG { 07:05:09 DEBUG { 07:05:09 DEBUG 07:05:09 DEBUG { 07:05:09 DEBUG } 07:05:09 DEBUG 07:05:09 DEBUG } 07:05:09 DEBUG "status": "success", 07:05:09 DEBUG "status": "success", 07:05:09 DEBUG 07:05:09 DEBUG "status": "success", 07:05:09 DEBUG } 07:05:09 DEBUG [...] 07:05:09 DEBUG "data": { 07:05:09 DEBUG "data": { 07:05:09 DEBUG [...] 07:05:09 DEBUG "data": { 07:05:09 DEBUG 07:05:09 DEBUG "resultType": "vector", 07:05:09 DEBUG "resultType": "vector", 07:05:09 DEBUG 07:05:09 DEBUG "resultType": "vector", 07:05:09 DEBUG 07:05:09 DEBUG "result": [ 07:05:09 DEBUG "result": [ 07:05:09 DEBUG 07:05:09 DEBUG "result": [ 07:05:09 DEBUG 07:05:09 DEBUG { 07:05:09 DEBUG { 07:05:09 DEBUG 07:05:09 DEBUG { 07:05:09 DEBUG "pod": "ds-cts-2" 07:05:09 DEBUG "metric": { 07:05:09 DEBUG "metric": { 07:05:09 DEBUG "pod": "idm-b85f9d978-wnhxb" 07:05:09 DEBUG "metric": { 07:05:09 DEBUG }, 07:05:09 DEBUG "pod": "ds-idrepo-0" 07:05:09 DEBUG "pod": "ds-idrepo-0" 07:05:09 DEBUG }, 07:05:09 DEBUG "instance": "10.142.0.107:9100" 07:05:09 DEBUG "value": [ 07:05:09 DEBUG }, 07:05:09 DEBUG }, 07:05:09 DEBUG "value": [ 07:05:09 DEBUG }, 07:05:09 DEBUG 1713593109, 07:05:09 DEBUG "value": [ 07:05:09 DEBUG "value": [ 07:05:09 DEBUG 1713593109, 07:05:09 DEBUG "value": [ 07:05:09 DEBUG "29.966666666666665" 07:05:09 DEBUG 07:05:09 DEBUG 07:05:09 DEBUG "575.4223299265972" 07:05:09 DEBUG 07:05:09 DEBUG ] 07:05:09 DEBUG 07:05:09 DEBUG 07:05:09 DEBUG ] 07:05:09 DEBUG 07:05:09 DEBUG } 07:05:09 DEBUG 07:05:09 DEBUG 07:05:09 DEBUG } 07:05:09 DEBUG 07:05:09 DEBUG ] 07:05:09 DEBUG [...] 07:05:09 DEBUG [...] 07:05:09 DEBUG ] 07:05:09 DEBUG [...] 07:05:09 DEBUG } 07:05:09 DEBUG 07:05:09 DEBUG 07:05:09 DEBUG } 07:05:09 DEBUG 07:05:09 DEBUG } 07:05:09 DEBUG 07:05:09 DEBUG 07:05:09 DEBUG } 07:05:09 DEBUG 07:05:09 DEBUG 07:05:09 DEBUG 07:05:09 DEBUG 07:05:09 DEBUG "pod": "ds-idrepo-2" 07:05:09 DEBUG "pod": "ds-idrepo-2" 07:05:09 DEBUG "instance": "10.142.0.43:9100" 07:05:09 DEBUG }, 07:05:09 DEBUG }, 07:05:09 DEBUG }, 07:05:09 DEBUG "value": [ 07:05:09 DEBUG "value": [ 07:05:09 DEBUG "value": [ 07:05:09 DEBUG 1713593109, 07:05:09 DEBUG 1713593109, 07:05:09 DEBUG 1713593109, 07:05:09 DEBUG "0" 07:05:09 DEBUG "0" 07:05:09 DEBUG "3284949144576" 07:05:09 DEBUG ] 07:05:09 DEBUG ] 07:05:09 DEBUG ] 07:05:09 DEBUG } 07:05:09 DEBUG } 07:05:09 DEBUG } 07:05:09 DEBUG ] 07:05:09 DEBUG ] 07:05:09 DEBUG ] 07:05:09 DEBUG } 07:05:09 DEBUG } 07:05:09 DEBUG } 07:05:09 DEBUG } 07:05:09 DEBUG } 07:05:09 DEBUG } 07:05:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713593109" 07:05:09 INFO [http_cmd]: http status code OK 07:05:09 DEBUG --- status code --- 07:05:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:05:09 DEBUG --- http response --- 07:05:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:05:09 DEBUG ----- output ----- 07:05:09 DEBUG { 07:05:09 DEBUG "status": "success", 07:05:09 DEBUG "data": { 07:05:09 DEBUG "resultType": "vector", 07:05:09 DEBUG "result": [ 07:05:09 DEBUG { 07:05:09 DEBUG "metric": { 07:05:09 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 07:05:09 DEBUG }, 07:05:09 DEBUG "value": [ 07:05:09 DEBUG 07:05:09 DEBUG 07:05:09 DEBUG 07:05:09 DEBUG [...] 07:05:09 DEBUG 07:05:09 DEBUG 07:05:09 DEBUG 07:05:09 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 07:05:09 DEBUG }, 07:05:09 DEBUG "value": [ 07:05:09 DEBUG 1713593109, 07:05:09 DEBUG "2349854720" 07:05:09 DEBUG ] 07:05:09 DEBUG } 07:05:09 DEBUG ] 07:05:09 DEBUG } 07:05:09 DEBUG } 07:05:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713593109" 07:05:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593109" 07:05:09 INFO [http_cmd]: http status code OK 07:05:09 DEBUG --- status code --- 07:05:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:05:09 DEBUG --- http response --- 07:05:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:05:09 DEBUG ----- output ----- 07:05:09 DEBUG { 07:05:09 DEBUG "status": "success", 07:05:09 DEBUG "data": { 07:05:09 DEBUG "resultType": "vector", 07:05:09 DEBUG "result": [ 07:05:09 DEBUG { 07:05:09 DEBUG "metric": { 07:05:09 DEBUG "instance": "10.142.0.107:9100" 07:05:09 DEBUG }, 07:05:09 DEBUG "value": [ 07:05:09 DEBUG 07:05:09 DEBUG 07:05:09 DEBUG 07:05:09 DEBUG [...] 07:05:09 DEBUG 07:05:09 DEBUG 07:05:09 DEBUG 07:05:09 DEBUG "instance": "10.142.0.43:9100" 07:05:09 DEBUG }, 07:05:09 DEBUG "value": [ 07:05:09 DEBUG 1713593109, 07:05:09 DEBUG "0.2380999999999195" 07:05:09 DEBUG ] 07:05:09 DEBUG } 07:05:09 DEBUG ] 07:05:09 DEBUG } 07:05:09 DEBUG } 07:05:09 INFO [http_cmd]: http status code OK 07:05:09 DEBUG --- status code --- 07:05:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:05:09 DEBUG --- http response --- 07:05:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:05:09 DEBUG ----- output ----- 07:05:09 DEBUG { 07:05:09 DEBUG "status": "success", 07:05:09 DEBUG "data": { 07:05:09 DEBUG "resultType": "vector", 07:05:09 DEBUG "result": [ 07:05:09 DEBUG { 07:05:09 DEBUG "metric": { 07:05:09 DEBUG "pod": "am-767f6b67b5-9zw22" 07:05:09 DEBUG }, 07:05:09 DEBUG "value": [ 07:05:09 DEBUG 07:05:09 DEBUG 07:05:09 DEBUG 07:05:09 DEBUG [...] 07:05:09 DEBUG 07:05:09 DEBUG 07:05:09 DEBUG 07:05:09 DEBUG "pod": "am-767f6b67b5-vbltz" 07:05:09 DEBUG }, 07:05:09 DEBUG "value": [ 07:05:09 DEBUG 1713593109, 07:05:09 DEBUG "121.03333333333333" 07:05:09 DEBUG ] 07:05:09 DEBUG } 07:05:09 DEBUG ] 07:05:09 DEBUG } 07:05:09 DEBUG } 07:05:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593109" 07:05:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593109" 07:05:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593109" 07:05:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713593109" 07:05:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593109" 07:05:09 INFO [http_cmd]: http status code OK 07:05:09 INFO [http_cmd]: http status code OK 07:05:09 DEBUG --- status code --- 07:05:09 DEBUG --- status code --- 07:05:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:05:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:05:09 DEBUG --- http response --- 07:05:09 DEBUG --- http response --- 07:05:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 07:05:09 DEBUG ----- output ----- 07:05:09 DEBUG { 07:05:09 DEBUG "status": "success", 07:05:09 DEBUG "data": { 07:05:09 DEBUG "resultType": "vector", 07:05:09 DEBUG "result": [ 07:05:09 DEBUG { 07:05:09 DEBUG "metric": { 07:05:09 DEBUG "pod": "ds-cts-0" 07:05:09 DEBUG }, 07:05:09 DEBUG "value": [ 07:05:09 DEBUG 07:05:09 DEBUG 07:05:09 DEBUG 07:05:09 DEBUG [...] 07:05:09 DEBUG 07:05:09 DEBUG 07:05:09 DEBUG 07:05:09 DEBUG "pod": "ds-cts-2" 07:05:09 DEBUG }, 07:05:09 DEBUG "value": [ 07:05:09 DEBUG 1713593109, 07:05:09 DEBUG "0" 07:05:09 DEBUG ] 07:05:09 DEBUG } 07:05:09 DEBUG ] 07:05:09 DEBUG } 07:05:09 DEBUG } 07:05:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 07:05:09 DEBUG ----- output ----- 07:05:09 DEBUG { 07:05:09 DEBUG "status": "success", 07:05:09 DEBUG "data": { 07:05:09 DEBUG "resultType": "vector", 07:05:09 DEBUG "result": [ 07:05:09 DEBUG { 07:05:09 DEBUG "metric": { 07:05:09 DEBUG "pod": "ds-cts-0" 07:05:09 DEBUG }, 07:05:09 DEBUG "value": [ 07:05:09 DEBUG 07:05:09 DEBUG 07:05:09 DEBUG 07:05:09 DEBUG [...] 07:05:09 DEBUG 07:05:09 DEBUG 07:05:09 DEBUG 07:05:09 DEBUG "pod": "ds-cts-2" 07:05:09 DEBUG }, 07:05:09 DEBUG "value": [ 07:05:09 DEBUG 1713593109, 07:05:09 DEBUG "0" 07:05:09 DEBUG ] 07:05:09 DEBUG } 07:05:09 DEBUG ] 07:05:09 DEBUG } 07:05:09 DEBUG } 07:05:09 INFO [http_cmd]: http status code OK 07:05:09 DEBUG --- status code --- 07:05:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:05:09 DEBUG --- http response --- 07:05:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:05:09 INFO [http_cmd]: http status code OK 07:05:09 DEBUG ----- output ----- 07:05:09 DEBUG --- status code --- 07:05:09 DEBUG { 07:05:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:05:09 DEBUG "status": "success", 07:05:09 DEBUG --- http response --- 07:05:09 DEBUG "data": { 07:05:09 DEBUG "resultType": "vector", 07:05:09 DEBUG "result": [ 07:05:09 DEBUG { 07:05:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 07:05:09 DEBUG "metric": { 07:05:09 DEBUG ----- output ----- 07:05:09 DEBUG "pod": "am-767f6b67b5-9zw22" 07:05:09 DEBUG { 07:05:09 DEBUG }, 07:05:09 DEBUG "status": "success", 07:05:09 DEBUG "value": [ 07:05:09 DEBUG "data": { 07:05:09 DEBUG 07:05:09 DEBUG "resultType": "vector", 07:05:09 DEBUG 07:05:09 DEBUG "result": [ 07:05:09 DEBUG 07:05:09 DEBUG { 07:05:09 DEBUG [...] 07:05:09 DEBUG "metric": { 07:05:09 DEBUG 07:05:09 DEBUG "pod": "ds-idrepo-0" 07:05:09 DEBUG 07:05:09 DEBUG }, 07:05:09 DEBUG 07:05:09 DEBUG "value": [ 07:05:09 DEBUG "pod": "am-767f6b67b5-vbltz" 07:05:09 DEBUG 07:05:09 DEBUG }, 07:05:09 DEBUG 07:05:09 DEBUG "value": [ 07:05:09 DEBUG 07:05:09 DEBUG 1713593109, 07:05:09 DEBUG [...] 07:05:09 DEBUG "120.89999999999999" 07:05:09 DEBUG 07:05:09 DEBUG ] 07:05:09 DEBUG 07:05:09 DEBUG } 07:05:09 DEBUG 07:05:09 DEBUG ] 07:05:09 DEBUG "pod": "ds-cts-2" 07:05:09 DEBUG } 07:05:09 DEBUG }, 07:05:09 DEBUG } 07:05:09 DEBUG "value": [ 07:05:09 DEBUG 1713593109, 07:05:09 DEBUG "0" 07:05:09 DEBUG ] 07:05:09 DEBUG } 07:05:09 DEBUG ] 07:05:09 DEBUG } 07:05:09 DEBUG } 07:05:09 INFO [http_cmd]: http status code OK 07:05:09 DEBUG --- status code --- 07:05:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:05:09 DEBUG --- http response --- 07:05:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:05:09 DEBUG ----- output ----- 07:05:09 DEBUG { 07:05:09 DEBUG "status": "success", 07:05:09 DEBUG "data": { 07:05:09 DEBUG "resultType": "vector", 07:05:09 DEBUG "result": [ 07:05:09 DEBUG { 07:05:09 DEBUG "metric": { 07:05:09 DEBUG "pod": "ds-idrepo-0" 07:05:09 DEBUG }, 07:05:09 DEBUG "value": [ 07:05:09 DEBUG 07:05:09 DEBUG 07:05:09 DEBUG 07:05:09 DEBUG [...] 07:05:09 DEBUG 07:05:09 DEBUG 07:05:09 DEBUG 07:05:09 DEBUG "pod": "ds-idrepo-2" 07:05:09 DEBUG }, 07:05:09 DEBUG "value": [ 07:05:09 DEBUG 1713593109, 07:05:09 DEBUG "0" 07:05:09 DEBUG ] 07:05:09 DEBUG } 07:05:09 DEBUG ] 07:05:09 DEBUG } 07:05:09 DEBUG } 07:05:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593109" 07:05:09 INFO [http_cmd]: http status code OK 07:05:09 DEBUG --- status code --- 07:05:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:05:09 DEBUG --- http response --- 07:05:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:05:09 DEBUG ----- output ----- 07:05:09 DEBUG { 07:05:09 DEBUG "status": "success", 07:05:09 DEBUG "data": { 07:05:09 DEBUG "resultType": "vector", 07:05:09 DEBUG "result": [ 07:05:09 DEBUG { 07:05:09 DEBUG "metric": { 07:05:09 DEBUG "pod": "am-767f6b67b5-9zw22" 07:05:09 DEBUG }, 07:05:09 DEBUG "value": [ 07:05:09 DEBUG 07:05:09 DEBUG 07:05:09 DEBUG 07:05:09 DEBUG [...] 07:05:09 DEBUG 07:05:09 DEBUG 07:05:09 DEBUG 07:05:09 DEBUG "pod": "am-767f6b67b5-vbltz" 07:05:09 DEBUG }, 07:05:09 DEBUG "value": [ 07:05:09 DEBUG 1713593109, 07:05:09 DEBUG "2668.9" 07:05:09 DEBUG ] 07:05:09 DEBUG } 07:05:09 DEBUG ] 07:05:09 DEBUG } 07:05:09 DEBUG } 07:05:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713593109" 07:05:09 INFO [http_cmd]: http status code OK 07:05:09 DEBUG --- status code --- 07:05:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:05:09 DEBUG --- http response --- 07:05:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:05:09 DEBUG ----- output ----- 07:05:09 DEBUG { 07:05:09 DEBUG "status": "success", 07:05:09 DEBUG "data": { 07:05:09 DEBUG "resultType": "vector", 07:05:09 DEBUG "result": [ 07:05:09 DEBUG { 07:05:09 DEBUG "metric": { 07:05:09 DEBUG "instance": "10.142.0.107:9100" 07:05:09 DEBUG }, 07:05:09 DEBUG "value": [ 07:05:09 DEBUG 07:05:09 DEBUG 07:05:09 DEBUG 07:05:09 DEBUG [...] 07:05:09 DEBUG 07:05:09 DEBUG 07:05:09 DEBUG 07:05:09 DEBUG "instance": "10.142.0.43:9100" 07:05:09 DEBUG }, 07:05:09 DEBUG "value": [ 07:05:09 DEBUG 1713593109, 07:05:09 DEBUG "42724121.2" 07:05:09 DEBUG ] 07:05:09 DEBUG } 07:05:09 DEBUG ] 07:05:09 DEBUG } 07:05:09 DEBUG } 07:05:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713593109" 07:05:09 INFO [http_cmd]: http status code OK 07:05:09 DEBUG --- status code --- 07:05:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:05:09 DEBUG --- http response --- 07:05:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:05:09 DEBUG ----- output ----- 07:05:09 DEBUG { 07:05:09 DEBUG "status": "success", 07:05:09 DEBUG "data": { 07:05:09 DEBUG "resultType": "vector", 07:05:09 DEBUG "result": [ 07:05:09 DEBUG { 07:05:09 DEBUG "metric": { 07:05:09 DEBUG "pod": "am-767f6b67b5-9zw22" 07:05:09 DEBUG }, 07:05:09 DEBUG "value": [ 07:05:09 DEBUG 07:05:09 DEBUG 07:05:09 DEBUG 07:05:09 DEBUG [...] 07:05:09 DEBUG 07:05:09 DEBUG 07:05:09 DEBUG 07:05:09 DEBUG "pod": "am-767f6b67b5-vbltz" 07:05:09 DEBUG }, 07:05:09 DEBUG "value": [ 07:05:09 DEBUG 1713593109, 07:05:09 DEBUG "0.097" 07:05:09 DEBUG ] 07:05:09 DEBUG } 07:05:09 DEBUG ] 07:05:09 DEBUG } 07:05:09 DEBUG } 07:05:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593109" 07:05:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593109" 07:05:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593109" 07:05:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713593109" 07:05:09 INFO [http_cmd]: http status code OK 07:05:09 DEBUG --- status code --- 07:05:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:05:09 DEBUG --- http response --- 07:05:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:05:09 DEBUG ----- output ----- 07:05:09 DEBUG { 07:05:09 DEBUG "status": "success", 07:05:09 DEBUG "data": { 07:05:09 DEBUG "resultType": "vector", 07:05:09 DEBUG "result": [ 07:05:09 DEBUG { 07:05:09 DEBUG "metric": { 07:05:09 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 07:05:09 DEBUG }, 07:05:09 DEBUG "value": [ 07:05:09 DEBUG 07:05:09 DEBUG 07:05:09 DEBUG 07:05:09 DEBUG [...] 07:05:09 DEBUG 07:05:09 DEBUG 07:05:09 DEBUG 07:05:09 DEBUG "pod": "idm-b85f9d978-wnhxb" 07:05:09 DEBUG }, 07:05:09 DEBUG "value": [ 07:05:09 DEBUG 1713593109, 07:05:09 DEBUG "0.002945246373760139" 07:05:09 DEBUG ] 07:05:09 DEBUG } 07:05:09 DEBUG ] 07:05:09 DEBUG } 07:05:09 DEBUG } 07:05:09 INFO [http_cmd]: http status code OK 07:05:09 DEBUG --- status code --- 07:05:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:05:09 DEBUG --- http response --- 07:05:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:05:09 DEBUG ----- output ----- 07:05:09 DEBUG { 07:05:09 DEBUG "status": "success", 07:05:09 DEBUG "data": { 07:05:09 DEBUG "resultType": "vector", 07:05:09 DEBUG "result": [ 07:05:09 DEBUG { 07:05:09 DEBUG "metric": { 07:05:09 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 07:05:09 DEBUG }, 07:05:09 DEBUG "value": [ 07:05:09 DEBUG 07:05:09 DEBUG 07:05:09 DEBUG 07:05:09 DEBUG [...] 07:05:09 DEBUG 07:05:09 DEBUG 07:05:09 DEBUG 07:05:09 DEBUG "pod": "idm-b85f9d978-wnhxb" 07:05:09 DEBUG }, 07:05:09 DEBUG "value": [ 07:05:09 DEBUG 1713593109, 07:05:09 DEBUG "143.8555824816493" 07:05:09 DEBUG ] 07:05:09 DEBUG } 07:05:09 DEBUG ] 07:05:09 DEBUG } 07:05:09 DEBUG } 07:05:09 INFO [http_cmd]: http status code OK 07:05:09 DEBUG --- status code --- 07:05:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:05:09 DEBUG --- http response --- 07:05:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:05:09 DEBUG ----- output ----- 07:05:09 DEBUG { 07:05:09 DEBUG "status": "success", 07:05:09 DEBUG "data": { 07:05:09 DEBUG "resultType": "vector", 07:05:09 DEBUG "result": [ 07:05:09 DEBUG { 07:05:09 DEBUG "metric": { 07:05:09 DEBUG "pod": "am-767f6b67b5-9zw22" 07:05:09 DEBUG }, 07:05:09 DEBUG "value": [ 07:05:09 DEBUG 07:05:09 DEBUG 07:05:09 DEBUG 07:05:09 DEBUG [...] 07:05:09 DEBUG 07:05:09 DEBUG 07:05:09 DEBUG 07:05:09 DEBUG "pod": "am-767f6b67b5-vbltz" 07:05:09 DEBUG }, 07:05:09 DEBUG "value": [ 07:05:09 DEBUG 1713593109, 07:05:09 DEBUG "0" 07:05:09 DEBUG ] 07:05:09 DEBUG } 07:05:09 DEBUG ] 07:05:09 DEBUG } 07:05:09 DEBUG } 07:05:09 INFO [http_cmd]: http status code OK 07:05:09 DEBUG --- status code --- 07:05:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:05:09 DEBUG --- http response --- 07:05:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:05:09 DEBUG ----- output ----- 07:05:09 DEBUG { 07:05:09 DEBUG "status": "success", 07:05:09 DEBUG "data": { 07:05:09 DEBUG "resultType": "vector", 07:05:09 DEBUG "result": [ 07:05:09 DEBUG { 07:05:09 DEBUG "metric": { 07:05:09 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 07:05:09 DEBUG }, 07:05:09 DEBUG "value": [ 07:05:09 DEBUG 07:05:09 DEBUG 07:05:09 DEBUG 07:05:09 DEBUG [...] 07:05:09 DEBUG 07:05:09 DEBUG 07:05:09 DEBUG 07:05:09 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 07:05:09 DEBUG }, 07:05:09 DEBUG "value": [ 07:05:09 DEBUG 1713593109, 07:05:09 DEBUG "3.137171293551375" 07:05:09 DEBUG ] 07:05:09 DEBUG } 07:05:09 DEBUG ] 07:05:09 DEBUG } 07:05:09 DEBUG } 07:05:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593109" 07:05:09 INFO [http_cmd]: http status code OK 07:05:09 DEBUG --- status code --- 07:05:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:05:09 DEBUG --- http response --- 07:05:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:05:09 DEBUG ----- output ----- 07:05:09 DEBUG { 07:05:09 DEBUG "status": "success", 07:05:09 DEBUG "data": { 07:05:09 DEBUG "resultType": "vector", 07:05:09 DEBUG "result": [ 07:05:09 DEBUG { 07:05:09 DEBUG "metric": { 07:05:09 DEBUG "pod": "am-767f6b67b5-9zw22" 07:05:09 DEBUG }, 07:05:09 DEBUG "value": [ 07:05:09 DEBUG 07:05:09 DEBUG 07:05:09 DEBUG 07:05:09 DEBUG [...] 07:05:09 DEBUG 07:05:09 DEBUG 07:05:09 DEBUG 07:05:09 DEBUG "pod": "am-767f6b67b5-vbltz" 07:05:09 DEBUG }, 07:05:09 DEBUG "value": [ 07:05:09 DEBUG 1713593109, 07:05:09 DEBUG "120.5" 07:05:09 DEBUG ] 07:05:09 DEBUG } 07:05:09 DEBUG ] 07:05:09 DEBUG } 07:05:09 DEBUG } 07:05:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713593109" 07:05:09 INFO [http_cmd]: http status code OK 07:05:09 DEBUG --- status code --- 07:05:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:05:09 DEBUG --- http response --- 07:05:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:05:09 DEBUG ----- output ----- 07:05:09 DEBUG { 07:05:09 DEBUG "status": "success", 07:05:09 DEBUG "data": { 07:05:09 DEBUG "resultType": "vector", 07:05:09 DEBUG "result": [ 07:05:09 DEBUG { 07:05:09 DEBUG "metric": { 07:05:09 DEBUG "instance": "10.142.0.107:9100" 07:05:09 DEBUG }, 07:05:09 DEBUG "value": [ 07:05:09 DEBUG 07:05:09 DEBUG 07:05:09 DEBUG 07:05:09 DEBUG [...] 07:05:09 DEBUG 07:05:09 DEBUG 07:05:09 DEBUG 07:05:09 DEBUG "instance": "10.142.0.43:9100" 07:05:09 DEBUG }, 07:05:09 DEBUG "value": [ 07:05:09 DEBUG 1713593109, 07:05:09 DEBUG "42725304.63333333" 07:05:09 DEBUG ] 07:05:09 DEBUG } 07:05:09 DEBUG ] 07:05:09 DEBUG } 07:05:09 DEBUG } 07:05:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713593109" 07:05:09 INFO [http_cmd]: http status code OK 07:05:09 DEBUG --- status code --- 07:05:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:05:09 DEBUG --- http response --- 07:05:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:05:09 DEBUG ----- output ----- 07:05:09 DEBUG { 07:05:09 DEBUG "status": "success", 07:05:09 DEBUG "data": { 07:05:09 DEBUG "resultType": "vector", 07:05:09 DEBUG "result": [ 07:05:09 DEBUG { 07:05:09 DEBUG "metric": { 07:05:09 DEBUG "instance": "10.142.0.107:9100" 07:05:09 DEBUG }, 07:05:09 DEBUG "value": [ 07:05:09 DEBUG 07:05:09 DEBUG 07:05:09 DEBUG 07:05:09 DEBUG [...] 07:05:09 DEBUG 07:05:09 DEBUG 07:05:09 DEBUG 07:05:09 DEBUG "instance": "10.142.0.43:9100" 07:05:09 DEBUG }, 07:05:09 DEBUG "value": [ 07:05:09 DEBUG 1713593109, 07:05:09 DEBUG "0.12099999999999418" 07:05:09 DEBUG ] 07:05:09 DEBUG } 07:05:09 DEBUG ] 07:05:09 DEBUG } 07:05:09 DEBUG } 07:05:11 INFO 07:05:11 INFO [loop_until]: kubectl --namespace=xlou top node 07:05:11 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 07:05:11 INFO [loop_until]: OK (rc = 0) 07:05:11 DEBUG --- stdout --- 07:05:11 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 71m 0% 6267Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 1442m 4% 7464Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 2870m 9% 11399Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 86m 0% 3364Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 8713m 29% 13593Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 155m 0% 5182Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 3046m 10% 11614Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 775m 2% 2957Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 3283m 10% 10893Mi 9% gke-xlou-cdm-ds-0f601751-j488 888m 2% 7027Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 7300m 24% 15589Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 1497m 5% 7709Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 8497m 28% 12929Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 2481m 15% 6179Mi 10% 07:05:11 DEBUG --- stderr --- 07:05:11 DEBUG 07:05:11 INFO 07:05:11 INFO [loop_until]: kubectl --namespace=xlou top pods 07:05:11 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 07:05:11 INFO [loop_until]: OK (rc = 0) 07:05:11 DEBUG --- stdout --- 07:05:11 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 2514m 8649Mi am-767f6b67b5-qcqz2 2835m 8709Mi am-767f6b67b5-vbltz 2952m 8650Mi ds-cts-0 7620m 10475Mi ds-cts-1 7271m 10649Mi ds-cts-2 8552m 10326Mi ds-idrepo-0 1261m 3786Mi ds-idrepo-1 805m 3802Mi ds-idrepo-2 828m 3711Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1177Mi idm-b85f9d978-wnhxb 4m 1236Mi lodemon-7cb5b9456d-dskfn 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 2458m 2363Mi 07:05:11 DEBUG --- stderr --- 07:05:11 DEBUG 07:06:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593169" 07:06:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593169" 07:06:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713593169" 07:06:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593169" 07:06:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593169" 07:06:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593169" 07:06:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713593169" 07:06:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593169" 07:06:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593169" 07:06:09 ERROR [http_cmd]: ERROR 07:06:09 ERROR --- status code --- 07:06:09 ERROR http status code is 400 (expected [200, 401, 403, 404]) 07:06:09 ERROR --- http response --- 07:06:09 INFO [http_cmd]: http status code OK 07:06:09 INFO [http_cmd]: http status code OK 07:06:09 DEBUG --- status code --- 07:06:09 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 07:06:09 DEBUG --- status code --- 07:06:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:06:09 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 07:06:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:06:09 DEBUG --- http response --- 07:06:09 DEBUG --- http response --- 07:06:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 07:06:09 DEBUG ----- output ----- 07:06:09 INFO [http_cmd]: http status code OK 07:06:09 DEBUG { 07:06:09 INFO [http_cmd]: http status code OK 07:06:09 INFO [http_cmd]: http status code OK 07:06:09 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 07:06:09 DEBUG --- status code --- 07:06:09 DEBUG "status": "success", 07:06:09 DEBUG --- status code --- 07:06:09 DEBUG --- status code --- 07:06:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:06:09 DEBUG "data": { 07:06:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:06:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:06:09 DEBUG --- http response --- 07:06:09 DEBUG "resultType": "vector", 07:06:09 DEBUG --- http response --- 07:06:09 DEBUG --- http response --- 07:06:09 INFO [http_cmd]: http status code OK 07:06:09 DEBUG "result": [ 07:06:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:06:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 07:06:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:06:09 DEBUG --- status code --- 07:06:09 DEBUG { 07:06:09 DEBUG ----- output ----- 07:06:09 DEBUG ----- output ----- 07:06:09 DEBUG ----- output ----- 07:06:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:06:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713593169" 07:06:09 DEBUG "metric": { 07:06:09 DEBUG { 07:06:09 DEBUG { 07:06:09 DEBUG { 07:06:09 DEBUG --- http response --- 07:06:09 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 07:06:09 INFO [http_cmd]: http status code OK 07:06:09 DEBUG "status": "success", 07:06:09 INFO [http_cmd]: http status code OK 07:06:09 DEBUG "status": "success", 07:06:09 DEBUG "status": "success", 07:06:09 DEBUG }, 07:06:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 07:06:09 DEBUG --- status code --- 07:06:09 DEBUG "data": { 07:06:09 DEBUG --- status code --- 07:06:09 DEBUG "data": { 07:06:09 DEBUG "data": { 07:06:09 DEBUG "value": [ 07:06:09 DEBUG ----- output ----- 07:06:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:06:09 DEBUG "resultType": "vector", 07:06:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:06:09 DEBUG "resultType": "vector", 07:06:09 DEBUG "resultType": "vector", 07:06:09 DEBUG 07:06:09 DEBUG { 07:06:09 DEBUG --- http response --- 07:06:09 DEBUG "result": [ 07:06:09 DEBUG --- http response --- 07:06:09 DEBUG "result": [ 07:06:09 DEBUG "result": [ 07:06:09 DEBUG 07:06:09 DEBUG "status": "success", 07:06:09 DEBUG { 07:06:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:06:09 DEBUG { 07:06:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:06:09 DEBUG { 07:06:09 DEBUG 07:06:09 DEBUG "data": { 07:06:09 DEBUG "metric": { 07:06:09 DEBUG ----- output ----- 07:06:09 DEBUG "metric": { 07:06:09 DEBUG ----- output ----- 07:06:09 DEBUG "metric": { 07:06:09 DEBUG [...] 07:06:09 DEBUG "resultType": "vector", 07:06:09 DEBUG "pod": "am-767f6b67b5-9zw22" 07:06:09 DEBUG { 07:06:09 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 07:06:09 DEBUG { 07:06:09 DEBUG "instance": "10.142.0.107:9100" 07:06:09 DEBUG 07:06:09 DEBUG "result": [ 07:06:09 DEBUG }, 07:06:09 DEBUG "status": "success", 07:06:09 DEBUG }, 07:06:09 DEBUG "status": "success", 07:06:09 DEBUG }, 07:06:09 DEBUG 07:06:09 DEBUG { 07:06:09 DEBUG "value": [ 07:06:09 DEBUG "data": { 07:06:09 DEBUG "value": [ 07:06:09 DEBUG "data": { 07:06:09 DEBUG "value": [ 07:06:09 DEBUG 07:06:09 DEBUG "metric": { 07:06:09 DEBUG 07:06:09 DEBUG "resultType": "vector", 07:06:09 DEBUG 07:06:09 DEBUG "resultType": "vector", 07:06:09 DEBUG 07:06:09 DEBUG "pod": "ds-idrepo-2" 07:06:09 DEBUG "pod": "ds-cts-0" 07:06:09 DEBUG 07:06:09 DEBUG "result": [ 07:06:09 DEBUG 07:06:09 DEBUG "result": [ 07:06:09 DEBUG 07:06:09 DEBUG }, 07:06:09 DEBUG }, 07:06:09 DEBUG 07:06:09 DEBUG { 07:06:09 DEBUG 07:06:09 DEBUG { 07:06:09 DEBUG 07:06:09 DEBUG "value": [ 07:06:09 DEBUG "value": [ 07:06:09 DEBUG [...] 07:06:09 DEBUG "metric": { 07:06:09 DEBUG [...] 07:06:09 DEBUG "metric": { 07:06:09 DEBUG [...] 07:06:09 DEBUG 1713593169, 07:06:09 DEBUG 07:06:09 DEBUG 07:06:09 DEBUG "pod": "ds-idrepo-0" 07:06:09 DEBUG 07:06:09 DEBUG "pod": "ds-idrepo-0" 07:06:09 DEBUG 07:06:09 DEBUG "2745.9012057565146" 07:06:09 DEBUG 07:06:09 DEBUG 07:06:09 DEBUG }, 07:06:09 DEBUG 07:06:09 DEBUG }, 07:06:09 DEBUG 07:06:09 DEBUG ] 07:06:09 DEBUG 07:06:09 DEBUG 07:06:09 DEBUG "value": [ 07:06:09 DEBUG 07:06:09 DEBUG "value": [ 07:06:09 DEBUG 07:06:09 DEBUG } 07:06:09 DEBUG [...] 07:06:09 DEBUG "pod": "am-767f6b67b5-vbltz" 07:06:09 DEBUG 07:06:09 DEBUG "pod": "ds-idrepo-2" 07:06:09 DEBUG 07:06:09 DEBUG "instance": "10.142.0.43:9100" 07:06:09 DEBUG ] 07:06:09 DEBUG 07:06:09 DEBUG }, 07:06:09 DEBUG 07:06:09 DEBUG }, 07:06:09 DEBUG 07:06:09 DEBUG }, 07:06:09 DEBUG } 07:06:09 DEBUG 07:06:09 DEBUG "value": [ 07:06:09 DEBUG 07:06:09 DEBUG "value": [ 07:06:09 DEBUG 07:06:09 DEBUG "value": [ 07:06:09 DEBUG } 07:06:09 DEBUG 07:06:09 DEBUG 1713593169, 07:06:09 DEBUG [...] 07:06:09 DEBUG 1713593169, 07:06:09 DEBUG [...] 07:06:09 DEBUG 1713593169, 07:06:09 DEBUG "pod": "ds-cts-2" 07:06:09 DEBUG "310761.5" 07:06:09 DEBUG 07:06:09 DEBUG "0" 07:06:09 DEBUG 07:06:09 DEBUG "3288268917760" 07:06:09 DEBUG }, 07:06:09 DEBUG ] 07:06:09 DEBUG 07:06:09 DEBUG ] 07:06:09 DEBUG 07:06:09 DEBUG ] 07:06:09 DEBUG "value": [ 07:06:09 DEBUG } 07:06:09 DEBUG 07:06:09 DEBUG } 07:06:09 DEBUG 07:06:09 DEBUG } 07:06:09 DEBUG 1713593169, 07:06:09 DEBUG ] 07:06:09 DEBUG "pod": "ds-idrepo-2" 07:06:09 DEBUG ] 07:06:09 DEBUG "pod": "ds-idrepo-2" 07:06:09 DEBUG ] 07:06:09 DEBUG "30" 07:06:09 DEBUG } 07:06:09 DEBUG }, 07:06:09 DEBUG } 07:06:09 DEBUG }, 07:06:09 DEBUG } 07:06:09 DEBUG ] 07:06:09 DEBUG } 07:06:09 DEBUG "value": [ 07:06:09 DEBUG } 07:06:09 DEBUG "value": [ 07:06:09 DEBUG } 07:06:09 DEBUG } 07:06:09 DEBUG 1713593169, 07:06:09 DEBUG 1713593169, 07:06:09 DEBUG ] 07:06:09 DEBUG "0" 07:06:09 DEBUG "0" 07:06:09 DEBUG } 07:06:09 DEBUG ] 07:06:09 DEBUG ] 07:06:09 INFO [http_cmd]: http status code OK 07:06:09 DEBUG } 07:06:09 DEBUG } 07:06:09 DEBUG } 07:06:09 DEBUG --- status code --- 07:06:09 DEBUG ] 07:06:09 DEBUG ] 07:06:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:06:09 DEBUG } 07:06:09 DEBUG } 07:06:09 DEBUG --- http response --- 07:06:09 DEBUG } 07:06:09 DEBUG } 07:06:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:06:09 DEBUG ----- output ----- 07:06:09 DEBUG { 07:06:09 DEBUG "status": "success", 07:06:09 DEBUG "data": { 07:06:09 DEBUG "resultType": "vector", 07:06:09 DEBUG "result": [ 07:06:09 DEBUG { 07:06:09 DEBUG "metric": { 07:06:09 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 07:06:09 DEBUG }, 07:06:09 DEBUG "value": [ 07:06:09 DEBUG 07:06:09 DEBUG 07:06:09 DEBUG 07:06:09 DEBUG [...] 07:06:09 DEBUG 07:06:09 DEBUG 07:06:09 DEBUG 07:06:09 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 07:06:09 DEBUG }, 07:06:09 DEBUG "value": [ 07:06:09 DEBUG 1713593169, 07:06:09 DEBUG "4221452288" 07:06:09 DEBUG ] 07:06:09 DEBUG } 07:06:09 DEBUG ] 07:06:09 DEBUG } 07:06:09 DEBUG } 07:06:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593169" 07:06:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713593169" 07:06:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593169" 07:06:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593169" 07:06:09 INFO [http_cmd]: http status code OK 07:06:09 INFO [http_cmd]: http status code OK 07:06:09 DEBUG --- status code --- 07:06:09 DEBUG --- status code --- 07:06:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:06:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:06:09 DEBUG --- http response --- 07:06:09 DEBUG --- http response --- 07:06:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:06:09 DEBUG ----- output ----- 07:06:09 DEBUG { 07:06:09 DEBUG "status": "success", 07:06:09 DEBUG "data": { 07:06:09 DEBUG "resultType": "vector", 07:06:09 DEBUG "result": [ 07:06:09 DEBUG { 07:06:09 DEBUG "metric": { 07:06:09 DEBUG "pod": "ds-idrepo-0" 07:06:09 DEBUG }, 07:06:09 DEBUG "value": [ 07:06:09 DEBUG 07:06:09 DEBUG 07:06:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:06:09 DEBUG 07:06:09 DEBUG ----- output ----- 07:06:09 INFO [http_cmd]: http status code OK 07:06:09 DEBUG [...] 07:06:09 DEBUG { 07:06:09 DEBUG --- status code --- 07:06:09 DEBUG 07:06:09 DEBUG "status": "success", 07:06:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:06:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713593169" 07:06:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713593169" 07:06:09 DEBUG 07:06:09 DEBUG "data": { 07:06:09 DEBUG --- http response --- 07:06:09 DEBUG 07:06:09 DEBUG "resultType": "vector", 07:06:09 DEBUG "pod": "ds-idrepo-2" 07:06:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:06:09 DEBUG "result": [ 07:06:09 DEBUG }, 07:06:09 DEBUG ----- output ----- 07:06:09 DEBUG { 07:06:09 DEBUG "value": [ 07:06:09 DEBUG { 07:06:09 DEBUG "metric": { 07:06:09 DEBUG 1713593169, 07:06:09 DEBUG "status": "success", 07:06:09 DEBUG "pod": "am-767f6b67b5-9zw22" 07:06:09 DEBUG "0" 07:06:09 DEBUG "data": { 07:06:09 DEBUG }, 07:06:09 DEBUG ] 07:06:09 DEBUG "resultType": "vector", 07:06:09 DEBUG "value": [ 07:06:09 DEBUG } 07:06:09 DEBUG "result": [ 07:06:09 DEBUG 07:06:09 DEBUG ] 07:06:09 DEBUG { 07:06:09 DEBUG 07:06:09 DEBUG } 07:06:09 DEBUG "metric": { 07:06:09 DEBUG 07:06:09 DEBUG } 07:06:09 DEBUG "instance": "10.142.0.107:9100" 07:06:09 DEBUG [...] 07:06:09 DEBUG }, 07:06:09 DEBUG 07:06:09 DEBUG "value": [ 07:06:09 DEBUG 07:06:09 DEBUG 07:06:09 DEBUG 07:06:09 DEBUG 07:06:09 DEBUG "pod": "am-767f6b67b5-vbltz" 07:06:09 DEBUG 07:06:09 DEBUG }, 07:06:09 DEBUG [...] 07:06:09 DEBUG "value": [ 07:06:09 DEBUG 07:06:09 DEBUG 1713593169, 07:06:09 DEBUG 07:06:09 DEBUG "1604.9333333333334" 07:06:09 DEBUG 07:06:09 DEBUG ] 07:06:09 DEBUG "instance": "10.142.0.43:9100" 07:06:09 DEBUG } 07:06:09 DEBUG }, 07:06:09 DEBUG ] 07:06:09 DEBUG "value": [ 07:06:09 DEBUG } 07:06:09 DEBUG 1713593169, 07:06:09 DEBUG } 07:06:09 DEBUG "0.23686666666661815" 07:06:09 DEBUG ] 07:06:09 DEBUG } 07:06:09 DEBUG ] 07:06:09 DEBUG } 07:06:09 DEBUG } 07:06:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593169" 07:06:09 INFO [http_cmd]: http status code OK 07:06:09 DEBUG --- status code --- 07:06:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:06:09 DEBUG --- http response --- 07:06:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 07:06:09 DEBUG ----- output ----- 07:06:09 DEBUG { 07:06:09 DEBUG "status": "success", 07:06:09 DEBUG "data": { 07:06:09 DEBUG "resultType": "vector", 07:06:09 DEBUG "result": [ 07:06:09 DEBUG { 07:06:09 DEBUG "metric": { 07:06:09 DEBUG "pod": "ds-cts-0" 07:06:09 DEBUG }, 07:06:09 DEBUG "value": [ 07:06:09 DEBUG 07:06:09 DEBUG 07:06:09 DEBUG 07:06:09 DEBUG [...] 07:06:09 DEBUG 07:06:09 DEBUG 07:06:09 DEBUG 07:06:09 DEBUG "pod": "ds-cts-2" 07:06:09 DEBUG }, 07:06:09 DEBUG "value": [ 07:06:09 DEBUG 1713593169, 07:06:09 DEBUG "0" 07:06:09 DEBUG ] 07:06:09 DEBUG } 07:06:09 DEBUG ] 07:06:09 DEBUG } 07:06:09 DEBUG } 07:06:09 INFO [http_cmd]: http status code OK 07:06:09 INFO [http_cmd]: http status code OK 07:06:09 DEBUG --- status code --- 07:06:09 DEBUG --- status code --- 07:06:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:06:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:06:09 DEBUG --- http response --- 07:06:09 DEBUG --- http response --- 07:06:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 07:06:09 DEBUG ----- output ----- 07:06:09 DEBUG { 07:06:09 DEBUG "status": "success", 07:06:09 DEBUG "data": { 07:06:09 DEBUG "resultType": "vector", 07:06:09 DEBUG "result": [ 07:06:09 DEBUG { 07:06:09 DEBUG "metric": { 07:06:09 DEBUG "pod": "ds-idrepo-0" 07:06:09 DEBUG }, 07:06:09 DEBUG "value": [ 07:06:09 DEBUG 07:06:09 DEBUG 07:06:09 DEBUG 07:06:09 DEBUG [...] 07:06:09 DEBUG 07:06:09 DEBUG 07:06:09 DEBUG 07:06:09 DEBUG "pod": "ds-cts-2" 07:06:09 DEBUG }, 07:06:09 DEBUG "value": [ 07:06:09 DEBUG 1713593169, 07:06:09 DEBUG "0" 07:06:09 DEBUG ] 07:06:09 DEBUG } 07:06:09 DEBUG ] 07:06:09 DEBUG } 07:06:09 DEBUG } 07:06:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:06:09 DEBUG ----- output ----- 07:06:09 DEBUG { 07:06:09 DEBUG "status": "success", 07:06:09 DEBUG "data": { 07:06:09 DEBUG "resultType": "vector", 07:06:09 DEBUG "result": [ 07:06:09 DEBUG { 07:06:09 DEBUG "metric": { 07:06:09 DEBUG "instance": "10.142.0.107:9100" 07:06:09 DEBUG }, 07:06:09 DEBUG "value": [ 07:06:09 DEBUG 07:06:09 DEBUG 07:06:09 DEBUG 07:06:09 DEBUG [...] 07:06:09 DEBUG 07:06:09 DEBUG 07:06:09 DEBUG 07:06:09 DEBUG "instance": "10.142.0.43:9100" 07:06:09 DEBUG }, 07:06:09 DEBUG "value": [ 07:06:09 DEBUG 1713593169, 07:06:09 DEBUG "50485067" 07:06:09 DEBUG ] 07:06:09 DEBUG } 07:06:09 DEBUG ] 07:06:09 DEBUG } 07:06:09 DEBUG } 07:06:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593169" 07:06:09 INFO [http_cmd]: http status code OK 07:06:09 DEBUG --- status code --- 07:06:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:06:09 DEBUG --- http response --- 07:06:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 07:06:09 DEBUG ----- output ----- 07:06:09 DEBUG { 07:06:09 DEBUG "status": "success", 07:06:09 DEBUG "data": { 07:06:09 DEBUG "resultType": "vector", 07:06:09 DEBUG "result": [ 07:06:09 DEBUG { 07:06:09 DEBUG "metric": { 07:06:09 DEBUG "pod": "ds-cts-0" 07:06:09 DEBUG }, 07:06:09 DEBUG "value": [ 07:06:09 DEBUG 07:06:09 DEBUG 07:06:09 DEBUG 07:06:09 DEBUG [...] 07:06:09 DEBUG 07:06:09 DEBUG 07:06:09 DEBUG 07:06:09 DEBUG "pod": "ds-cts-2" 07:06:09 DEBUG }, 07:06:09 DEBUG "value": [ 07:06:09 DEBUG 1713593169, 07:06:09 DEBUG "0.00036666666666666667" 07:06:09 DEBUG ] 07:06:09 DEBUG } 07:06:09 DEBUG ] 07:06:09 DEBUG } 07:06:09 DEBUG } 07:06:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593169" 07:06:09 INFO [http_cmd]: http status code OK 07:06:09 DEBUG --- status code --- 07:06:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:06:09 DEBUG --- http response --- 07:06:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:06:09 DEBUG ----- output ----- 07:06:09 DEBUG { 07:06:09 DEBUG "status": "success", 07:06:09 DEBUG "data": { 07:06:09 DEBUG "resultType": "vector", 07:06:09 DEBUG "result": [ 07:06:09 DEBUG { 07:06:09 DEBUG "metric": { 07:06:09 DEBUG "pod": "am-767f6b67b5-9zw22" 07:06:09 DEBUG }, 07:06:09 DEBUG "value": [ 07:06:09 DEBUG 07:06:09 DEBUG 07:06:09 DEBUG 07:06:09 DEBUG [...] 07:06:09 DEBUG 07:06:09 DEBUG 07:06:09 DEBUG 07:06:09 DEBUG "pod": "am-767f6b67b5-vbltz" 07:06:09 DEBUG }, 07:06:09 DEBUG "value": [ 07:06:09 DEBUG 1713593169, 07:06:09 DEBUG "2663.733333333333" 07:06:09 DEBUG ] 07:06:09 DEBUG } 07:06:09 DEBUG ] 07:06:09 DEBUG } 07:06:09 DEBUG } 07:06:09 INFO [http_cmd]: http status code OK 07:06:09 DEBUG --- status code --- 07:06:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:06:09 DEBUG --- http response --- 07:06:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:06:09 DEBUG ----- output ----- 07:06:09 DEBUG { 07:06:09 DEBUG "status": "success", 07:06:09 DEBUG "data": { 07:06:09 DEBUG "resultType": "vector", 07:06:09 DEBUG "result": [ 07:06:09 DEBUG { 07:06:09 DEBUG "metric": { 07:06:09 DEBUG "pod": "am-767f6b67b5-9zw22" 07:06:09 DEBUG }, 07:06:09 DEBUG "value": [ 07:06:09 DEBUG 07:06:09 DEBUG 07:06:09 DEBUG 07:06:09 DEBUG [...] 07:06:09 DEBUG 07:06:09 DEBUG 07:06:09 DEBUG 07:06:09 DEBUG "pod": "am-767f6b67b5-vbltz" 07:06:09 DEBUG }, 07:06:09 DEBUG "value": [ 07:06:09 DEBUG 1713593169, 07:06:09 DEBUG "1606.3333333333333" 07:06:09 DEBUG ] 07:06:09 DEBUG } 07:06:09 DEBUG ] 07:06:09 DEBUG } 07:06:09 DEBUG } 07:06:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713593169" 07:06:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713593169" 07:06:09 INFO [http_cmd]: http status code OK 07:06:09 DEBUG --- status code --- 07:06:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:06:09 DEBUG --- http response --- 07:06:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:06:09 DEBUG ----- output ----- 07:06:09 DEBUG { 07:06:09 DEBUG "status": "success", 07:06:09 DEBUG "data": { 07:06:09 DEBUG "resultType": "vector", 07:06:09 DEBUG "result": [ 07:06:09 DEBUG { 07:06:09 DEBUG "metric": { 07:06:09 DEBUG "pod": "am-767f6b67b5-9zw22" 07:06:09 DEBUG }, 07:06:09 DEBUG "value": [ 07:06:09 DEBUG 07:06:09 DEBUG 07:06:09 DEBUG 07:06:09 DEBUG [...] 07:06:09 DEBUG 07:06:09 DEBUG 07:06:09 DEBUG 07:06:09 DEBUG "pod": "am-767f6b67b5-vbltz" 07:06:09 DEBUG }, 07:06:09 DEBUG "value": [ 07:06:09 DEBUG 1713593169, 07:06:09 DEBUG "0.097" 07:06:09 DEBUG ] 07:06:09 DEBUG } 07:06:09 DEBUG ] 07:06:09 DEBUG } 07:06:09 DEBUG } 07:06:09 INFO [http_cmd]: http status code OK 07:06:09 DEBUG --- status code --- 07:06:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:06:09 DEBUG --- http response --- 07:06:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:06:09 DEBUG ----- output ----- 07:06:09 DEBUG { 07:06:09 DEBUG "status": "success", 07:06:09 DEBUG "data": { 07:06:09 DEBUG "resultType": "vector", 07:06:09 DEBUG "result": [ 07:06:09 DEBUG { 07:06:09 DEBUG "metric": { 07:06:09 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 07:06:09 DEBUG }, 07:06:09 DEBUG "value": [ 07:06:09 DEBUG 07:06:09 DEBUG 07:06:09 DEBUG 07:06:09 DEBUG [...] 07:06:09 DEBUG 07:06:09 DEBUG 07:06:09 DEBUG 07:06:09 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 07:06:09 DEBUG }, 07:06:09 DEBUG "value": [ 07:06:09 DEBUG 1713593169, 07:06:09 DEBUG "7.228247948482097" 07:06:09 DEBUG ] 07:06:09 DEBUG } 07:06:09 DEBUG ] 07:06:09 DEBUG } 07:06:09 DEBUG } 07:06:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593169" 07:06:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593169" 07:06:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593169" 07:06:09 INFO [http_cmd]: http status code OK 07:06:09 DEBUG --- status code --- 07:06:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:06:09 DEBUG --- http response --- 07:06:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:06:09 DEBUG ----- output ----- 07:06:09 DEBUG { 07:06:09 DEBUG "status": "success", 07:06:09 DEBUG "data": { 07:06:09 DEBUG "resultType": "vector", 07:06:09 DEBUG "result": [ 07:06:09 DEBUG { 07:06:09 DEBUG "metric": { 07:06:09 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 07:06:09 DEBUG }, 07:06:09 DEBUG "value": [ 07:06:09 INFO [http_cmd]: http status code OK 07:06:09 INFO [http_cmd]: http status code OK 07:06:09 DEBUG 07:06:09 DEBUG --- status code --- 07:06:09 DEBUG --- status code --- 07:06:09 DEBUG 07:06:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:06:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:06:09 DEBUG 07:06:09 DEBUG --- http response --- 07:06:09 DEBUG --- http response --- 07:06:09 DEBUG [...] 07:06:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:06:09 DEBUG 07:06:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:06:09 DEBUG ----- output ----- 07:06:09 DEBUG 07:06:09 DEBUG ----- output ----- 07:06:09 DEBUG { 07:06:09 DEBUG 07:06:09 DEBUG { 07:06:09 DEBUG "status": "success", 07:06:09 DEBUG "pod": "ds-idrepo-2" 07:06:09 DEBUG "status": "success", 07:06:09 DEBUG "data": { 07:06:09 DEBUG }, 07:06:09 DEBUG "data": { 07:06:09 DEBUG "resultType": "vector", 07:06:09 DEBUG "value": [ 07:06:09 DEBUG "resultType": "vector", 07:06:09 DEBUG "result": [ 07:06:09 DEBUG 1713593169, 07:06:09 DEBUG "result": [ 07:06:09 DEBUG { 07:06:09 DEBUG "2.9155357175961645" 07:06:09 DEBUG { 07:06:09 DEBUG "metric": { 07:06:09 DEBUG ] 07:06:09 DEBUG "metric": { 07:06:09 DEBUG "pod": "am-767f6b67b5-9zw22" 07:06:09 DEBUG } 07:06:09 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 07:06:09 DEBUG }, 07:06:09 DEBUG ] 07:06:09 DEBUG }, 07:06:09 DEBUG "value": [ 07:06:09 DEBUG } 07:06:09 DEBUG "value": [ 07:06:09 DEBUG 07:06:09 DEBUG } 07:06:09 DEBUG 07:06:09 DEBUG 07:06:09 DEBUG 07:06:09 DEBUG 07:06:09 DEBUG 07:06:09 DEBUG [...] 07:06:09 DEBUG [...] 07:06:09 DEBUG 07:06:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593169" 07:06:09 DEBUG 07:06:09 DEBUG 07:06:09 DEBUG 07:06:09 DEBUG 07:06:09 DEBUG 07:06:09 DEBUG "pod": "am-767f6b67b5-vbltz" 07:06:09 DEBUG "pod": "ds-idrepo-2" 07:06:09 DEBUG }, 07:06:09 DEBUG }, 07:06:09 DEBUG "value": [ 07:06:09 DEBUG "value": [ 07:06:09 DEBUG 1713593169, 07:06:09 DEBUG 1713593169, 07:06:09 DEBUG "0" 07:06:09 DEBUG "135902291.44146246" 07:06:09 DEBUG ] 07:06:09 DEBUG ] 07:06:09 DEBUG } 07:06:09 DEBUG } 07:06:09 DEBUG ] 07:06:09 DEBUG ] 07:06:09 DEBUG } 07:06:09 DEBUG } 07:06:09 DEBUG } 07:06:09 DEBUG } 07:06:09 INFO [http_cmd]: http status code OK 07:06:09 DEBUG --- status code --- 07:06:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:06:09 DEBUG --- http response --- 07:06:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:06:09 DEBUG ----- output ----- 07:06:09 DEBUG { 07:06:09 DEBUG "status": "success", 07:06:09 DEBUG "data": { 07:06:09 DEBUG "resultType": "vector", 07:06:09 DEBUG "result": [ 07:06:09 DEBUG { 07:06:09 DEBUG "metric": { 07:06:09 DEBUG "pod": "am-767f6b67b5-9zw22" 07:06:09 DEBUG }, 07:06:09 DEBUG "value": [ 07:06:09 DEBUG 07:06:09 DEBUG 07:06:09 DEBUG 07:06:09 DEBUG [...] 07:06:09 DEBUG 07:06:09 DEBUG 07:06:09 DEBUG 07:06:09 DEBUG "pod": "am-767f6b67b5-vbltz" 07:06:09 DEBUG }, 07:06:09 DEBUG "value": [ 07:06:09 DEBUG 1713593169, 07:06:09 DEBUG "1605.6333333333332" 07:06:09 DEBUG ] 07:06:09 DEBUG } 07:06:09 DEBUG ] 07:06:09 DEBUG } 07:06:09 DEBUG } 07:06:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713593169" 07:06:09 INFO [http_cmd]: http status code OK 07:06:09 DEBUG --- status code --- 07:06:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:06:09 DEBUG --- http response --- 07:06:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:06:09 DEBUG ----- output ----- 07:06:09 DEBUG { 07:06:09 DEBUG "status": "success", 07:06:09 DEBUG "data": { 07:06:09 DEBUG "resultType": "vector", 07:06:09 DEBUG "result": [ 07:06:09 DEBUG { 07:06:09 DEBUG "metric": { 07:06:09 DEBUG "instance": "10.142.0.107:9100" 07:06:09 DEBUG }, 07:06:09 DEBUG "value": [ 07:06:09 DEBUG 07:06:09 DEBUG 07:06:09 DEBUG 07:06:09 DEBUG [...] 07:06:09 DEBUG 07:06:09 DEBUG 07:06:09 DEBUG 07:06:09 DEBUG "instance": "10.142.0.43:9100" 07:06:09 DEBUG }, 07:06:09 DEBUG "value": [ 07:06:09 DEBUG 1713593169, 07:06:09 DEBUG "50486424.83333333" 07:06:09 DEBUG ] 07:06:09 DEBUG } 07:06:09 DEBUG ] 07:06:09 DEBUG } 07:06:09 DEBUG } 07:06:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713593169" 07:06:09 INFO [http_cmd]: http status code OK 07:06:09 DEBUG --- status code --- 07:06:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:06:09 DEBUG --- http response --- 07:06:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:06:09 DEBUG ----- output ----- 07:06:09 DEBUG { 07:06:09 DEBUG "status": "success", 07:06:09 DEBUG "data": { 07:06:09 DEBUG "resultType": "vector", 07:06:09 DEBUG "result": [ 07:06:09 DEBUG { 07:06:09 DEBUG "metric": { 07:06:09 DEBUG "instance": "10.142.0.107:9100" 07:06:09 DEBUG }, 07:06:09 DEBUG "value": [ 07:06:09 DEBUG 07:06:09 DEBUG 07:06:09 DEBUG 07:06:09 DEBUG [...] 07:06:09 DEBUG 07:06:09 DEBUG 07:06:09 DEBUG 07:06:09 DEBUG "instance": "10.142.0.43:9100" 07:06:09 DEBUG }, 07:06:09 DEBUG "value": [ 07:06:09 DEBUG 1713593169, 07:06:09 DEBUG "0.11533333333334403" 07:06:09 DEBUG ] 07:06:09 DEBUG } 07:06:09 DEBUG ] 07:06:09 DEBUG } 07:06:09 DEBUG } 07:06:11 INFO 07:06:11 INFO [loop_until]: kubectl --namespace=xlou top node 07:06:11 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 07:06:11 INFO 07:06:11 INFO [loop_until]: kubectl --namespace=xlou top pods 07:06:11 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 07:06:11 INFO [loop_until]: OK (rc = 0) 07:06:11 DEBUG --- stdout --- 07:06:11 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 68m 0% 6263Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 4666m 15% 7464Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 11602m 38% 11371Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 80m 0% 3368Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 13869m 46% 13622Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 163m 0% 5177Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 11696m 39% 11592Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 10240m 34% 3682Mi 3% gke-xlou-cdm-ds-0f601751-hmm7 13651m 45% 10862Mi 9% gke-xlou-cdm-ds-0f601751-j488 5541m 18% 7030Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 13021m 43% 15667Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 4796m 16% 7698Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 14260m 47% 12887Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 8159m 51% 8767Mi 14% 07:06:11 DEBUG --- stderr --- 07:06:11 DEBUG 07:06:11 INFO [loop_until]: OK (rc = 0) 07:06:11 DEBUG --- stdout --- 07:06:11 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 11055m 8653Mi am-767f6b67b5-qcqz2 10623m 8712Mi am-767f6b67b5-vbltz 13913m 8650Mi ds-cts-0 14533m 10478Mi ds-cts-1 11932m 10662Mi ds-cts-2 14206m 10276Mi ds-idrepo-0 5467m 3786Mi ds-idrepo-1 5264m 3803Mi ds-idrepo-2 5352m 3712Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1177Mi idm-b85f9d978-wnhxb 3m 1236Mi lodemon-7cb5b9456d-dskfn 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7974m 5200Mi 07:06:11 DEBUG --- stderr --- 07:06:11 DEBUG 07:07:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713593229" 07:07:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713593229" 07:07:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593229" 07:07:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593229" 07:07:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593229" 07:07:09 ERROR [http_cmd]: ERROR 07:07:09 INFO [http_cmd]: http status code OK 07:07:09 ERROR --- status code --- 07:07:09 DEBUG --- status code --- 07:07:09 ERROR http status code is 400 (expected [200, 401, 403, 404]) 07:07:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:07:09 ERROR --- http response --- 07:07:09 DEBUG --- http response --- 07:07:09 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 07:07:09 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 07:07:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 07:07:09 DEBUG ----- output ----- 07:07:09 DEBUG { 07:07:09 DEBUG "status": "success", 07:07:09 DEBUG "data": { 07:07:09 DEBUG "resultType": "vector", 07:07:09 DEBUG "result": [ 07:07:09 DEBUG { 07:07:09 DEBUG "metric": { 07:07:09 DEBUG "pod": "ds-cts-0" 07:07:09 DEBUG }, 07:07:09 DEBUG "value": [ 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG [...] 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG "pod": "ds-cts-2" 07:07:09 DEBUG }, 07:07:09 DEBUG "value": [ 07:07:09 DEBUG 1713593229, 07:07:09 DEBUG "29.966666666666665" 07:07:09 DEBUG ] 07:07:09 DEBUG } 07:07:09 DEBUG ] 07:07:09 DEBUG } 07:07:09 DEBUG } 07:07:09 INFO [http_cmd]: http status code OK 07:07:09 DEBUG --- status code --- 07:07:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:07:09 DEBUG --- http response --- 07:07:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:07:09 DEBUG ----- output ----- 07:07:09 DEBUG { 07:07:09 DEBUG "status": "success", 07:07:09 DEBUG "data": { 07:07:09 DEBUG "resultType": "vector", 07:07:09 DEBUG "result": [ 07:07:09 DEBUG { 07:07:09 DEBUG "metric": { 07:07:09 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 07:07:09 DEBUG }, 07:07:09 DEBUG "value": [ 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG [...] 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 07:07:09 DEBUG }, 07:07:09 DEBUG "value": [ 07:07:09 DEBUG 1713593229, 07:07:09 DEBUG "7154626560" 07:07:09 DEBUG ] 07:07:09 DEBUG } 07:07:09 DEBUG ] 07:07:09 DEBUG } 07:07:09 DEBUG } 07:07:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593229" 07:07:09 INFO [http_cmd]: http status code OK 07:07:09 DEBUG --- status code --- 07:07:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:07:09 DEBUG --- http response --- 07:07:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:07:09 DEBUG ----- output ----- 07:07:09 DEBUG { 07:07:09 DEBUG "status": "success", 07:07:09 DEBUG "data": { 07:07:09 DEBUG "resultType": "vector", 07:07:09 DEBUG "result": [ 07:07:09 DEBUG { 07:07:09 DEBUG "metric": { 07:07:09 DEBUG "pod": "ds-idrepo-0" 07:07:09 DEBUG }, 07:07:09 DEBUG "value": [ 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG [...] 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG "pod": "ds-idrepo-2" 07:07:09 DEBUG }, 07:07:09 DEBUG "value": [ 07:07:09 DEBUG 1713593229, 07:07:09 DEBUG "0" 07:07:09 DEBUG ] 07:07:09 DEBUG } 07:07:09 DEBUG ] 07:07:09 DEBUG } 07:07:09 DEBUG } 07:07:09 INFO [http_cmd]: http status code OK 07:07:09 DEBUG --- status code --- 07:07:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:07:09 DEBUG --- http response --- 07:07:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:07:09 DEBUG ----- output ----- 07:07:09 DEBUG { 07:07:09 DEBUG "status": "success", 07:07:09 DEBUG "data": { 07:07:09 DEBUG "resultType": "vector", 07:07:09 DEBUG "result": [ 07:07:09 DEBUG { 07:07:09 DEBUG "metric": { 07:07:09 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 07:07:09 DEBUG }, 07:07:09 DEBUG "value": [ 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG [...] 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG "pod": "am-767f6b67b5-qcqz2" 07:07:09 DEBUG }, 07:07:09 DEBUG "value": [ 07:07:09 DEBUG 1713593229, 07:07:09 DEBUG "430.38508273987685" 07:07:09 DEBUG ] 07:07:09 DEBUG } 07:07:09 DEBUG ] 07:07:09 DEBUG } 07:07:09 DEBUG } 07:07:09 INFO [http_cmd]: http status code OK 07:07:09 DEBUG --- status code --- 07:07:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:07:09 DEBUG --- http response --- 07:07:09 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 07:07:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593229" 07:07:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593229" 07:07:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713593229" 07:07:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593229" 07:07:09 INFO [http_cmd]: http status code OK 07:07:09 DEBUG --- status code --- 07:07:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:07:09 DEBUG --- http response --- 07:07:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:07:09 DEBUG ----- output ----- 07:07:09 DEBUG { 07:07:09 DEBUG "status": "success", 07:07:09 DEBUG "data": { 07:07:09 DEBUG "resultType": "vector", 07:07:09 DEBUG "result": [ 07:07:09 DEBUG { 07:07:09 DEBUG "metric": { 07:07:09 DEBUG "pod": "ds-idrepo-0" 07:07:09 DEBUG }, 07:07:09 DEBUG "value": [ 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG [...] 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG "pod": "ds-idrepo-2" 07:07:09 DEBUG }, 07:07:09 DEBUG "value": [ 07:07:09 DEBUG 1713593229, 07:07:09 DEBUG "0" 07:07:09 DEBUG ] 07:07:09 DEBUG } 07:07:09 DEBUG ] 07:07:09 DEBUG } 07:07:09 DEBUG } 07:07:09 INFO [http_cmd]: http status code OK 07:07:09 DEBUG --- status code --- 07:07:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:07:09 DEBUG --- http response --- 07:07:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:07:09 DEBUG ----- output ----- 07:07:09 DEBUG { 07:07:09 DEBUG "status": "success", 07:07:09 DEBUG "data": { 07:07:09 DEBUG "resultType": "vector", 07:07:09 DEBUG "result": [ 07:07:09 DEBUG { 07:07:09 DEBUG "metric": { 07:07:09 DEBUG "pod": "am-767f6b67b5-9zw22" 07:07:09 DEBUG }, 07:07:09 DEBUG "value": [ 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG [...] 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG "pod": "am-767f6b67b5-vbltz" 07:07:09 DEBUG }, 07:07:09 DEBUG "value": [ 07:07:09 DEBUG 1713593229, 07:07:09 DEBUG "235430.5" 07:07:09 DEBUG ] 07:07:09 DEBUG } 07:07:09 DEBUG ] 07:07:09 DEBUG } 07:07:09 DEBUG } 07:07:09 INFO [http_cmd]: http status code OK 07:07:09 DEBUG --- status code --- 07:07:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:07:09 DEBUG --- http response --- 07:07:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:07:09 DEBUG ----- output ----- 07:07:09 DEBUG { 07:07:09 DEBUG "status": "success", 07:07:09 DEBUG "data": { 07:07:09 DEBUG "resultType": "vector", 07:07:09 DEBUG "result": [ 07:07:09 DEBUG { 07:07:09 DEBUG "metric": { 07:07:09 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 07:07:09 DEBUG }, 07:07:09 DEBUG "value": [ 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG [...] 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG "pod": "am-767f6b67b5-qcqz2" 07:07:09 DEBUG }, 07:07:09 DEBUG "value": [ 07:07:09 DEBUG 1713593229, 07:07:09 DEBUG "0" 07:07:09 DEBUG ] 07:07:09 DEBUG } 07:07:09 DEBUG ] 07:07:09 DEBUG } 07:07:09 DEBUG } 07:07:09 INFO [http_cmd]: http status code OK 07:07:09 DEBUG --- status code --- 07:07:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:07:09 DEBUG --- http response --- 07:07:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:07:09 DEBUG ----- output ----- 07:07:09 DEBUG { 07:07:09 DEBUG "status": "success", 07:07:09 DEBUG "data": { 07:07:09 DEBUG "resultType": "vector", 07:07:09 DEBUG "result": [ 07:07:09 DEBUG { 07:07:09 DEBUG "metric": { 07:07:09 DEBUG "instance": "10.142.0.107:9100" 07:07:09 DEBUG }, 07:07:09 DEBUG "value": [ 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG [...] 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG "instance": "10.142.0.43:9100" 07:07:09 DEBUG }, 07:07:09 DEBUG "value": [ 07:07:09 DEBUG 1713593229, 07:07:09 DEBUG "3291089050624" 07:07:09 DEBUG ] 07:07:09 DEBUG } 07:07:09 DEBUG ] 07:07:09 DEBUG } 07:07:09 DEBUG } 07:07:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593229" 07:07:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713593229" 07:07:09 INFO [http_cmd]: http status code OK 07:07:09 DEBUG --- status code --- 07:07:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:07:09 DEBUG --- http response --- 07:07:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:07:09 DEBUG ----- output ----- 07:07:09 DEBUG { 07:07:09 DEBUG "status": "success", 07:07:09 DEBUG "data": { 07:07:09 DEBUG "resultType": "vector", 07:07:09 DEBUG "result": [ 07:07:09 DEBUG { 07:07:09 DEBUG "metric": { 07:07:09 DEBUG "pod": "ds-idrepo-0" 07:07:09 DEBUG }, 07:07:09 DEBUG "value": [ 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG [...] 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG "pod": "ds-idrepo-2" 07:07:09 DEBUG }, 07:07:09 DEBUG "value": [ 07:07:09 DEBUG 1713593229, 07:07:09 DEBUG "0" 07:07:09 DEBUG ] 07:07:09 DEBUG } 07:07:09 DEBUG ] 07:07:09 DEBUG } 07:07:09 DEBUG } 07:07:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713593229" 07:07:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593229" 07:07:09 INFO [http_cmd]: http status code OK 07:07:09 DEBUG --- status code --- 07:07:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:07:09 DEBUG --- http response --- 07:07:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:07:09 DEBUG ----- output ----- 07:07:09 DEBUG { 07:07:09 DEBUG "status": "success", 07:07:09 DEBUG "data": { 07:07:09 DEBUG "resultType": "vector", 07:07:09 DEBUG "result": [ 07:07:09 DEBUG { 07:07:09 DEBUG "metric": { 07:07:09 DEBUG "instance": "10.142.0.107:9100" 07:07:09 DEBUG }, 07:07:09 DEBUG "value": [ 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG [...] 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG "instance": "10.142.0.43:9100" 07:07:09 DEBUG }, 07:07:09 DEBUG "value": [ 07:07:09 DEBUG 1713593229, 07:07:09 DEBUG "0.1511333333333681" 07:07:09 DEBUG ] 07:07:09 DEBUG } 07:07:09 DEBUG ] 07:07:09 DEBUG } 07:07:09 DEBUG } 07:07:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593229" 07:07:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593229" 07:07:09 INFO [http_cmd]: http status code OK 07:07:09 DEBUG --- status code --- 07:07:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:07:09 DEBUG --- http response --- 07:07:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:07:09 DEBUG ----- output ----- 07:07:09 DEBUG { 07:07:09 DEBUG "status": "success", 07:07:09 DEBUG "data": { 07:07:09 DEBUG "resultType": "vector", 07:07:09 DEBUG "result": [ 07:07:09 DEBUG { 07:07:09 DEBUG "metric": { 07:07:09 DEBUG "instance": "10.142.0.107:9100" 07:07:09 DEBUG }, 07:07:09 DEBUG "value": [ 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG [...] 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG "instance": "10.142.0.43:9100" 07:07:09 DEBUG }, 07:07:09 DEBUG "value": [ 07:07:09 DEBUG 1713593229, 07:07:09 DEBUG "43922299.53333333" 07:07:09 DEBUG ] 07:07:09 DEBUG } 07:07:09 DEBUG ] 07:07:09 DEBUG } 07:07:09 DEBUG } 07:07:09 INFO [http_cmd]: http status code OK 07:07:09 DEBUG --- status code --- 07:07:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:07:09 DEBUG --- http response --- 07:07:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:07:09 DEBUG ----- output ----- 07:07:09 DEBUG { 07:07:09 DEBUG "status": "success", 07:07:09 DEBUG "data": { 07:07:09 DEBUG "resultType": "vector", 07:07:09 DEBUG "result": [ 07:07:09 DEBUG { 07:07:09 DEBUG "metric": { 07:07:09 DEBUG "pod": "am-767f6b67b5-9zw22" 07:07:09 DEBUG }, 07:07:09 DEBUG "value": [ 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG [...] 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG "pod": "am-767f6b67b5-vbltz" 07:07:09 DEBUG }, 07:07:09 DEBUG "value": [ 07:07:09 DEBUG 1713593229, 07:07:09 DEBUG "2664.0666666666666" 07:07:09 DEBUG ] 07:07:09 DEBUG } 07:07:09 DEBUG ] 07:07:09 DEBUG } 07:07:09 DEBUG } 07:07:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713593229" 07:07:09 INFO [http_cmd]: http status code OK 07:07:09 DEBUG --- status code --- 07:07:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:07:09 DEBUG --- http response --- 07:07:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:07:09 DEBUG ----- output ----- 07:07:09 DEBUG { 07:07:09 DEBUG "status": "success", 07:07:09 DEBUG "data": { 07:07:09 DEBUG "resultType": "vector", 07:07:09 DEBUG "result": [ 07:07:09 DEBUG { 07:07:09 DEBUG "metric": { 07:07:09 DEBUG "pod": "am-767f6b67b5-9zw22" 07:07:09 DEBUG }, 07:07:09 DEBUG "value": [ 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG [...] 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG "pod": "am-767f6b67b5-vbltz" 07:07:09 DEBUG }, 07:07:09 DEBUG "value": [ 07:07:09 DEBUG 1713593229, 07:07:09 DEBUG "1300.7333333333333" 07:07:09 DEBUG ] 07:07:09 DEBUG } 07:07:09 DEBUG ] 07:07:09 DEBUG } 07:07:09 DEBUG } 07:07:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593229" 07:07:09 INFO [http_cmd]: http status code OK 07:07:09 DEBUG --- status code --- 07:07:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:07:09 DEBUG --- http response --- 07:07:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 07:07:09 DEBUG ----- output ----- 07:07:09 DEBUG { 07:07:09 DEBUG "status": "success", 07:07:09 DEBUG "data": { 07:07:09 DEBUG "resultType": "vector", 07:07:09 DEBUG "result": [ 07:07:09 DEBUG { 07:07:09 DEBUG "metric": { 07:07:09 DEBUG "pod": "ds-cts-0" 07:07:09 DEBUG }, 07:07:09 DEBUG "value": [ 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG [...] 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG "pod": "ds-cts-2" 07:07:09 DEBUG }, 07:07:09 DEBUG "value": [ 07:07:09 DEBUG 1713593229, 07:07:09 DEBUG "0" 07:07:09 DEBUG ] 07:07:09 DEBUG } 07:07:09 DEBUG ] 07:07:09 DEBUG } 07:07:09 DEBUG } 07:07:09 INFO [http_cmd]: http status code OK 07:07:09 DEBUG --- status code --- 07:07:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:07:09 DEBUG --- http response --- 07:07:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 07:07:09 DEBUG ----- output ----- 07:07:09 DEBUG { 07:07:09 DEBUG "status": "success", 07:07:09 DEBUG "data": { 07:07:09 DEBUG "resultType": "vector", 07:07:09 DEBUG "result": [ 07:07:09 DEBUG { 07:07:09 DEBUG "metric": { 07:07:09 DEBUG "pod": "ds-idrepo-0" 07:07:09 DEBUG }, 07:07:09 DEBUG "value": [ 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG [...] 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG "pod": "ds-cts-2" 07:07:09 DEBUG }, 07:07:09 DEBUG "value": [ 07:07:09 DEBUG 1713593229, 07:07:09 DEBUG "0" 07:07:09 DEBUG ] 07:07:09 DEBUG } 07:07:09 DEBUG ] 07:07:09 DEBUG } 07:07:09 DEBUG } 07:07:09 INFO [http_cmd]: http status code OK 07:07:09 DEBUG --- status code --- 07:07:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:07:09 DEBUG --- http response --- 07:07:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 07:07:09 DEBUG ----- output ----- 07:07:09 DEBUG { 07:07:09 DEBUG "status": "success", 07:07:09 DEBUG "data": { 07:07:09 DEBUG "resultType": "vector", 07:07:09 DEBUG "result": [ 07:07:09 DEBUG { 07:07:09 DEBUG "metric": { 07:07:09 DEBUG "pod": "ds-cts-0" 07:07:09 DEBUG }, 07:07:09 DEBUG "value": [ 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG [...] 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG "pod": "ds-cts-2" 07:07:09 DEBUG }, 07:07:09 DEBUG "value": [ 07:07:09 DEBUG 1713593229, 07:07:09 DEBUG "0.00064384" 07:07:09 DEBUG ] 07:07:09 DEBUG } 07:07:09 DEBUG ] 07:07:09 DEBUG } 07:07:09 DEBUG } 07:07:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593229" 07:07:09 INFO [http_cmd]: http status code OK 07:07:09 DEBUG --- status code --- 07:07:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:07:09 DEBUG --- http response --- 07:07:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:07:09 DEBUG ----- output ----- 07:07:09 DEBUG { 07:07:09 DEBUG "status": "success", 07:07:09 DEBUG "data": { 07:07:09 DEBUG "resultType": "vector", 07:07:09 DEBUG "result": [ 07:07:09 DEBUG { 07:07:09 DEBUG "metric": { 07:07:09 DEBUG "pod": "am-767f6b67b5-9zw22" 07:07:09 DEBUG }, 07:07:09 DEBUG "value": [ 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG [...] 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG "pod": "am-767f6b67b5-vbltz" 07:07:09 DEBUG }, 07:07:09 DEBUG "value": [ 07:07:09 DEBUG 1713593229, 07:07:09 DEBUG "1301.0333333333333" 07:07:09 DEBUG ] 07:07:09 DEBUG } 07:07:09 DEBUG ] 07:07:09 DEBUG } 07:07:09 DEBUG } 07:07:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713593229" 07:07:09 INFO [http_cmd]: http status code OK 07:07:09 DEBUG --- status code --- 07:07:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:07:09 DEBUG --- http response --- 07:07:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:07:09 DEBUG ----- output ----- 07:07:09 DEBUG { 07:07:09 DEBUG "status": "success", 07:07:09 DEBUG "data": { 07:07:09 DEBUG "resultType": "vector", 07:07:09 DEBUG "result": [ 07:07:09 DEBUG { 07:07:09 DEBUG "metric": { 07:07:09 DEBUG "pod": "am-767f6b67b5-9zw22" 07:07:09 DEBUG }, 07:07:09 DEBUG "value": [ 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG [...] 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG "pod": "am-767f6b67b5-vbltz" 07:07:09 DEBUG }, 07:07:09 DEBUG "value": [ 07:07:09 DEBUG 1713593229, 07:07:09 DEBUG "0.097" 07:07:09 DEBUG ] 07:07:09 DEBUG } 07:07:09 DEBUG ] 07:07:09 DEBUG } 07:07:09 DEBUG } 07:07:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713593229" 07:07:09 INFO [http_cmd]: http status code OK 07:07:09 DEBUG --- status code --- 07:07:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:07:09 DEBUG --- http response --- 07:07:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:07:09 DEBUG ----- output ----- 07:07:09 DEBUG { 07:07:09 DEBUG "status": "success", 07:07:09 DEBUG "data": { 07:07:09 DEBUG "resultType": "vector", 07:07:09 DEBUG "result": [ 07:07:09 DEBUG { 07:07:09 DEBUG "metric": { 07:07:09 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 07:07:09 DEBUG }, 07:07:09 DEBUG "value": [ 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG [...] 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 07:07:09 DEBUG }, 07:07:09 DEBUG "value": [ 07:07:09 DEBUG 1713593229, 07:07:09 DEBUG "8.031323108322855" 07:07:09 DEBUG ] 07:07:09 DEBUG } 07:07:09 DEBUG ] 07:07:09 DEBUG } 07:07:09 DEBUG } 07:07:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593229" 07:07:09 INFO [http_cmd]: http status code OK 07:07:09 DEBUG --- status code --- 07:07:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:07:09 DEBUG --- http response --- 07:07:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:07:09 DEBUG ----- output ----- 07:07:09 DEBUG { 07:07:09 DEBUG "status": "success", 07:07:09 DEBUG "data": { 07:07:09 DEBUG "resultType": "vector", 07:07:09 DEBUG "result": [ 07:07:09 DEBUG { 07:07:09 DEBUG "metric": { 07:07:09 DEBUG "pod": "am-767f6b67b5-9zw22" 07:07:09 DEBUG }, 07:07:09 DEBUG "value": [ 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG [...] 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG "pod": "am-767f6b67b5-vbltz" 07:07:09 DEBUG }, 07:07:09 DEBUG "value": [ 07:07:09 DEBUG 1713593229, 07:07:09 DEBUG "1303.7333333333333" 07:07:09 DEBUG ] 07:07:09 DEBUG } 07:07:09 DEBUG ] 07:07:09 DEBUG } 07:07:09 DEBUG } 07:07:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593229" 07:07:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593229" 07:07:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593229" 07:07:09 INFO [http_cmd]: http status code OK 07:07:09 DEBUG --- status code --- 07:07:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:07:09 DEBUG --- http response --- 07:07:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 07:07:09 DEBUG ----- output ----- 07:07:09 DEBUG { 07:07:09 DEBUG "status": "success", 07:07:09 DEBUG "data": { 07:07:09 DEBUG "resultType": "vector", 07:07:09 DEBUG "result": [ 07:07:09 DEBUG { 07:07:09 DEBUG "metric": { 07:07:09 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 07:07:09 DEBUG }, 07:07:09 DEBUG "value": [ 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG [...] 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG "pod": "ds-idrepo-2" 07:07:09 DEBUG }, 07:07:09 DEBUG "value": [ 07:07:09 DEBUG 1713593229, 07:07:09 DEBUG "0" 07:07:09 DEBUG ] 07:07:09 DEBUG } 07:07:09 DEBUG ] 07:07:09 DEBUG } 07:07:09 DEBUG } 07:07:09 INFO [http_cmd]: http status code OK 07:07:09 DEBUG --- status code --- 07:07:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:07:09 DEBUG --- http response --- 07:07:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:07:09 DEBUG ----- output ----- 07:07:09 DEBUG { 07:07:09 DEBUG "status": "success", 07:07:09 DEBUG "data": { 07:07:09 DEBUG "resultType": "vector", 07:07:09 DEBUG "result": [ 07:07:09 DEBUG { 07:07:09 DEBUG "metric": { 07:07:09 DEBUG "pod": "am-767f6b67b5-9zw22" 07:07:09 DEBUG }, 07:07:09 DEBUG "value": [ 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG [...] 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG "pod": "am-767f6b67b5-vbltz" 07:07:09 DEBUG }, 07:07:09 DEBUG "value": [ 07:07:09 DEBUG 1713593229, 07:07:09 DEBUG "0" 07:07:09 DEBUG ] 07:07:09 DEBUG } 07:07:09 DEBUG ] 07:07:09 DEBUG } 07:07:09 DEBUG } 07:07:09 INFO [http_cmd]: http status code OK 07:07:09 DEBUG --- status code --- 07:07:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:07:09 DEBUG --- http response --- 07:07:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 07:07:09 DEBUG ----- output ----- 07:07:09 DEBUG { 07:07:09 DEBUG "status": "success", 07:07:09 DEBUG "data": { 07:07:09 DEBUG "resultType": "vector", 07:07:09 DEBUG "result": [ 07:07:09 DEBUG { 07:07:09 DEBUG "metric": { 07:07:09 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 07:07:09 DEBUG }, 07:07:09 DEBUG "value": [ 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG [...] 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG "pod": "ds-idrepo-2" 07:07:09 DEBUG }, 07:07:09 DEBUG "value": [ 07:07:09 DEBUG 1713593229, 07:07:09 DEBUG "0" 07:07:09 DEBUG ] 07:07:09 DEBUG } 07:07:09 DEBUG ] 07:07:09 DEBUG } 07:07:09 DEBUG } 07:07:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713593229" 07:07:09 INFO [http_cmd]: http status code OK 07:07:09 DEBUG --- status code --- 07:07:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:07:09 DEBUG --- http response --- 07:07:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:07:09 DEBUG ----- output ----- 07:07:09 DEBUG { 07:07:09 DEBUG "status": "success", 07:07:09 DEBUG "data": { 07:07:09 DEBUG "resultType": "vector", 07:07:09 DEBUG "result": [ 07:07:09 DEBUG { 07:07:09 DEBUG "metric": { 07:07:09 DEBUG "instance": "10.142.0.107:9100" 07:07:09 DEBUG }, 07:07:09 DEBUG "value": [ 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG [...] 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG "instance": "10.142.0.43:9100" 07:07:09 DEBUG }, 07:07:09 DEBUG "value": [ 07:07:09 DEBUG 1713593229, 07:07:09 DEBUG "43923186.766666666" 07:07:09 DEBUG ] 07:07:09 DEBUG } 07:07:09 DEBUG ] 07:07:09 DEBUG } 07:07:09 DEBUG } 07:07:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713593229" 07:07:09 INFO [http_cmd]: http status code OK 07:07:09 DEBUG --- status code --- 07:07:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:07:09 DEBUG --- http response --- 07:07:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:07:09 DEBUG ----- output ----- 07:07:09 DEBUG { 07:07:09 DEBUG "status": "success", 07:07:09 DEBUG "data": { 07:07:09 DEBUG "resultType": "vector", 07:07:09 DEBUG "result": [ 07:07:09 DEBUG { 07:07:09 DEBUG "metric": { 07:07:09 DEBUG "instance": "10.142.0.107:9100" 07:07:09 DEBUG }, 07:07:09 DEBUG "value": [ 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG [...] 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG 07:07:09 DEBUG "instance": "10.142.0.43:9100" 07:07:09 DEBUG }, 07:07:09 DEBUG "value": [ 07:07:09 DEBUG 1713593229, 07:07:09 DEBUG "0.07866666666668228" 07:07:09 DEBUG ] 07:07:09 DEBUG } 07:07:09 DEBUG ] 07:07:09 DEBUG } 07:07:09 DEBUG } 07:07:11 INFO 07:07:11 INFO [loop_until]: kubectl --namespace=xlou top node 07:07:11 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 07:07:11 INFO 07:07:11 INFO [loop_until]: kubectl --namespace=xlou top pods 07:07:11 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 07:07:11 INFO [loop_until]: OK (rc = 0) 07:07:11 DEBUG --- stdout --- 07:07:11 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 72m 0% 6271Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 4177m 13% 7467Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 9463m 31% 11362Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 83m 0% 3362Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 12390m 41% 13622Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 130m 0% 5195Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 9623m 32% 11593Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 8370m 28% 3702Mi 3% gke-xlou-cdm-ds-0f601751-hmm7 10559m 35% 10874Mi 9% gke-xlou-cdm-ds-0f601751-j488 4333m 14% 7034Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 11116m 37% 15581Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 4324m 14% 7706Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 12765m 42% 13422Mi 12% gke-xlou-cdm-frontend-a8771548-i82n 8237m 51% 11133Mi 18% 07:07:11 DEBUG --- stderr --- 07:07:11 DEBUG 07:07:11 INFO [loop_until]: OK (rc = 0) 07:07:11 DEBUG --- stdout --- 07:07:11 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 9282m 8650Mi am-767f6b67b5-qcqz2 9377m 8711Mi am-767f6b67b5-vbltz 10469m 8651Mi ds-cts-0 12478m 10488Mi ds-cts-1 11046m 10671Mi ds-cts-2 12720m 10639Mi ds-idrepo-0 4175m 3786Mi ds-idrepo-1 3993m 3808Mi ds-idrepo-2 4029m 3714Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1177Mi idm-b85f9d978-wnhxb 3m 1236Mi lodemon-7cb5b9456d-dskfn 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7979m 7741Mi 07:07:11 DEBUG --- stderr --- 07:07:11 DEBUG 07:08:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713593289" 07:08:09 ERROR [http_cmd]: ERROR 07:08:09 ERROR --- status code --- 07:08:09 ERROR http status code is 400 (expected [200, 401, 403, 404]) 07:08:09 ERROR --- http response --- 07:08:09 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 07:08:09 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 07:08:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593289" 07:08:09 INFO [http_cmd]: http status code OK 07:08:09 DEBUG --- status code --- 07:08:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:08:09 DEBUG --- http response --- 07:08:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:08:09 DEBUG ----- output ----- 07:08:09 DEBUG { 07:08:09 DEBUG "status": "success", 07:08:09 DEBUG "data": { 07:08:09 DEBUG "resultType": "vector", 07:08:09 DEBUG "result": [ 07:08:09 DEBUG { 07:08:09 DEBUG "metric": { 07:08:09 DEBUG "pod": "ds-idrepo-0" 07:08:09 DEBUG }, 07:08:09 DEBUG "value": [ 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG [...] 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG "pod": "ds-idrepo-2" 07:08:09 DEBUG }, 07:08:09 DEBUG "value": [ 07:08:09 DEBUG 1713593289, 07:08:09 DEBUG "0" 07:08:09 DEBUG ] 07:08:09 DEBUG } 07:08:09 DEBUG ] 07:08:09 DEBUG } 07:08:09 DEBUG } 07:08:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593289" 07:08:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593289" 07:08:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713593289" 07:08:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593289" 07:08:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593289" 07:08:09 INFO [http_cmd]: http status code OK 07:08:09 DEBUG --- status code --- 07:08:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:08:09 DEBUG --- http response --- 07:08:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 07:08:09 DEBUG ----- output ----- 07:08:09 DEBUG { 07:08:09 DEBUG "status": "success", 07:08:09 DEBUG "data": { 07:08:09 DEBUG "resultType": "vector", 07:08:09 DEBUG "result": [ 07:08:09 DEBUG { 07:08:09 DEBUG "metric": { 07:08:09 DEBUG "pod": "ds-cts-0" 07:08:09 DEBUG }, 07:08:09 DEBUG "value": [ 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG [...] 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG "pod": "ds-cts-2" 07:08:09 DEBUG }, 07:08:09 DEBUG "value": [ 07:08:09 DEBUG 1713593289, 07:08:09 DEBUG "29.933333333333334" 07:08:09 DEBUG ] 07:08:09 DEBUG } 07:08:09 DEBUG ] 07:08:09 DEBUG } 07:08:09 INFO [http_cmd]: http status code OK 07:08:09 DEBUG } 07:08:09 DEBUG --- status code --- 07:08:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:08:09 DEBUG --- http response --- 07:08:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:08:09 DEBUG ----- output ----- 07:08:09 DEBUG { 07:08:09 DEBUG "status": "success", 07:08:09 DEBUG "data": { 07:08:09 DEBUG "resultType": "vector", 07:08:09 DEBUG "result": [ 07:08:09 DEBUG { 07:08:09 DEBUG "metric": { 07:08:09 DEBUG "pod": "ds-idrepo-0" 07:08:09 DEBUG }, 07:08:09 DEBUG "value": [ 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG [...] 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG "pod": "ds-idrepo-2" 07:08:09 DEBUG }, 07:08:09 DEBUG "value": [ 07:08:09 DEBUG 1713593289, 07:08:09 DEBUG "0" 07:08:09 DEBUG ] 07:08:09 DEBUG } 07:08:09 DEBUG ] 07:08:09 DEBUG } 07:08:09 DEBUG } 07:08:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593289" 07:08:09 INFO [http_cmd]: http status code OK 07:08:09 DEBUG --- status code --- 07:08:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:08:09 DEBUG --- http response --- 07:08:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:08:09 DEBUG ----- output ----- 07:08:09 DEBUG { 07:08:09 DEBUG "status": "success", 07:08:09 DEBUG "data": { 07:08:09 DEBUG "resultType": "vector", 07:08:09 DEBUG "result": [ 07:08:09 DEBUG { 07:08:09 DEBUG "metric": { 07:08:09 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 07:08:09 DEBUG }, 07:08:09 DEBUG "value": [ 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG [...] 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 07:08:09 DEBUG }, 07:08:09 DEBUG "value": [ 07:08:09 DEBUG 1713593289, 07:08:09 DEBUG "8391929856" 07:08:09 DEBUG ] 07:08:09 DEBUG } 07:08:09 DEBUG ] 07:08:09 DEBUG } 07:08:09 DEBUG } 07:08:09 INFO [http_cmd]: http status code OK 07:08:09 DEBUG --- status code --- 07:08:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:08:09 DEBUG --- http response --- 07:08:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 07:08:09 DEBUG ----- output ----- 07:08:09 DEBUG { 07:08:09 DEBUG "status": "success", 07:08:09 DEBUG "data": { 07:08:09 DEBUG "resultType": "vector", 07:08:09 DEBUG "result": [ 07:08:09 DEBUG { 07:08:09 DEBUG "metric": { 07:08:09 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 07:08:09 DEBUG }, 07:08:09 DEBUG "value": [ 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG [...] 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG "pod": "ds-cts-2" 07:08:09 DEBUG }, 07:08:09 DEBUG "value": [ 07:08:09 DEBUG 1713593289, 07:08:09 DEBUG "22971057.727257706" 07:08:09 DEBUG ] 07:08:09 DEBUG } 07:08:09 DEBUG ] 07:08:09 DEBUG } 07:08:09 DEBUG } 07:08:09 INFO [http_cmd]: http status code OK 07:08:09 DEBUG --- status code --- 07:08:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:08:09 DEBUG --- http response --- 07:08:09 INFO [http_cmd]: http status code OK 07:08:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:08:09 DEBUG --- status code --- 07:08:09 DEBUG ----- output ----- 07:08:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:08:09 DEBUG { 07:08:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713593289" 07:08:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593289" 07:08:09 DEBUG --- http response --- 07:08:09 DEBUG "status": "success", 07:08:09 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 07:08:09 DEBUG "data": { 07:08:09 DEBUG "resultType": "vector", 07:08:09 DEBUG "result": [ 07:08:09 DEBUG { 07:08:09 DEBUG "metric": { 07:08:09 DEBUG "pod": "am-767f6b67b5-9zw22" 07:08:09 DEBUG }, 07:08:09 DEBUG "value": [ 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG [...] 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG "pod": "am-767f6b67b5-vbltz" 07:08:09 DEBUG }, 07:08:09 DEBUG "value": [ 07:08:09 DEBUG 1713593289, 07:08:09 DEBUG "250015" 07:08:09 DEBUG ] 07:08:09 DEBUG } 07:08:09 DEBUG ] 07:08:09 DEBUG } 07:08:09 DEBUG } 07:08:09 INFO [http_cmd]: http status code OK 07:08:09 INFO [http_cmd]: http status code OK 07:08:09 DEBUG --- status code --- 07:08:09 DEBUG --- status code --- 07:08:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:08:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:08:09 DEBUG --- http response --- 07:08:09 DEBUG --- http response --- 07:08:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:08:09 DEBUG ----- output ----- 07:08:09 DEBUG { 07:08:09 DEBUG "status": "success", 07:08:09 DEBUG "data": { 07:08:09 DEBUG "resultType": "vector", 07:08:09 DEBUG "result": [ 07:08:09 DEBUG { 07:08:09 DEBUG "metric": { 07:08:09 DEBUG "instance": "10.142.0.107:9100" 07:08:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 07:08:09 DEBUG }, 07:08:09 DEBUG ----- output ----- 07:08:09 DEBUG "value": [ 07:08:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713593289" 07:08:09 DEBUG { 07:08:09 DEBUG 07:08:09 DEBUG "status": "success", 07:08:09 DEBUG 07:08:09 DEBUG "data": { 07:08:09 DEBUG 07:08:09 DEBUG "resultType": "vector", 07:08:09 DEBUG [...] 07:08:09 DEBUG "result": [ 07:08:09 DEBUG 07:08:09 DEBUG { 07:08:09 DEBUG 07:08:09 DEBUG "metric": { 07:08:09 DEBUG 07:08:09 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 07:08:09 DEBUG "instance": "10.142.0.43:9100" 07:08:09 DEBUG }, 07:08:09 DEBUG }, 07:08:09 DEBUG "value": [ 07:08:09 DEBUG "value": [ 07:08:09 DEBUG 07:08:09 DEBUG 1713593289, 07:08:09 DEBUG 07:08:09 DEBUG "3292821971968" 07:08:09 DEBUG 07:08:09 DEBUG ] 07:08:09 DEBUG [...] 07:08:09 DEBUG } 07:08:09 DEBUG 07:08:09 DEBUG ] 07:08:09 DEBUG 07:08:09 DEBUG } 07:08:09 DEBUG 07:08:09 DEBUG } 07:08:09 DEBUG "pod": "ds-cts-2" 07:08:09 DEBUG }, 07:08:09 DEBUG "value": [ 07:08:09 DEBUG 1713593289, 07:08:09 DEBUG "135.3467931137032" 07:08:09 DEBUG ] 07:08:09 DEBUG } 07:08:09 DEBUG ] 07:08:09 DEBUG } 07:08:09 DEBUG } 07:08:09 INFO [http_cmd]: http status code OK 07:08:09 DEBUG --- status code --- 07:08:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:08:09 DEBUG --- http response --- 07:08:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:08:09 DEBUG ----- output ----- 07:08:09 DEBUG { 07:08:09 DEBUG "status": "success", 07:08:09 DEBUG "data": { 07:08:09 DEBUG "resultType": "vector", 07:08:09 DEBUG "result": [ 07:08:09 DEBUG { 07:08:09 DEBUG "metric": { 07:08:09 DEBUG "instance": "10.142.0.107:9100" 07:08:09 DEBUG }, 07:08:09 DEBUG "value": [ 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG [...] 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG "instance": "10.142.0.43:9100" 07:08:09 DEBUG }, 07:08:09 DEBUG "value": [ 07:08:09 DEBUG 1713593289, 07:08:09 DEBUG "37462734.733333334" 07:08:09 DEBUG ] 07:08:09 DEBUG } 07:08:09 DEBUG ] 07:08:09 DEBUG } 07:08:09 DEBUG } 07:08:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713593289" 07:08:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593289" 07:08:09 INFO [http_cmd]: http status code OK 07:08:09 INFO [http_cmd]: http status code OK 07:08:09 DEBUG --- status code --- 07:08:09 DEBUG --- status code --- 07:08:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:08:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:08:09 DEBUG --- http response --- 07:08:09 DEBUG --- http response --- 07:08:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:08:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 07:08:09 DEBUG ----- output ----- 07:08:09 DEBUG ----- output ----- 07:08:09 DEBUG { 07:08:09 DEBUG { 07:08:09 DEBUG "status": "success", 07:08:09 DEBUG "status": "success", 07:08:09 DEBUG "data": { 07:08:09 DEBUG "data": { 07:08:09 DEBUG "resultType": "vector", 07:08:09 DEBUG "resultType": "vector", 07:08:09 DEBUG "result": [ 07:08:09 DEBUG "result": [ 07:08:09 DEBUG { 07:08:09 DEBUG { 07:08:09 DEBUG "metric": { 07:08:09 DEBUG "metric": { 07:08:09 DEBUG "pod": "ds-idrepo-0" 07:08:09 DEBUG "pod": "ds-idrepo-0" 07:08:09 DEBUG }, 07:08:09 DEBUG }, 07:08:09 DEBUG "value": [ 07:08:09 DEBUG "value": [ 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG [...] 07:08:09 DEBUG [...] 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713593289" 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG "pod": "ds-idrepo-2" 07:08:09 DEBUG "pod": "ds-cts-2" 07:08:09 DEBUG }, 07:08:09 DEBUG }, 07:08:09 DEBUG "value": [ 07:08:09 DEBUG "value": [ 07:08:09 DEBUG 1713593289, 07:08:09 DEBUG 1713593289, 07:08:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593289" 07:08:09 DEBUG "0" 07:08:09 DEBUG "0" 07:08:09 DEBUG ] 07:08:09 DEBUG ] 07:08:09 DEBUG } 07:08:09 DEBUG } 07:08:09 DEBUG ] 07:08:09 DEBUG ] 07:08:09 DEBUG } 07:08:09 DEBUG } 07:08:09 DEBUG } 07:08:09 DEBUG } 07:08:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593289" 07:08:09 INFO [http_cmd]: http status code OK 07:08:09 DEBUG --- status code --- 07:08:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:08:09 DEBUG --- http response --- 07:08:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 07:08:09 DEBUG ----- output ----- 07:08:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593289" 07:08:09 DEBUG { 07:08:09 DEBUG "status": "success", 07:08:09 DEBUG "data": { 07:08:09 DEBUG "resultType": "vector", 07:08:09 DEBUG "result": [ 07:08:09 DEBUG { 07:08:09 DEBUG "metric": { 07:08:09 DEBUG "pod": "ds-cts-0" 07:08:09 DEBUG }, 07:08:09 DEBUG "value": [ 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG [...] 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 INFO [http_cmd]: http status code OK 07:08:09 DEBUG "pod": "ds-cts-2" 07:08:09 DEBUG --- status code --- 07:08:09 DEBUG }, 07:08:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:08:09 DEBUG "value": [ 07:08:09 DEBUG --- http response --- 07:08:09 DEBUG 1713593289, 07:08:09 DEBUG "0.000025437777777777778" 07:08:09 DEBUG ] 07:08:09 DEBUG } 07:08:09 DEBUG ] 07:08:09 DEBUG } 07:08:09 DEBUG } 07:08:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:08:09 DEBUG ----- output ----- 07:08:09 DEBUG { 07:08:09 DEBUG "status": "success", 07:08:09 DEBUG "data": { 07:08:09 DEBUG "resultType": "vector", 07:08:09 DEBUG "result": [ 07:08:09 DEBUG { 07:08:09 DEBUG "metric": { 07:08:09 DEBUG "instance": "10.142.0.107:9100" 07:08:09 DEBUG }, 07:08:09 DEBUG "value": [ 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG [...] 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG "instance": "10.142.0.43:9100" 07:08:09 DEBUG }, 07:08:09 DEBUG "value": [ 07:08:09 DEBUG 1713593289, 07:08:09 DEBUG "0.0356333333333841" 07:08:09 DEBUG ] 07:08:09 DEBUG } 07:08:09 DEBUG ] 07:08:09 DEBUG } 07:08:09 DEBUG } 07:08:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593289" 07:08:09 INFO [http_cmd]: http status code OK 07:08:09 DEBUG --- status code --- 07:08:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:08:09 DEBUG --- http response --- 07:08:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:08:09 DEBUG ----- output ----- 07:08:09 DEBUG { 07:08:09 DEBUG "status": "success", 07:08:09 DEBUG "data": { 07:08:09 DEBUG "resultType": "vector", 07:08:09 DEBUG "result": [ 07:08:09 DEBUG { 07:08:09 DEBUG "metric": { 07:08:09 DEBUG "pod": "am-767f6b67b5-9zw22" 07:08:09 DEBUG }, 07:08:09 DEBUG "value": [ 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG [...] 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG "pod": "am-767f6b67b5-vbltz" 07:08:09 DEBUG }, 07:08:09 DEBUG "value": [ 07:08:09 DEBUG 1713593289, 07:08:09 DEBUG "120.39999999999999" 07:08:09 DEBUG ] 07:08:09 DEBUG } 07:08:09 DEBUG ] 07:08:09 DEBUG } 07:08:09 DEBUG } 07:08:09 INFO [http_cmd]: http status code OK 07:08:09 DEBUG --- status code --- 07:08:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:08:09 DEBUG --- http response --- 07:08:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:08:09 DEBUG ----- output ----- 07:08:09 DEBUG { 07:08:09 DEBUG "status": "success", 07:08:09 DEBUG "data": { 07:08:09 DEBUG "resultType": "vector", 07:08:09 DEBUG "result": [ 07:08:09 DEBUG { 07:08:09 DEBUG "metric": { 07:08:09 DEBUG "pod": "am-767f6b67b5-9zw22" 07:08:09 DEBUG }, 07:08:09 DEBUG "value": [ 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG [...] 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG "pod": "am-767f6b67b5-vbltz" 07:08:09 DEBUG }, 07:08:09 DEBUG "value": [ 07:08:09 DEBUG 1713593289, 07:08:09 DEBUG "1307.5333333333333" 07:08:09 DEBUG ] 07:08:09 DEBUG } 07:08:09 DEBUG ] 07:08:09 DEBUG } 07:08:09 DEBUG } 07:08:09 INFO [http_cmd]: http status code OK 07:08:09 DEBUG --- status code --- 07:08:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:08:09 DEBUG --- http response --- 07:08:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 07:08:09 DEBUG ----- output ----- 07:08:09 DEBUG { 07:08:09 DEBUG "status": "success", 07:08:09 DEBUG "data": { 07:08:09 DEBUG "resultType": "vector", 07:08:09 DEBUG "result": [ 07:08:09 DEBUG { 07:08:09 DEBUG "metric": { 07:08:09 DEBUG "pod": "ds-cts-0" 07:08:09 DEBUG }, 07:08:09 DEBUG "value": [ 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG [...] 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG "pod": "ds-cts-2" 07:08:09 DEBUG }, 07:08:09 DEBUG "value": [ 07:08:09 DEBUG 1713593289, 07:08:09 DEBUG "0" 07:08:09 DEBUG ] 07:08:09 DEBUG } 07:08:09 DEBUG ] 07:08:09 DEBUG } 07:08:09 DEBUG } 07:08:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593289" 07:08:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713593289" 07:08:09 INFO [http_cmd]: http status code OK 07:08:09 DEBUG --- status code --- 07:08:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:08:09 DEBUG --- http response --- 07:08:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:08:09 DEBUG ----- output ----- 07:08:09 DEBUG { 07:08:09 DEBUG "status": "success", 07:08:09 DEBUG "data": { 07:08:09 DEBUG "resultType": "vector", 07:08:09 DEBUG "result": [ 07:08:09 DEBUG { 07:08:09 DEBUG "metric": { 07:08:09 DEBUG "pod": "am-767f6b67b5-9zw22" 07:08:09 DEBUG }, 07:08:09 DEBUG "value": [ 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG [...] 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG "pod": "am-767f6b67b5-vbltz" 07:08:09 DEBUG }, 07:08:09 DEBUG "value": [ 07:08:09 DEBUG 1713593289, 07:08:09 DEBUG "1307.5666666666666" 07:08:09 DEBUG ] 07:08:09 DEBUG } 07:08:09 DEBUG ] 07:08:09 DEBUG } 07:08:09 DEBUG } 07:08:09 INFO [http_cmd]: http status code OK 07:08:09 DEBUG --- status code --- 07:08:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:08:09 DEBUG --- http response --- 07:08:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:08:09 DEBUG ----- output ----- 07:08:09 DEBUG { 07:08:09 DEBUG "status": "success", 07:08:09 DEBUG "data": { 07:08:09 DEBUG "resultType": "vector", 07:08:09 DEBUG "result": [ 07:08:09 DEBUG { 07:08:09 DEBUG "metric": { 07:08:09 DEBUG "pod": "am-767f6b67b5-9zw22" 07:08:09 DEBUG }, 07:08:09 DEBUG "value": [ 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG [...] 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG "pod": "am-767f6b67b5-vbltz" 07:08:09 DEBUG }, 07:08:09 DEBUG "value": [ 07:08:09 DEBUG 1713593289, 07:08:09 DEBUG "0.097" 07:08:09 DEBUG ] 07:08:09 DEBUG } 07:08:09 DEBUG ] 07:08:09 DEBUG } 07:08:09 DEBUG } 07:08:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713593289" 07:08:09 INFO [http_cmd]: http status code OK 07:08:09 DEBUG --- status code --- 07:08:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:08:09 DEBUG --- http response --- 07:08:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:08:09 DEBUG ----- output ----- 07:08:09 DEBUG { 07:08:09 DEBUG "status": "success", 07:08:09 DEBUG "data": { 07:08:09 DEBUG "resultType": "vector", 07:08:09 DEBUG "result": [ 07:08:09 DEBUG { 07:08:09 DEBUG "metric": { 07:08:09 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 07:08:09 DEBUG }, 07:08:09 DEBUG "value": [ 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG [...] 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 07:08:09 DEBUG }, 07:08:09 DEBUG "value": [ 07:08:09 DEBUG 1713593289, 07:08:09 DEBUG "8.026946395194651" 07:08:09 DEBUG ] 07:08:09 DEBUG } 07:08:09 DEBUG ] 07:08:09 DEBUG } 07:08:09 DEBUG } 07:08:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593289" 07:08:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593289" 07:08:09 INFO [http_cmd]: http status code OK 07:08:09 DEBUG --- status code --- 07:08:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:08:09 DEBUG --- http response --- 07:08:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:08:09 DEBUG ----- output ----- 07:08:09 DEBUG { 07:08:09 DEBUG "status": "success", 07:08:09 DEBUG "data": { 07:08:09 DEBUG "resultType": "vector", 07:08:09 DEBUG "result": [ 07:08:09 DEBUG { 07:08:09 DEBUG "metric": { 07:08:09 DEBUG "pod": "am-767f6b67b5-9zw22" 07:08:09 DEBUG }, 07:08:09 DEBUG "value": [ 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG [...] 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG "pod": "am-767f6b67b5-vbltz" 07:08:09 DEBUG }, 07:08:09 DEBUG "value": [ 07:08:09 DEBUG 1713593289, 07:08:09 DEBUG "1307.9333333333334" 07:08:09 DEBUG ] 07:08:09 DEBUG } 07:08:09 DEBUG ] 07:08:09 DEBUG } 07:08:09 DEBUG } 07:08:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593289" 07:08:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593289" 07:08:09 INFO [http_cmd]: http status code OK 07:08:09 DEBUG --- status code --- 07:08:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:08:09 DEBUG --- http response --- 07:08:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:08:09 DEBUG ----- output ----- 07:08:09 DEBUG { 07:08:09 DEBUG "status": "success", 07:08:09 DEBUG "data": { 07:08:09 DEBUG "resultType": "vector", 07:08:09 DEBUG "result": [ 07:08:09 DEBUG { 07:08:09 DEBUG "metric": { 07:08:09 DEBUG "pod": "am-767f6b67b5-9zw22" 07:08:09 DEBUG }, 07:08:09 DEBUG "value": [ 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG [...] 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG "pod": "am-767f6b67b5-vbltz" 07:08:09 DEBUG }, 07:08:09 DEBUG "value": [ 07:08:09 DEBUG 1713593289, 07:08:09 DEBUG "0" 07:08:09 DEBUG ] 07:08:09 DEBUG } 07:08:09 DEBUG ] 07:08:09 DEBUG } 07:08:09 DEBUG } 07:08:09 INFO [http_cmd]: http status code OK 07:08:09 DEBUG --- status code --- 07:08:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:08:09 DEBUG --- http response --- 07:08:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:08:09 DEBUG ----- output ----- 07:08:09 DEBUG { 07:08:09 DEBUG "status": "success", 07:08:09 DEBUG "data": { 07:08:09 DEBUG "resultType": "vector", 07:08:09 DEBUG "result": [ 07:08:09 DEBUG { 07:08:09 DEBUG "metric": { 07:08:09 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 07:08:09 DEBUG }, 07:08:09 DEBUG "value": [ 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG [...] 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG "pod": "idm-b85f9d978-wnhxb" 07:08:09 DEBUG }, 07:08:09 DEBUG "value": [ 07:08:09 DEBUG 1713593289, 07:08:09 DEBUG "0" 07:08:09 DEBUG ] 07:08:09 DEBUG } 07:08:09 DEBUG ] 07:08:09 DEBUG } 07:08:09 DEBUG } 07:08:09 INFO [http_cmd]: http status code OK 07:08:09 DEBUG --- status code --- 07:08:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:08:09 DEBUG --- http response --- 07:08:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:08:09 DEBUG ----- output ----- 07:08:09 DEBUG { 07:08:09 DEBUG "status": "success", 07:08:09 DEBUG "data": { 07:08:09 DEBUG "resultType": "vector", 07:08:09 DEBUG "result": [ 07:08:09 DEBUG { 07:08:09 DEBUG "metric": { 07:08:09 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 07:08:09 DEBUG }, 07:08:09 DEBUG "value": [ 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG [...] 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG 07:08:09 DEBUG "pod": "idm-b85f9d978-wnhxb" 07:08:09 DEBUG }, 07:08:09 DEBUG "value": [ 07:08:09 DEBUG 1713593289, 07:08:09 DEBUG "0" 07:08:09 DEBUG ] 07:08:09 DEBUG } 07:08:09 DEBUG ] 07:08:09 DEBUG } 07:08:09 DEBUG } 07:08:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713593289" 07:08:10 INFO [http_cmd]: http status code OK 07:08:10 DEBUG --- status code --- 07:08:10 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:08:10 DEBUG --- http response --- 07:08:10 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:08:10 DEBUG ----- output ----- 07:08:10 DEBUG { 07:08:10 DEBUG "status": "success", 07:08:10 DEBUG "data": { 07:08:10 DEBUG "resultType": "vector", 07:08:10 DEBUG "result": [ 07:08:10 DEBUG { 07:08:10 DEBUG "metric": { 07:08:10 DEBUG "instance": "10.142.0.107:9100" 07:08:10 DEBUG }, 07:08:10 DEBUG "value": [ 07:08:10 DEBUG 07:08:10 DEBUG 07:08:10 DEBUG 07:08:10 DEBUG [...] 07:08:10 DEBUG 07:08:10 DEBUG 07:08:10 DEBUG 07:08:10 DEBUG "instance": "10.142.0.43:9100" 07:08:10 DEBUG }, 07:08:10 DEBUG "value": [ 07:08:10 DEBUG 1713593289, 07:08:10 DEBUG "37463752.03333333" 07:08:10 DEBUG ] 07:08:10 DEBUG } 07:08:10 DEBUG ] 07:08:10 DEBUG } 07:08:10 DEBUG } 07:08:10 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713593290" 07:08:10 INFO [http_cmd]: http status code OK 07:08:10 DEBUG --- status code --- 07:08:10 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:08:10 DEBUG --- http response --- 07:08:10 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:08:10 DEBUG ----- output ----- 07:08:10 DEBUG { 07:08:10 DEBUG "status": "success", 07:08:10 DEBUG "data": { 07:08:10 DEBUG "resultType": "vector", 07:08:10 DEBUG "result": [ 07:08:10 DEBUG { 07:08:10 DEBUG "metric": { 07:08:10 DEBUG "instance": "10.142.0.107:9100" 07:08:10 DEBUG }, 07:08:10 DEBUG "value": [ 07:08:10 DEBUG 07:08:10 DEBUG 07:08:10 DEBUG 07:08:10 DEBUG [...] 07:08:10 DEBUG 07:08:10 DEBUG 07:08:10 DEBUG 07:08:10 DEBUG "instance": "10.142.0.43:9100" 07:08:10 DEBUG }, 07:08:10 DEBUG "value": [ 07:08:10 DEBUG 1713593290, 07:08:10 DEBUG "0.023000000000001814" 07:08:10 DEBUG ] 07:08:10 DEBUG } 07:08:10 DEBUG ] 07:08:10 DEBUG } 07:08:10 DEBUG } 07:08:11 INFO 07:08:11 INFO [loop_until]: kubectl --namespace=xlou top node 07:08:11 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 07:08:11 INFO 07:08:11 INFO [loop_until]: kubectl --namespace=xlou top pods 07:08:11 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 07:08:11 INFO [loop_until]: OK (rc = 0) 07:08:11 DEBUG --- stdout --- 07:08:11 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 6523m 8650Mi am-767f6b67b5-qcqz2 6516m 8711Mi am-767f6b67b5-vbltz 7293m 8648Mi ds-cts-0 3786m 10491Mi ds-cts-1 4002m 10679Mi ds-cts-2 3933m 10219Mi ds-idrepo-0 3224m 3785Mi ds-idrepo-1 3274m 3807Mi ds-idrepo-2 3206m 3714Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1177Mi idm-b85f9d978-wnhxb 3m 1236Mi lodemon-7cb5b9456d-dskfn 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 7832m 7945Mi 07:08:11 DEBUG --- stderr --- 07:08:11 DEBUG 07:08:11 INFO [loop_until]: OK (rc = 0) 07:08:11 DEBUG --- stdout --- 07:08:11 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 71m 0% 6275Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 3444m 11% 7466Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 6751m 22% 11368Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 85m 0% 3358Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 4226m 14% 13652Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 150m 0% 5188Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 6719m 22% 11596Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 8166m 27% 3700Mi 3% gke-xlou-cdm-ds-0f601751-hmm7 7516m 25% 10873Mi 9% gke-xlou-cdm-ds-0f601751-j488 3467m 11% 7026Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 3639m 12% 15597Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 3448m 11% 7701Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 4038m 13% 12869Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 8233m 51% 10987Mi 18% 07:08:11 DEBUG --- stderr --- 07:08:11 DEBUG 07:08:42 WARNING No values available for selected timeframe. 07:08:42 WARNING No values available for selected timeframe. 07:08:42 WARNING No values available for selected timeframe. 07:08:42 WARNING No values available for selected timeframe. 07:08:42 WARNING No values available for selected timeframe. 07:08:42 WARNING No values available for selected timeframe. 07:08:42 WARNING No values available for selected timeframe. 07:08:42 WARNING No values available for selected timeframe. 07:08:42 WARNING No values available for selected timeframe. 07:08:42 WARNING No values available for selected timeframe. 07:08:42 WARNING No values available for selected timeframe. 07:08:42 WARNING No values available for selected timeframe. 07:08:42 WARNING No values available for selected timeframe. 07:08:42 WARNING No values available for selected timeframe. 07:08:42 WARNING No values available for selected timeframe. 07:08:42 WARNING No values available for selected timeframe. 07:08:42 WARNING File /tmp/lodemon_data-20240420_063158/node_disk_read_bytes_total.json does not exist. Skipping average value computation... 127.0.0.1 - - [20/Apr/2024 07:08:42] "GET /monitoring/average?start_time=20240420_060622&stop_time=20240420_060641 HTTP/1.1" 200 - 07:09:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593349" 07:09:09 INFO [http_cmd]: http status code OK 07:09:09 DEBUG --- status code --- 07:09:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:09:09 DEBUG --- http response --- 07:09:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:09:09 DEBUG ----- output ----- 07:09:09 DEBUG { 07:09:09 DEBUG "status": "success", 07:09:09 DEBUG "data": { 07:09:09 DEBUG "resultType": "vector", 07:09:09 DEBUG "result": [ 07:09:09 DEBUG { 07:09:09 DEBUG "metric": { 07:09:09 DEBUG "pod": "ds-idrepo-0" 07:09:09 DEBUG }, 07:09:09 DEBUG "value": [ 07:09:09 DEBUG 07:09:09 DEBUG 07:09:09 DEBUG 07:09:09 DEBUG [...] 07:09:09 DEBUG 07:09:09 DEBUG 07:09:09 DEBUG 07:09:09 DEBUG "pod": "ds-idrepo-2" 07:09:09 DEBUG }, 07:09:09 DEBUG "value": [ 07:09:09 DEBUG 1713593349, 07:09:09 DEBUG "0" 07:09:09 DEBUG ] 07:09:09 DEBUG } 07:09:09 DEBUG ] 07:09:09 DEBUG } 07:09:09 DEBUG } 07:09:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713593349" 07:09:09 ERROR [http_cmd]: ERROR 07:09:09 ERROR --- status code --- 07:09:09 ERROR http status code is 400 (expected [200, 401, 403, 404]) 07:09:09 ERROR --- http response --- 07:09:09 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 07:09:09 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 07:09:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593349" 07:09:09 INFO [http_cmd]: http status code OK 07:09:09 DEBUG --- status code --- 07:09:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:09:09 DEBUG --- http response --- 07:09:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:09:09 DEBUG ----- output ----- 07:09:09 DEBUG { 07:09:09 DEBUG "status": "success", 07:09:09 DEBUG "data": { 07:09:09 DEBUG "resultType": "vector", 07:09:09 DEBUG "result": [ 07:09:09 DEBUG { 07:09:09 DEBUG "metric": { 07:09:09 DEBUG "pod": "am-767f6b67b5-9zw22" 07:09:09 DEBUG }, 07:09:09 DEBUG "value": [ 07:09:09 DEBUG 07:09:09 DEBUG 07:09:09 DEBUG 07:09:09 DEBUG [...] 07:09:09 DEBUG 07:09:09 DEBUG 07:09:09 DEBUG 07:09:09 DEBUG "pod": "am-767f6b67b5-vbltz" 07:09:09 DEBUG }, 07:09:09 DEBUG "value": [ 07:09:09 DEBUG 1713593349, 07:09:09 DEBUG "216263" 07:09:09 DEBUG ] 07:09:09 DEBUG } 07:09:09 DEBUG ] 07:09:09 DEBUG } 07:09:09 DEBUG } 07:09:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593349" 07:09:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593349" 07:09:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713593349" 07:09:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593349" 07:09:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593349" 07:09:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713593349" 07:09:09 INFO [http_cmd]: http status code OK 07:09:09 DEBUG --- status code --- 07:09:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:09:09 DEBUG --- http response --- 07:09:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:09:09 DEBUG ----- output ----- 07:09:09 DEBUG { 07:09:09 DEBUG "status": "success", 07:09:09 DEBUG "data": { 07:09:09 DEBUG "resultType": "vector", 07:09:09 DEBUG "result": [ 07:09:09 DEBUG { 07:09:09 DEBUG "metric": { 07:09:09 DEBUG "pod": "ds-idrepo-0" 07:09:09 DEBUG }, 07:09:09 DEBUG "value": [ 07:09:09 DEBUG 07:09:09 DEBUG 07:09:09 DEBUG 07:09:09 DEBUG [...] 07:09:09 DEBUG 07:09:09 DEBUG 07:09:09 DEBUG 07:09:09 DEBUG "pod": "ds-idrepo-2" 07:09:09 DEBUG }, 07:09:09 DEBUG "value": [ 07:09:09 DEBUG 1713593349, 07:09:09 DEBUG "0" 07:09:09 DEBUG ] 07:09:09 DEBUG } 07:09:09 DEBUG ] 07:09:09 DEBUG } 07:09:09 DEBUG } 07:09:09 INFO [http_cmd]: http status code OK 07:09:09 DEBUG --- status code --- 07:09:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:09:09 DEBUG --- http response --- 07:09:09 INFO [http_cmd]: http status code OK 07:09:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 07:09:09 DEBUG --- status code --- 07:09:09 DEBUG ----- output ----- 07:09:09 INFO [http_cmd]: http status code OK 07:09:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:09:09 DEBUG { 07:09:09 DEBUG --- status code --- 07:09:09 DEBUG --- http response --- 07:09:09 DEBUG "status": "success", 07:09:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:09:09 DEBUG "data": { 07:09:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 07:09:09 DEBUG --- http response --- 07:09:09 DEBUG "resultType": "vector", 07:09:09 DEBUG ----- output ----- 07:09:09 DEBUG "result": [ 07:09:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:09:09 DEBUG { 07:09:09 DEBUG { 07:09:09 DEBUG ----- output ----- 07:09:09 DEBUG "status": "success", 07:09:09 DEBUG "metric": { 07:09:09 DEBUG { 07:09:09 DEBUG "data": { 07:09:09 DEBUG "pod": "ds-cts-0" 07:09:09 DEBUG "status": "success", 07:09:09 DEBUG "resultType": "vector", 07:09:09 DEBUG }, 07:09:09 DEBUG "data": { 07:09:09 DEBUG "result": [ 07:09:09 DEBUG "value": [ 07:09:09 DEBUG "resultType": "vector", 07:09:09 DEBUG { 07:09:09 DEBUG 07:09:09 DEBUG "result": [ 07:09:09 DEBUG "metric": { 07:09:09 DEBUG 07:09:09 DEBUG { 07:09:09 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 07:09:09 DEBUG 07:09:09 DEBUG "metric": { 07:09:09 DEBUG }, 07:09:09 DEBUG [...] 07:09:09 INFO [http_cmd]: http status code OK 07:09:09 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 07:09:09 DEBUG "value": [ 07:09:09 DEBUG 07:09:09 INFO [http_cmd]: http status code OK 07:09:09 DEBUG --- status code --- 07:09:09 DEBUG }, 07:09:09 DEBUG 07:09:09 DEBUG 07:09:09 DEBUG --- status code --- 07:09:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:09:09 DEBUG "value": [ 07:09:09 DEBUG 07:09:09 DEBUG 07:09:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:09:09 DEBUG --- http response --- 07:09:09 DEBUG 07:09:09 DEBUG 07:09:09 DEBUG "pod": "ds-cts-2" 07:09:09 DEBUG --- http response --- 07:09:09 DEBUG 07:09:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:09:09 DEBUG [...] 07:09:09 DEBUG }, 07:09:09 DEBUG 07:09:09 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 07:09:09 DEBUG ----- output ----- 07:09:09 DEBUG 07:09:09 DEBUG "value": [ 07:09:09 DEBUG [...] 07:09:09 DEBUG { 07:09:09 DEBUG 07:09:09 DEBUG 1713593349, 07:09:09 DEBUG 07:09:09 DEBUG "status": "success", 07:09:09 DEBUG 07:09:09 DEBUG "29.966666666666665" 07:09:09 DEBUG 07:09:09 DEBUG "data": { 07:09:09 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 07:09:09 DEBUG ] 07:09:09 DEBUG 07:09:09 DEBUG "resultType": "vector", 07:09:09 DEBUG }, 07:09:09 DEBUG } 07:09:09 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 07:09:09 DEBUG "result": [ 07:09:09 DEBUG "value": [ 07:09:09 DEBUG ] 07:09:09 DEBUG }, 07:09:09 DEBUG { 07:09:09 DEBUG 1713593349, 07:09:09 DEBUG } 07:09:09 DEBUG "value": [ 07:09:09 DEBUG "metric": { 07:09:09 DEBUG "0" 07:09:09 DEBUG } 07:09:09 DEBUG 1713593349, 07:09:09 DEBUG "instance": "10.142.0.107:9100" 07:09:09 DEBUG ] 07:09:09 DEBUG "111677440" 07:09:09 DEBUG }, 07:09:09 DEBUG } 07:09:09 DEBUG ] 07:09:09 DEBUG "value": [ 07:09:09 DEBUG ] 07:09:09 DEBUG } 07:09:09 DEBUG 07:09:09 DEBUG } 07:09:09 DEBUG ] 07:09:09 DEBUG 07:09:09 DEBUG } 07:09:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593349" 07:09:09 DEBUG } 07:09:09 DEBUG 07:09:09 DEBUG } 07:09:09 DEBUG [...] 07:09:09 DEBUG 07:09:09 DEBUG 07:09:09 DEBUG 07:09:09 DEBUG "instance": "10.142.0.43:9100" 07:09:09 DEBUG }, 07:09:09 DEBUG "value": [ 07:09:09 DEBUG 1713593349, 07:09:09 DEBUG "3294127397888" 07:09:09 DEBUG ] 07:09:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593349" 07:09:09 DEBUG } 07:09:09 DEBUG ] 07:09:09 DEBUG } 07:09:09 DEBUG } 07:09:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713593349" 07:09:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713593349" 07:09:09 INFO [http_cmd]: http status code OK 07:09:09 DEBUG --- status code --- 07:09:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:09:09 DEBUG --- http response --- 07:09:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 07:09:09 DEBUG ----- output ----- 07:09:09 DEBUG { 07:09:09 DEBUG "status": "success", 07:09:09 DEBUG "data": { 07:09:09 DEBUG "resultType": "vector", 07:09:09 DEBUG "result": [ 07:09:09 DEBUG { 07:09:09 DEBUG "metric": { 07:09:09 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 07:09:09 DEBUG }, 07:09:09 DEBUG "value": [ 07:09:09 DEBUG 07:09:09 DEBUG 07:09:09 DEBUG 07:09:09 DEBUG [...] 07:09:09 DEBUG 07:09:09 DEBUG 07:09:09 DEBUG 07:09:09 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 07:09:09 DEBUG }, 07:09:09 DEBUG "value": [ 07:09:09 DEBUG 1713593349, 07:09:09 DEBUG "40849.02592388238" 07:09:09 DEBUG ] 07:09:09 DEBUG } 07:09:09 DEBUG ] 07:09:09 DEBUG } 07:09:09 DEBUG } 07:09:09 INFO [http_cmd]: http status code OK 07:09:09 INFO [http_cmd]: http status code OK 07:09:09 INFO [http_cmd]: http status code OK 07:09:09 DEBUG --- status code --- 07:09:09 DEBUG --- status code --- 07:09:09 DEBUG --- status code --- 07:09:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:09:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:09:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:09:09 DEBUG --- http response --- 07:09:09 DEBUG --- http response --- 07:09:09 DEBUG --- http response --- 07:09:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:09:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 07:09:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:09:09 DEBUG ----- output ----- 07:09:09 DEBUG ----- output ----- 07:09:09 DEBUG ----- output ----- 07:09:09 DEBUG { 07:09:09 DEBUG { 07:09:09 DEBUG { 07:09:09 DEBUG "status": "success", 07:09:09 DEBUG "status": "success", 07:09:09 DEBUG "status": "success", 07:09:09 DEBUG "data": { 07:09:09 DEBUG "data": { 07:09:09 DEBUG "data": { 07:09:09 DEBUG "resultType": "vector", 07:09:09 DEBUG "resultType": "vector", 07:09:09 DEBUG "resultType": "vector", 07:09:09 DEBUG "result": [ 07:09:09 DEBUG "result": [ 07:09:09 DEBUG "result": [ 07:09:09 DEBUG { 07:09:09 DEBUG { 07:09:09 DEBUG { 07:09:09 DEBUG "metric": { 07:09:09 DEBUG "metric": { 07:09:09 DEBUG "metric": { 07:09:09 DEBUG "instance": "10.142.0.107:9100" 07:09:09 DEBUG "pod": "ds-cts-0" 07:09:09 DEBUG "instance": "10.142.0.107:9100" 07:09:09 DEBUG }, 07:09:09 DEBUG }, 07:09:09 DEBUG }, 07:09:09 DEBUG "value": [ 07:09:09 DEBUG "value": [ 07:09:09 DEBUG "value": [ 07:09:09 DEBUG 07:09:09 DEBUG 07:09:09 DEBUG 07:09:09 DEBUG 07:09:09 DEBUG 07:09:09 DEBUG 07:09:09 DEBUG 07:09:09 DEBUG 07:09:09 DEBUG 07:09:09 DEBUG [...] 07:09:09 DEBUG [...] 07:09:09 DEBUG [...] 07:09:09 DEBUG 07:09:09 DEBUG 07:09:09 DEBUG 07:09:09 DEBUG 07:09:09 DEBUG 07:09:09 DEBUG 07:09:09 DEBUG 07:09:09 DEBUG 07:09:09 DEBUG 07:09:09 DEBUG "instance": "10.142.0.43:9100" 07:09:09 DEBUG "pod": "ds-cts-2" 07:09:09 DEBUG "instance": "10.142.0.43:9100" 07:09:09 DEBUG }, 07:09:09 DEBUG }, 07:09:09 DEBUG }, 07:09:09 DEBUG "value": [ 07:09:09 DEBUG "value": [ 07:09:09 DEBUG "value": [ 07:09:09 DEBUG 1713593349, 07:09:09 DEBUG 1713593349, 07:09:09 DEBUG 1713593349, 07:09:09 DEBUG "0.015233333333350174" 07:09:09 DEBUG "0" 07:09:09 DEBUG "9504056.8" 07:09:09 DEBUG ] 07:09:09 DEBUG ] 07:09:09 DEBUG ] 07:09:09 DEBUG } 07:09:09 DEBUG } 07:09:09 DEBUG } 07:09:09 DEBUG ] 07:09:09 DEBUG ] 07:09:09 DEBUG ] 07:09:09 DEBUG } 07:09:09 DEBUG } 07:09:09 DEBUG } 07:09:09 DEBUG } 07:09:09 DEBUG } 07:09:09 DEBUG } 07:09:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593349" 07:09:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593349" 07:09:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713593349" 07:09:09 INFO [http_cmd]: http status code OK 07:09:09 DEBUG --- status code --- 07:09:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:09:09 DEBUG --- http response --- 07:09:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:09:09 DEBUG ----- output ----- 07:09:09 DEBUG { 07:09:09 DEBUG "status": "success", 07:09:09 DEBUG "data": { 07:09:09 DEBUG "resultType": "vector", 07:09:09 DEBUG "result": [ 07:09:09 DEBUG { 07:09:09 DEBUG "metric": { 07:09:09 DEBUG "pod": "am-767f6b67b5-9zw22" 07:09:09 DEBUG }, 07:09:09 DEBUG "value": [ 07:09:09 DEBUG 07:09:09 DEBUG 07:09:09 DEBUG 07:09:09 DEBUG [...] 07:09:09 DEBUG 07:09:09 DEBUG 07:09:09 DEBUG 07:09:09 DEBUG "pod": "am-767f6b67b5-vbltz" 07:09:09 DEBUG }, 07:09:09 DEBUG "value": [ 07:09:09 DEBUG 1713593349, 07:09:09 DEBUG "0" 07:09:09 DEBUG ] 07:09:09 DEBUG } 07:09:09 DEBUG ] 07:09:09 DEBUG } 07:09:09 DEBUG } 07:09:09 INFO [http_cmd]: http status code OK 07:09:09 INFO [http_cmd]: http status code OK 07:09:09 DEBUG --- status code --- 07:09:09 DEBUG --- status code --- 07:09:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:09:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:09:09 DEBUG --- http response --- 07:09:09 DEBUG --- http response --- 07:09:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 07:09:09 DEBUG ----- output ----- 07:09:09 DEBUG { 07:09:09 DEBUG "status": "success", 07:09:09 DEBUG "data": { 07:09:09 DEBUG "resultType": "vector", 07:09:09 DEBUG "result": [ 07:09:09 DEBUG { 07:09:09 DEBUG "metric": { 07:09:09 DEBUG "pod": "ds-idrepo-0" 07:09:09 DEBUG }, 07:09:09 DEBUG "value": [ 07:09:09 DEBUG 07:09:09 DEBUG 07:09:09 DEBUG 07:09:09 DEBUG [...] 07:09:09 DEBUG 07:09:09 DEBUG 07:09:09 DEBUG 07:09:09 DEBUG "pod": "ds-cts-2" 07:09:09 DEBUG }, 07:09:09 DEBUG "value": [ 07:09:09 DEBUG 1713593349, 07:09:09 DEBUG "0" 07:09:09 DEBUG ] 07:09:09 DEBUG } 07:09:09 DEBUG ] 07:09:09 DEBUG } 07:09:09 DEBUG } 07:09:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:09:09 DEBUG ----- output ----- 07:09:09 DEBUG { 07:09:09 DEBUG "status": "success", 07:09:09 DEBUG "data": { 07:09:09 DEBUG "resultType": "vector", 07:09:09 DEBUG "result": [ 07:09:09 DEBUG { 07:09:09 DEBUG "metric": { 07:09:09 DEBUG "pod": "ds-idrepo-0" 07:09:09 DEBUG }, 07:09:09 DEBUG "value": [ 07:09:09 DEBUG 07:09:09 DEBUG 07:09:09 DEBUG 07:09:09 DEBUG [...] 07:09:09 DEBUG 07:09:09 DEBUG 07:09:09 DEBUG 07:09:09 DEBUG "pod": "ds-idrepo-2" 07:09:09 DEBUG }, 07:09:09 DEBUG "value": [ 07:09:09 DEBUG 1713593349, 07:09:09 DEBUG "0" 07:09:09 DEBUG ] 07:09:09 DEBUG } 07:09:09 DEBUG ] 07:09:09 DEBUG } 07:09:09 DEBUG } 07:09:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593349" 07:09:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593349" 07:09:09 INFO [http_cmd]: http status code OK 07:09:09 DEBUG --- status code --- 07:09:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:09:09 DEBUG --- http response --- 07:09:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 07:09:09 DEBUG ----- output ----- 07:09:09 DEBUG { 07:09:09 DEBUG "status": "success", 07:09:09 DEBUG "data": { 07:09:09 DEBUG "resultType": "vector", 07:09:09 DEBUG "result": [ 07:09:09 DEBUG { 07:09:09 DEBUG "metric": { 07:09:09 DEBUG "pod": "ds-cts-0" 07:09:09 DEBUG }, 07:09:09 DEBUG "value": [ 07:09:09 DEBUG 07:09:09 DEBUG 07:09:09 DEBUG 07:09:09 DEBUG [...] 07:09:09 DEBUG 07:09:09 DEBUG 07:09:09 DEBUG 07:09:09 DEBUG "pod": "ds-cts-2" 07:09:09 DEBUG }, 07:09:09 DEBUG "value": [ 07:09:09 DEBUG 1713593349, 07:09:09 DEBUG "0" 07:09:09 DEBUG ] 07:09:09 DEBUG } 07:09:09 DEBUG ] 07:09:09 DEBUG } 07:09:09 DEBUG } 07:09:09 INFO [http_cmd]: http status code OK 07:09:09 DEBUG --- status code --- 07:09:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:09:09 DEBUG --- http response --- 07:09:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:09:09 DEBUG ----- output ----- 07:09:09 DEBUG { 07:09:09 DEBUG "status": "success", 07:09:09 DEBUG "data": { 07:09:09 DEBUG "resultType": "vector", 07:09:09 DEBUG "result": [ 07:09:09 DEBUG { 07:09:09 DEBUG "metric": { 07:09:09 DEBUG "pod": "am-767f6b67b5-9zw22" 07:09:09 DEBUG }, 07:09:09 DEBUG "value": [ 07:09:09 DEBUG 07:09:09 DEBUG 07:09:09 DEBUG 07:09:09 DEBUG [...] 07:09:09 DEBUG 07:09:09 DEBUG 07:09:09 DEBUG 07:09:09 DEBUG "pod": "am-767f6b67b5-vbltz" 07:09:09 DEBUG }, 07:09:09 DEBUG "value": [ 07:09:09 DEBUG 1713593349, 07:09:09 DEBUG "1605.0333333333333" 07:09:09 DEBUG ] 07:09:09 DEBUG } 07:09:09 DEBUG ] 07:09:09 DEBUG } 07:09:09 DEBUG } 07:09:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593349" 07:09:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713593349" 07:09:09 INFO [http_cmd]: http status code OK 07:09:09 DEBUG --- status code --- 07:09:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:09:09 DEBUG --- http response --- 07:09:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:09:09 DEBUG ----- output ----- 07:09:09 DEBUG { 07:09:09 DEBUG "status": "success", 07:09:09 DEBUG "data": { 07:09:09 DEBUG "resultType": "vector", 07:09:09 DEBUG "result": [ 07:09:09 DEBUG { 07:09:09 DEBUG "metric": { 07:09:09 DEBUG "pod": "am-767f6b67b5-9zw22" 07:09:09 DEBUG }, 07:09:09 DEBUG "value": [ 07:09:09 DEBUG 07:09:09 DEBUG 07:09:09 DEBUG 07:09:09 DEBUG [...] 07:09:09 DEBUG 07:09:09 DEBUG 07:09:09 DEBUG 07:09:09 DEBUG "pod": "am-767f6b67b5-vbltz" 07:09:09 DEBUG }, 07:09:09 DEBUG "value": [ 07:09:09 DEBUG 1713593349, 07:09:09 DEBUG "0" 07:09:09 DEBUG ] 07:09:09 DEBUG } 07:09:09 DEBUG ] 07:09:09 DEBUG } 07:09:09 DEBUG } 07:09:09 INFO [http_cmd]: http status code OK 07:09:09 DEBUG --- status code --- 07:09:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:09:09 DEBUG --- http response --- 07:09:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:09:09 DEBUG ----- output ----- 07:09:09 DEBUG { 07:09:09 DEBUG "status": "success", 07:09:09 DEBUG "data": { 07:09:09 DEBUG "resultType": "vector", 07:09:09 DEBUG "result": [ 07:09:09 DEBUG { 07:09:09 DEBUG "metric": { 07:09:09 DEBUG "pod": "am-767f6b67b5-9zw22" 07:09:09 DEBUG }, 07:09:09 DEBUG "value": [ 07:09:09 DEBUG 07:09:09 DEBUG 07:09:09 DEBUG 07:09:09 DEBUG [...] 07:09:09 DEBUG 07:09:09 DEBUG 07:09:09 DEBUG 07:09:09 DEBUG "pod": "am-767f6b67b5-vbltz" 07:09:09 DEBUG }, 07:09:09 DEBUG "value": [ 07:09:09 DEBUG 1713593349, 07:09:09 DEBUG "0.097" 07:09:09 DEBUG ] 07:09:09 DEBUG } 07:09:09 DEBUG ] 07:09:09 DEBUG } 07:09:09 DEBUG } 07:09:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713593349" 07:09:09 INFO [http_cmd]: http status code OK 07:09:09 DEBUG --- status code --- 07:09:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:09:09 DEBUG --- http response --- 07:09:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:09:09 DEBUG ----- output ----- 07:09:09 DEBUG { 07:09:09 DEBUG "status": "success", 07:09:09 DEBUG "data": { 07:09:09 DEBUG "resultType": "vector", 07:09:09 DEBUG "result": [ 07:09:09 DEBUG { 07:09:09 DEBUG "metric": { 07:09:09 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 07:09:09 DEBUG }, 07:09:09 DEBUG "value": [ 07:09:09 DEBUG 07:09:09 DEBUG 07:09:09 DEBUG 07:09:09 DEBUG [...] 07:09:09 DEBUG 07:09:09 DEBUG 07:09:09 DEBUG 07:09:09 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 07:09:09 DEBUG }, 07:09:09 DEBUG "value": [ 07:09:09 DEBUG 1713593349, 07:09:09 DEBUG "5.780093878395471" 07:09:09 DEBUG ] 07:09:09 DEBUG } 07:09:09 DEBUG ] 07:09:09 DEBUG } 07:09:09 DEBUG } 07:09:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593349" 07:09:10 INFO [http_cmd]: http status code OK 07:09:10 DEBUG --- status code --- 07:09:10 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:09:10 DEBUG --- http response --- 07:09:10 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:09:10 DEBUG ----- output ----- 07:09:10 DEBUG { 07:09:10 DEBUG "status": "success", 07:09:10 DEBUG "data": { 07:09:10 DEBUG "resultType": "vector", 07:09:10 DEBUG "result": [ 07:09:10 DEBUG { 07:09:10 DEBUG "metric": { 07:09:10 DEBUG "pod": "am-767f6b67b5-9zw22" 07:09:10 DEBUG }, 07:09:10 DEBUG "value": [ 07:09:10 DEBUG 07:09:10 DEBUG 07:09:10 DEBUG 07:09:10 DEBUG [...] 07:09:10 DEBUG 07:09:10 DEBUG 07:09:10 DEBUG 07:09:10 DEBUG "pod": "am-767f6b67b5-vbltz" 07:09:10 DEBUG }, 07:09:10 DEBUG "value": [ 07:09:10 DEBUG 1713593349, 07:09:10 DEBUG "0" 07:09:10 DEBUG ] 07:09:10 DEBUG } 07:09:10 DEBUG ] 07:09:10 DEBUG } 07:09:10 DEBUG } 07:09:10 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593350" 07:09:10 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593350" 07:09:10 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593350" 07:09:10 INFO [http_cmd]: http status code OK 07:09:10 INFO [http_cmd]: http status code OK 07:09:10 DEBUG --- status code --- 07:09:10 DEBUG --- status code --- 07:09:10 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:09:10 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:09:10 DEBUG --- http response --- 07:09:10 DEBUG --- http response --- 07:09:10 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:09:10 DEBUG ----- output ----- 07:09:10 DEBUG { 07:09:10 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 07:09:10 DEBUG "status": "success", 07:09:10 DEBUG ----- output ----- 07:09:10 DEBUG "data": { 07:09:10 DEBUG { 07:09:10 DEBUG "resultType": "vector", 07:09:10 DEBUG "status": "success", 07:09:10 DEBUG "result": [ 07:09:10 DEBUG "data": { 07:09:10 DEBUG { 07:09:10 DEBUG "resultType": "vector", 07:09:10 DEBUG "metric": { 07:09:10 DEBUG "result": [ 07:09:10 DEBUG "pod": "am-767f6b67b5-9zw22" 07:09:10 DEBUG { 07:09:10 DEBUG }, 07:09:10 DEBUG "metric": { 07:09:10 DEBUG "value": [ 07:09:10 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 07:09:10 DEBUG 07:09:10 DEBUG }, 07:09:10 DEBUG 07:09:10 DEBUG "value": [ 07:09:10 DEBUG 07:09:10 DEBUG 07:09:10 DEBUG [...] 07:09:10 DEBUG 07:09:10 DEBUG 07:09:10 DEBUG 07:09:10 DEBUG 07:09:10 DEBUG [...] 07:09:10 DEBUG 07:09:10 DEBUG 07:09:10 DEBUG "pod": "am-767f6b67b5-vbltz" 07:09:10 DEBUG 07:09:10 DEBUG }, 07:09:10 DEBUG 07:09:10 DEBUG "value": [ 07:09:10 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 07:09:10 DEBUG 1713593350, 07:09:10 DEBUG }, 07:09:10 DEBUG "0" 07:09:10 DEBUG "value": [ 07:09:10 DEBUG ] 07:09:10 DEBUG 1713593350, 07:09:10 DEBUG } 07:09:10 DEBUG "0.006032623339931538" 07:09:10 DEBUG ] 07:09:10 DEBUG ] 07:09:10 DEBUG } 07:09:10 DEBUG } 07:09:10 DEBUG } 07:09:10 DEBUG ] 07:09:10 DEBUG } 07:09:10 DEBUG } 07:09:10 INFO [http_cmd]: http status code OK 07:09:10 DEBUG --- status code --- 07:09:10 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:09:10 DEBUG --- http response --- 07:09:10 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 07:09:10 DEBUG ----- output ----- 07:09:10 DEBUG { 07:09:10 DEBUG "status": "success", 07:09:10 DEBUG "data": { 07:09:10 DEBUG "resultType": "vector", 07:09:10 DEBUG "result": [ 07:09:10 DEBUG { 07:09:10 DEBUG "metric": { 07:09:10 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 07:09:10 DEBUG }, 07:09:10 DEBUG "value": [ 07:09:10 DEBUG 07:09:10 DEBUG 07:09:10 DEBUG 07:09:10 DEBUG [...] 07:09:10 DEBUG 07:09:10 DEBUG 07:09:10 DEBUG 07:09:10 DEBUG "pod": "overseer-0-5c67bf94db-77mh9" 07:09:10 DEBUG }, 07:09:10 DEBUG "value": [ 07:09:10 DEBUG 1713593350, 07:09:10 DEBUG "9655.816379311556" 07:09:10 DEBUG ] 07:09:10 DEBUG } 07:09:10 DEBUG ] 07:09:10 DEBUG } 07:09:10 DEBUG } 07:09:10 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713593350" 07:09:10 INFO [http_cmd]: http status code OK 07:09:10 DEBUG --- status code --- 07:09:10 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:09:10 DEBUG --- http response --- 07:09:10 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:09:10 DEBUG ----- output ----- 07:09:10 DEBUG { 07:09:10 DEBUG "status": "success", 07:09:10 DEBUG "data": { 07:09:10 DEBUG "resultType": "vector", 07:09:10 DEBUG "result": [ 07:09:10 DEBUG { 07:09:10 DEBUG "metric": { 07:09:10 DEBUG "instance": "10.142.0.107:9100" 07:09:10 DEBUG }, 07:09:10 DEBUG "value": [ 07:09:10 DEBUG 07:09:10 DEBUG 07:09:10 DEBUG 07:09:10 DEBUG [...] 07:09:10 DEBUG 07:09:10 DEBUG 07:09:10 DEBUG 07:09:10 DEBUG "instance": "10.142.0.43:9100" 07:09:10 DEBUG }, 07:09:10 DEBUG "value": [ 07:09:10 DEBUG 1713593350, 07:09:10 DEBUG "9241808.666666668" 07:09:10 DEBUG ] 07:09:10 DEBUG } 07:09:10 DEBUG ] 07:09:10 DEBUG } 07:09:10 DEBUG } 07:09:10 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713593350" 07:09:10 INFO [http_cmd]: http status code OK 07:09:10 DEBUG --- status code --- 07:09:10 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:09:10 DEBUG --- http response --- 07:09:10 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:09:10 DEBUG ----- output ----- 07:09:10 DEBUG { 07:09:10 DEBUG "status": "success", 07:09:10 DEBUG "data": { 07:09:10 DEBUG "resultType": "vector", 07:09:10 DEBUG "result": [ 07:09:10 DEBUG { 07:09:10 DEBUG "metric": { 07:09:10 DEBUG "instance": "10.142.0.107:9100" 07:09:10 DEBUG }, 07:09:10 DEBUG "value": [ 07:09:10 DEBUG 07:09:10 DEBUG 07:09:10 DEBUG 07:09:10 DEBUG [...] 07:09:10 DEBUG 07:09:10 DEBUG 07:09:10 DEBUG 07:09:10 DEBUG "instance": "10.142.0.43:9100" 07:09:10 DEBUG }, 07:09:10 DEBUG "value": [ 07:09:10 DEBUG 1713593350, 07:09:10 DEBUG "0.01000000000000417" 07:09:10 DEBUG ] 07:09:10 DEBUG } 07:09:10 DEBUG ] 07:09:10 DEBUG } 07:09:10 DEBUG } 07:09:11 INFO 07:09:11 INFO [loop_until]: kubectl --namespace=xlou top pods 07:09:11 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 07:09:11 INFO 07:09:11 INFO [loop_until]: kubectl --namespace=xlou top node 07:09:11 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 07:09:12 INFO [loop_until]: OK (rc = 0) 07:09:12 DEBUG --- stdout --- 07:09:12 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 1721m 8656Mi am-767f6b67b5-qcqz2 1587m 8710Mi am-767f6b67b5-vbltz 2416m 8648Mi ds-cts-0 5097m 10473Mi ds-cts-1 4639m 10642Mi ds-cts-2 5897m 10253Mi ds-idrepo-0 439m 3785Mi ds-idrepo-1 496m 3807Mi ds-idrepo-2 431m 3714Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1177Mi idm-b85f9d978-wnhxb 4m 1236Mi lodemon-7cb5b9456d-dskfn 3m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 2113m 16Mi 07:09:12 DEBUG --- stderr --- 07:09:12 DEBUG 07:09:12 INFO [loop_until]: OK (rc = 0) 07:09:12 DEBUG --- stdout --- 07:09:12 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 68m 0% 6275Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 499m 1% 7461Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 1637m 5% 11397Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 85m 0% 3371Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 4582m 15% 13621Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 156m 0% 5194Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 1704m 5% 11626Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 77m 0% 3547Mi 3% gke-xlou-cdm-ds-0f601751-hmm7 2024m 6% 10891Mi 9% gke-xlou-cdm-ds-0f601751-j488 554m 1% 7034Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 4426m 14% 15600Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 532m 1% 7705Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 4868m 16% 12851Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 87m 0% 4000Mi 6% 07:09:12 DEBUG --- stderr --- 07:09:12 DEBUG 07:10:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593409" 07:10:09 INFO [http_cmd]: http status code OK 07:10:09 DEBUG --- status code --- 07:10:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:10:09 DEBUG --- http response --- 07:10:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:10:09 DEBUG ----- output ----- 07:10:09 DEBUG { 07:10:09 DEBUG "status": "success", 07:10:09 DEBUG "data": { 07:10:09 DEBUG "resultType": "vector", 07:10:09 DEBUG "result": [ 07:10:09 DEBUG { 07:10:09 DEBUG "metric": { 07:10:09 DEBUG "pod": "ds-idrepo-0" 07:10:09 DEBUG }, 07:10:09 DEBUG "value": [ 07:10:09 DEBUG 07:10:09 DEBUG 07:10:09 DEBUG 07:10:09 DEBUG [...] 07:10:09 DEBUG 07:10:09 DEBUG 07:10:09 DEBUG 07:10:09 DEBUG "pod": "ds-idrepo-2" 07:10:09 DEBUG }, 07:10:09 DEBUG "value": [ 07:10:09 DEBUG 1713593409, 07:10:09 DEBUG "0.13333333333333333" 07:10:09 DEBUG ] 07:10:09 DEBUG } 07:10:09 DEBUG ] 07:10:09 DEBUG } 07:10:09 DEBUG } 07:10:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713593409" 07:10:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593409" 07:10:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593409" 07:10:09 ERROR [http_cmd]: ERROR 07:10:09 ERROR --- status code --- 07:10:09 ERROR http status code is 400 (expected [200, 401, 403, 404]) 07:10:09 ERROR --- http response --- 07:10:09 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 07:10:09 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 07:10:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593409" 07:10:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713593409" 07:10:09 INFO [http_cmd]: http status code OK 07:10:09 DEBUG --- status code --- 07:10:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:10:09 DEBUG --- http response --- 07:10:09 INFO [http_cmd]: http status code OK 07:10:09 INFO [http_cmd]: http status code OK 07:10:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:10:09 DEBUG --- status code --- 07:10:09 DEBUG --- status code --- 07:10:09 DEBUG ----- output ----- 07:10:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:10:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:10:09 DEBUG { 07:10:09 DEBUG --- http response --- 07:10:09 DEBUG --- http response --- 07:10:09 DEBUG "status": "success", 07:10:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (107 lines): 07:10:09 DEBUG "data": { 07:10:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:10:09 DEBUG ----- output ----- 07:10:09 DEBUG "resultType": "vector", 07:10:09 DEBUG ----- output ----- 07:10:09 DEBUG { 07:10:09 DEBUG "result": [ 07:10:09 DEBUG { 07:10:09 DEBUG "status": "success", 07:10:09 DEBUG { 07:10:09 DEBUG "status": "success", 07:10:09 DEBUG "data": { 07:10:09 DEBUG "metric": { 07:10:09 DEBUG "data": { 07:10:09 DEBUG "resultType": "vector", 07:10:09 DEBUG "pod": "am-767f6b67b5-9zw22" 07:10:09 DEBUG "resultType": "vector", 07:10:09 DEBUG "result": [ 07:10:09 DEBUG }, 07:10:09 DEBUG "result": [ 07:10:09 DEBUG { 07:10:09 DEBUG "value": [ 07:10:09 DEBUG { 07:10:09 DEBUG "metric": { 07:10:09 DEBUG 07:10:09 DEBUG "metric": { 07:10:09 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 07:10:09 DEBUG 07:10:09 DEBUG "pod": "ds-idrepo-0" 07:10:09 DEBUG }, 07:10:09 DEBUG 07:10:09 DEBUG }, 07:10:09 DEBUG "value": [ 07:10:09 DEBUG [...] 07:10:09 DEBUG "value": [ 07:10:09 DEBUG 07:10:09 DEBUG 07:10:09 DEBUG 07:10:09 DEBUG 07:10:09 DEBUG 07:10:09 DEBUG 07:10:09 DEBUG 07:10:09 DEBUG 07:10:09 DEBUG 07:10:09 DEBUG [...] 07:10:09 DEBUG "pod": "am-767f6b67b5-vbltz" 07:10:09 DEBUG [...] 07:10:09 DEBUG 07:10:09 DEBUG }, 07:10:09 DEBUG 07:10:09 DEBUG 07:10:09 DEBUG "value": [ 07:10:09 DEBUG 07:10:09 DEBUG 07:10:09 DEBUG 1713593409, 07:10:09 DEBUG 07:10:09 DEBUG "pod": "idm-b85f9d978-wnhxb" 07:10:09 DEBUG "131598.5" 07:10:09 DEBUG "pod": "ds-idrepo-2" 07:10:09 DEBUG }, 07:10:09 DEBUG ] 07:10:09 DEBUG }, 07:10:09 DEBUG "value": [ 07:10:09 DEBUG } 07:10:09 DEBUG "value": [ 07:10:09 DEBUG 1713593409, 07:10:09 DEBUG ] 07:10:09 DEBUG 1713593409, 07:10:09 DEBUG "447.12496247577985" 07:10:09 DEBUG } 07:10:09 DEBUG "0.13333333333333333" 07:10:09 DEBUG ] 07:10:09 DEBUG } 07:10:09 DEBUG ] 07:10:09 DEBUG } 07:10:09 DEBUG } 07:10:09 DEBUG ] 07:10:09 DEBUG ] 07:10:09 DEBUG } 07:10:09 DEBUG } 07:10:09 DEBUG } 07:10:09 DEBUG } 07:10:09 INFO [http_cmd]: http status code OK 07:10:09 DEBUG --- status code --- 07:10:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:10:09 DEBUG --- http response --- 07:10:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:10:09 DEBUG ----- output ----- 07:10:09 DEBUG { 07:10:09 DEBUG "status": "success", 07:10:09 DEBUG "data": { 07:10:09 DEBUG "resultType": "vector", 07:10:09 DEBUG "result": [ 07:10:09 DEBUG { 07:10:09 DEBUG "metric": { 07:10:09 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 07:10:09 DEBUG }, 07:10:09 DEBUG "value": [ 07:10:09 DEBUG 07:10:09 DEBUG 07:10:09 DEBUG 07:10:09 DEBUG [...] 07:10:09 DEBUG 07:10:09 DEBUG 07:10:09 DEBUG 07:10:09 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 07:10:09 DEBUG }, 07:10:09 DEBUG "value": [ 07:10:09 DEBUG 1713593409, 07:10:09 DEBUG "115900416" 07:10:09 DEBUG ] 07:10:09 DEBUG } 07:10:09 DEBUG ] 07:10:09 DEBUG } 07:10:09 DEBUG } 07:10:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593409" 07:10:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593409" 07:10:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713593409" 07:10:09 INFO [http_cmd]: http status code OK 07:10:09 INFO [http_cmd]: http status code OK 07:10:09 DEBUG --- status code --- 07:10:09 DEBUG --- status code --- 07:10:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:10:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:10:09 DEBUG --- http response --- 07:10:09 DEBUG --- http response --- 07:10:09 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 07:10:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 07:10:09 DEBUG ----- output ----- 07:10:09 DEBUG { 07:10:09 DEBUG "status": "success", 07:10:09 DEBUG "data": { 07:10:09 DEBUG "resultType": "vector", 07:10:09 DEBUG "result": [ 07:10:09 DEBUG { 07:10:09 DEBUG "metric": { 07:10:09 DEBUG "pod": "ds-cts-0" 07:10:09 DEBUG }, 07:10:09 DEBUG "value": [ 07:10:09 DEBUG 07:10:09 DEBUG 07:10:09 DEBUG 07:10:09 DEBUG [...] 07:10:09 DEBUG 07:10:09 DEBUG 07:10:09 DEBUG 07:10:09 DEBUG "pod": "ds-cts-2" 07:10:09 DEBUG }, 07:10:09 DEBUG "value": [ 07:10:09 DEBUG 1713593409, 07:10:09 DEBUG "29.966666666666665" 07:10:09 DEBUG ] 07:10:09 DEBUG } 07:10:09 DEBUG ] 07:10:09 DEBUG } 07:10:09 DEBUG } 07:10:09 INFO [http_cmd]: http status code OK 07:10:09 DEBUG --- status code --- 07:10:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:10:09 DEBUG --- http response --- 07:10:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593409" 07:10:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:10:09 DEBUG ----- output ----- 07:10:09 DEBUG { 07:10:09 DEBUG "status": "success", 07:10:09 DEBUG "data": { 07:10:09 DEBUG "resultType": "vector", 07:10:09 DEBUG "result": [ 07:10:09 DEBUG { 07:10:09 DEBUG "metric": { 07:10:09 DEBUG "instance": "10.142.0.107:9100" 07:10:09 DEBUG }, 07:10:09 DEBUG "value": [ 07:10:09 DEBUG 07:10:09 DEBUG 07:10:09 DEBUG 07:10:09 DEBUG [...] 07:10:09 DEBUG 07:10:09 DEBUG 07:10:09 DEBUG 07:10:09 DEBUG "instance": "10.142.0.43:9100" 07:10:09 DEBUG }, 07:10:09 DEBUG "value": [ 07:10:09 DEBUG 1713593409, 07:10:09 DEBUG "3294671395840" 07:10:09 DEBUG ] 07:10:09 DEBUG } 07:10:09 DEBUG ] 07:10:09 DEBUG } 07:10:09 DEBUG } 07:10:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593409" 07:10:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713593409" 07:10:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713593409" 07:10:09 INFO [http_cmd]: http status code OK 07:10:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593409" 07:10:09 DEBUG --- status code --- 07:10:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:10:09 DEBUG --- http response --- 07:10:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (107 lines): 07:10:09 DEBUG ----- output ----- 07:10:09 DEBUG { 07:10:09 DEBUG "status": "success", 07:10:09 DEBUG "data": { 07:10:09 DEBUG "resultType": "vector", 07:10:09 DEBUG "result": [ 07:10:09 DEBUG { 07:10:09 DEBUG "metric": { 07:10:09 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 07:10:09 DEBUG }, 07:10:09 DEBUG "value": [ 07:10:09 DEBUG 07:10:09 DEBUG 07:10:09 DEBUG 07:10:09 DEBUG [...] 07:10:09 DEBUG 07:10:09 DEBUG 07:10:09 DEBUG 07:10:09 DEBUG "pod": "idm-b85f9d978-wnhxb" 07:10:09 DEBUG }, 07:10:09 DEBUG "value": [ 07:10:09 DEBUG 1713593409, 07:10:09 DEBUG "0" 07:10:09 DEBUG ] 07:10:09 DEBUG } 07:10:09 DEBUG ] 07:10:09 DEBUG } 07:10:09 DEBUG } 07:10:09 INFO [http_cmd]: http status code OK 07:10:09 DEBUG --- status code --- 07:10:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:10:09 DEBUG --- http response --- 07:10:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:10:09 DEBUG ----- output ----- 07:10:09 DEBUG { 07:10:09 DEBUG "status": "success", 07:10:09 DEBUG "data": { 07:10:09 DEBUG "resultType": "vector", 07:10:09 DEBUG "result": [ 07:10:09 DEBUG { 07:10:09 DEBUG "metric": { 07:10:09 DEBUG "pod": "ds-idrepo-0" 07:10:09 DEBUG }, 07:10:09 DEBUG "value": [ 07:10:09 DEBUG 07:10:09 DEBUG 07:10:09 DEBUG 07:10:09 DEBUG [...] 07:10:09 DEBUG 07:10:09 DEBUG 07:10:09 DEBUG 07:10:09 DEBUG "pod": "ds-idrepo-2" 07:10:09 DEBUG }, 07:10:09 DEBUG "value": [ 07:10:09 DEBUG 1713593409, 07:10:09 DEBUG "0.13333333333333333" 07:10:09 DEBUG ] 07:10:09 DEBUG } 07:10:09 DEBUG ] 07:10:09 DEBUG } 07:10:09 DEBUG } 07:10:09 INFO [http_cmd]: http status code OK 07:10:09 INFO [http_cmd]: http status code OK 07:10:09 DEBUG --- status code --- 07:10:09 DEBUG --- status code --- 07:10:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:10:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:10:09 DEBUG --- http response --- 07:10:09 DEBUG --- http response --- 07:10:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 07:10:09 DEBUG ----- output ----- 07:10:09 DEBUG { 07:10:09 DEBUG "status": "success", 07:10:09 DEBUG "data": { 07:10:09 DEBUG "resultType": "vector", 07:10:09 DEBUG "result": [ 07:10:09 DEBUG { 07:10:09 DEBUG "metric": { 07:10:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:10:09 DEBUG "pod": "ds-cts-0" 07:10:09 DEBUG ----- output ----- 07:10:09 DEBUG }, 07:10:09 DEBUG { 07:10:09 DEBUG "value": [ 07:10:09 DEBUG "status": "success", 07:10:09 INFO [http_cmd]: http status code OK 07:10:09 DEBUG 07:10:09 DEBUG "data": { 07:10:09 DEBUG --- status code --- 07:10:09 DEBUG 07:10:09 DEBUG "resultType": "vector", 07:10:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:10:09 DEBUG 07:10:09 DEBUG "result": [ 07:10:09 DEBUG --- http response --- 07:10:09 DEBUG [...] 07:10:09 DEBUG { 07:10:09 DEBUG 07:10:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:10:09 DEBUG "metric": { 07:10:09 DEBUG 07:10:09 DEBUG ----- output ----- 07:10:09 DEBUG "instance": "10.142.0.107:9100" 07:10:09 DEBUG 07:10:09 DEBUG { 07:10:09 DEBUG }, 07:10:09 DEBUG "pod": "ds-cts-2" 07:10:09 DEBUG "status": "success", 07:10:09 DEBUG "value": [ 07:10:09 DEBUG }, 07:10:09 DEBUG "data": { 07:10:09 DEBUG 07:10:09 DEBUG "value": [ 07:10:09 DEBUG "resultType": "vector", 07:10:09 DEBUG 07:10:09 DEBUG 1713593409, 07:10:09 DEBUG "result": [ 07:10:09 DEBUG 07:10:09 DEBUG "0" 07:10:09 DEBUG { 07:10:09 DEBUG [...] 07:10:09 DEBUG ] 07:10:09 DEBUG "metric": { 07:10:09 DEBUG 07:10:09 DEBUG } 07:10:09 DEBUG "instance": "10.142.0.107:9100" 07:10:09 DEBUG 07:10:09 DEBUG ] 07:10:09 DEBUG }, 07:10:09 DEBUG 07:10:09 DEBUG } 07:10:09 DEBUG "value": [ 07:10:09 DEBUG "instance": "10.142.0.43:9100" 07:10:09 DEBUG } 07:10:09 DEBUG 07:10:09 DEBUG }, 07:10:09 DEBUG 07:10:09 DEBUG "value": [ 07:10:09 DEBUG 07:10:09 DEBUG 1713593409, 07:10:09 DEBUG [...] 07:10:09 DEBUG "0.01443333333331509" 07:10:09 DEBUG 07:10:09 DEBUG ] 07:10:09 DEBUG 07:10:09 DEBUG } 07:10:09 DEBUG 07:10:09 DEBUG ] 07:10:09 DEBUG "instance": "10.142.0.43:9100" 07:10:09 DEBUG } 07:10:09 DEBUG }, 07:10:09 DEBUG } 07:10:09 DEBUG "value": [ 07:10:09 DEBUG 1713593409, 07:10:09 DEBUG "8863675.566666666" 07:10:09 DEBUG ] 07:10:09 DEBUG } 07:10:09 DEBUG ] 07:10:09 DEBUG } 07:10:09 DEBUG } 07:10:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593409" 07:10:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713593409" 07:10:09 INFO [http_cmd]: http status code OK 07:10:09 DEBUG --- status code --- 07:10:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:10:09 DEBUG --- http response --- 07:10:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:10:09 DEBUG ----- output ----- 07:10:09 DEBUG { 07:10:09 DEBUG "status": "success", 07:10:09 DEBUG "data": { 07:10:09 DEBUG "resultType": "vector", 07:10:09 DEBUG "result": [ 07:10:09 DEBUG { 07:10:09 DEBUG "metric": { 07:10:09 DEBUG "pod": "am-767f6b67b5-9zw22" 07:10:09 DEBUG }, 07:10:09 DEBUG "value": [ 07:10:09 DEBUG 07:10:09 DEBUG 07:10:09 DEBUG 07:10:09 DEBUG [...] 07:10:09 DEBUG 07:10:09 DEBUG 07:10:09 DEBUG 07:10:09 DEBUG "pod": "am-767f6b67b5-vbltz" 07:10:09 DEBUG }, 07:10:09 DEBUG "value": [ 07:10:09 DEBUG 1713593409, 07:10:09 DEBUG "0.03333333333333333" 07:10:09 DEBUG ] 07:10:09 DEBUG } 07:10:09 DEBUG ] 07:10:09 DEBUG } 07:10:09 DEBUG } 07:10:09 INFO [http_cmd]: http status code OK 07:10:09 DEBUG --- status code --- 07:10:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:10:09 DEBUG --- http response --- 07:10:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713593409" 07:10:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 07:10:09 DEBUG ----- output ----- 07:10:09 DEBUG { 07:10:09 DEBUG "status": "success", 07:10:09 DEBUG "data": { 07:10:09 DEBUG "resultType": "vector", 07:10:09 DEBUG "result": [ 07:10:09 DEBUG { 07:10:09 DEBUG "metric": { 07:10:09 DEBUG "pod": "ds-idrepo-0" 07:10:09 DEBUG }, 07:10:09 DEBUG "value": [ 07:10:09 DEBUG 07:10:09 DEBUG 07:10:09 DEBUG 07:10:09 DEBUG [...] 07:10:09 DEBUG 07:10:09 DEBUG 07:10:09 DEBUG 07:10:09 DEBUG "pod": "ds-cts-2" 07:10:09 DEBUG }, 07:10:09 DEBUG "value": [ 07:10:09 DEBUG 1713593409, 07:10:09 DEBUG "0" 07:10:09 DEBUG ] 07:10:09 DEBUG } 07:10:09 DEBUG ] 07:10:09 DEBUG } 07:10:09 DEBUG } 07:10:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593409" 07:10:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593409" 07:10:09 INFO [http_cmd]: http status code OK 07:10:09 INFO [http_cmd]: http status code OK 07:10:09 DEBUG --- status code --- 07:10:09 DEBUG --- status code --- 07:10:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:10:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:10:09 DEBUG --- http response --- 07:10:09 DEBUG --- http response --- 07:10:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:10:09 DEBUG ----- output ----- 07:10:09 DEBUG { 07:10:09 DEBUG "status": "success", 07:10:09 DEBUG "data": { 07:10:09 DEBUG "resultType": "vector", 07:10:09 DEBUG "result": [ 07:10:09 DEBUG { 07:10:09 DEBUG "metric": { 07:10:09 DEBUG "pod": "am-767f6b67b5-9zw22" 07:10:09 DEBUG }, 07:10:09 DEBUG "value": [ 07:10:09 DEBUG 07:10:09 DEBUG 07:10:09 DEBUG 07:10:09 DEBUG [...] 07:10:09 DEBUG 07:10:09 DEBUG 07:10:09 DEBUG 07:10:09 DEBUG "pod": "am-767f6b67b5-vbltz" 07:10:09 DEBUG }, 07:10:09 DEBUG "value": [ 07:10:09 DEBUG 1713593409, 07:10:09 DEBUG "0.097" 07:10:09 DEBUG ] 07:10:09 DEBUG } 07:10:09 DEBUG ] 07:10:09 DEBUG } 07:10:09 DEBUG } 07:10:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 07:10:09 DEBUG ----- output ----- 07:10:09 DEBUG { 07:10:09 DEBUG "status": "success", 07:10:09 DEBUG "data": { 07:10:09 DEBUG "resultType": "vector", 07:10:09 DEBUG "result": [ 07:10:09 DEBUG { 07:10:09 DEBUG "metric": { 07:10:09 DEBUG "pod": "ds-cts-0" 07:10:09 DEBUG }, 07:10:09 DEBUG "value": [ 07:10:09 DEBUG 07:10:09 DEBUG 07:10:09 DEBUG 07:10:09 DEBUG [...] 07:10:09 DEBUG 07:10:09 DEBUG 07:10:09 DEBUG 07:10:09 DEBUG "pod": "ds-cts-2" 07:10:09 DEBUG }, 07:10:09 DEBUG "value": [ 07:10:09 DEBUG 1713593409, 07:10:09 DEBUG "0.000025437777777777778" 07:10:09 DEBUG ] 07:10:09 DEBUG } 07:10:09 DEBUG ] 07:10:09 DEBUG } 07:10:09 DEBUG } 07:10:09 INFO [http_cmd]: http status code OK 07:10:09 DEBUG --- status code --- 07:10:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:10:09 DEBUG --- http response --- 07:10:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:10:09 DEBUG ----- output ----- 07:10:09 DEBUG { 07:10:09 DEBUG "status": "success", 07:10:09 DEBUG "data": { 07:10:09 DEBUG "resultType": "vector", 07:10:09 DEBUG "result": [ 07:10:09 DEBUG { 07:10:09 DEBUG "metric": { 07:10:09 DEBUG "pod": "am-767f6b67b5-9zw22" 07:10:09 DEBUG }, 07:10:09 DEBUG "value": [ 07:10:09 DEBUG 07:10:09 DEBUG 07:10:09 DEBUG 07:10:09 DEBUG [...] 07:10:09 DEBUG 07:10:09 DEBUG 07:10:09 DEBUG 07:10:09 DEBUG "pod": "am-767f6b67b5-vbltz" 07:10:09 DEBUG }, 07:10:09 DEBUG "value": [ 07:10:09 DEBUG 1713593409, 07:10:09 DEBUG "1303.6666666666667" 07:10:09 DEBUG ] 07:10:09 DEBUG } 07:10:09 DEBUG ] 07:10:09 DEBUG } 07:10:09 DEBUG } 07:10:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593409" 07:10:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713593409" 07:10:09 INFO [http_cmd]: http status code OK 07:10:09 DEBUG --- status code --- 07:10:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:10:09 DEBUG --- http response --- 07:10:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:10:09 DEBUG ----- output ----- 07:10:09 DEBUG { 07:10:09 DEBUG "status": "success", 07:10:09 DEBUG "data": { 07:10:09 DEBUG "resultType": "vector", 07:10:09 DEBUG "result": [ 07:10:09 DEBUG { 07:10:09 DEBUG "metric": { 07:10:09 DEBUG "pod": "am-767f6b67b5-9zw22" 07:10:09 DEBUG }, 07:10:09 DEBUG "value": [ 07:10:09 DEBUG 07:10:09 DEBUG 07:10:09 DEBUG 07:10:09 DEBUG [...] 07:10:09 DEBUG 07:10:09 DEBUG 07:10:09 DEBUG 07:10:09 DEBUG "pod": "am-767f6b67b5-vbltz" 07:10:09 DEBUG }, 07:10:09 DEBUG "value": [ 07:10:09 DEBUG 1713593409, 07:10:09 DEBUG "0" 07:10:09 DEBUG ] 07:10:09 DEBUG } 07:10:09 DEBUG ] 07:10:09 DEBUG } 07:10:09 DEBUG } 07:10:09 INFO [http_cmd]: http status code OK 07:10:09 DEBUG --- status code --- 07:10:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:10:09 DEBUG --- http response --- 07:10:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:10:09 DEBUG ----- output ----- 07:10:09 DEBUG { 07:10:09 DEBUG "status": "success", 07:10:09 DEBUG "data": { 07:10:09 DEBUG "resultType": "vector", 07:10:09 DEBUG "result": [ 07:10:09 DEBUG { 07:10:09 DEBUG "metric": { 07:10:09 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 07:10:09 DEBUG }, 07:10:09 DEBUG "value": [ 07:10:09 DEBUG 07:10:09 DEBUG 07:10:09 DEBUG 07:10:09 DEBUG [...] 07:10:09 DEBUG 07:10:09 DEBUG 07:10:09 DEBUG 07:10:09 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 07:10:09 DEBUG }, 07:10:09 DEBUG "value": [ 07:10:09 DEBUG 1713593409, 07:10:09 DEBUG "0.03634268889813116" 07:10:09 DEBUG ] 07:10:09 DEBUG } 07:10:09 DEBUG ] 07:10:09 DEBUG } 07:10:09 DEBUG } 07:10:10 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593410" 07:10:10 INFO [http_cmd]: http status code OK 07:10:10 DEBUG --- status code --- 07:10:10 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:10:10 DEBUG --- http response --- 07:10:10 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593410" 07:10:10 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 07:10:10 DEBUG ----- output ----- 07:10:10 DEBUG { 07:10:10 DEBUG "status": "success", 07:10:10 DEBUG "data": { 07:10:10 DEBUG "resultType": "vector", 07:10:10 DEBUG "result": [ 07:10:10 DEBUG { 07:10:10 DEBUG "metric": { 07:10:10 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 07:10:10 DEBUG }, 07:10:10 DEBUG "value": [ 07:10:10 DEBUG 07:10:10 DEBUG 07:10:10 DEBUG 07:10:10 DEBUG [...] 07:10:10 DEBUG 07:10:10 DEBUG 07:10:10 DEBUG 07:10:10 DEBUG "pod": "ds-idrepo-2" 07:10:10 DEBUG }, 07:10:10 DEBUG "value": [ 07:10:10 DEBUG 1713593410, 07:10:10 DEBUG "0" 07:10:10 DEBUG ] 07:10:10 DEBUG } 07:10:10 DEBUG ] 07:10:10 DEBUG } 07:10:10 DEBUG } 07:10:10 INFO [http_cmd]: http status code OK 07:10:10 DEBUG --- status code --- 07:10:10 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:10:10 DEBUG --- http response --- 07:10:10 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:10:10 DEBUG ----- output ----- 07:10:10 DEBUG { 07:10:10 DEBUG "status": "success", 07:10:10 DEBUG "data": { 07:10:10 DEBUG "resultType": "vector", 07:10:10 DEBUG "result": [ 07:10:10 DEBUG { 07:10:10 DEBUG "metric": { 07:10:10 DEBUG "pod": "am-767f6b67b5-9zw22" 07:10:10 DEBUG }, 07:10:10 DEBUG "value": [ 07:10:10 DEBUG 07:10:10 DEBUG 07:10:10 DEBUG 07:10:10 DEBUG [...] 07:10:10 DEBUG 07:10:10 DEBUG 07:10:10 DEBUG 07:10:10 DEBUG "pod": "am-767f6b67b5-vbltz" 07:10:10 DEBUG }, 07:10:10 DEBUG "value": [ 07:10:10 DEBUG 1713593410, 07:10:10 DEBUG "0.1" 07:10:10 DEBUG ] 07:10:10 DEBUG } 07:10:10 DEBUG ] 07:10:10 DEBUG } 07:10:10 DEBUG } 07:10:10 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593410" 07:10:10 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593410" 07:10:10 INFO [http_cmd]: http status code OK 07:10:10 DEBUG --- status code --- 07:10:10 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:10:10 DEBUG --- http response --- 07:10:10 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:10:10 DEBUG ----- output ----- 07:10:10 DEBUG { 07:10:10 DEBUG "status": "success", 07:10:10 DEBUG "data": { 07:10:10 DEBUG "resultType": "vector", 07:10:10 DEBUG "result": [ 07:10:10 DEBUG { 07:10:10 DEBUG "metric": { 07:10:10 DEBUG "pod": "am-767f6b67b5-9zw22" 07:10:10 DEBUG }, 07:10:10 DEBUG "value": [ 07:10:10 DEBUG 07:10:10 DEBUG 07:10:10 DEBUG 07:10:10 DEBUG [...] 07:10:10 DEBUG 07:10:10 DEBUG 07:10:10 DEBUG 07:10:10 DEBUG "pod": "am-767f6b67b5-vbltz" 07:10:10 DEBUG }, 07:10:10 DEBUG "value": [ 07:10:10 DEBUG 1713593410, 07:10:10 DEBUG "0" 07:10:10 DEBUG ] 07:10:10 DEBUG } 07:10:10 DEBUG ] 07:10:10 DEBUG } 07:10:10 DEBUG } 07:10:10 INFO [http_cmd]: http status code OK 07:10:10 DEBUG --- status code --- 07:10:10 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:10:10 DEBUG --- http response --- 07:10:10 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 07:10:10 DEBUG ----- output ----- 07:10:10 DEBUG { 07:10:10 DEBUG "status": "success", 07:10:10 DEBUG "data": { 07:10:10 DEBUG "resultType": "vector", 07:10:10 DEBUG "result": [ 07:10:10 DEBUG { 07:10:10 DEBUG "metric": { 07:10:10 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 07:10:10 DEBUG }, 07:10:10 DEBUG "value": [ 07:10:10 DEBUG 07:10:10 DEBUG 07:10:10 DEBUG 07:10:10 DEBUG [...] 07:10:10 DEBUG 07:10:10 DEBUG 07:10:10 DEBUG 07:10:10 DEBUG "pod": "ds-idrepo-2" 07:10:10 DEBUG }, 07:10:10 DEBUG "value": [ 07:10:10 DEBUG 1713593410, 07:10:10 DEBUG "0" 07:10:10 DEBUG ] 07:10:10 DEBUG } 07:10:10 DEBUG ] 07:10:10 DEBUG } 07:10:10 DEBUG } 07:10:10 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713593410" 07:10:10 INFO [http_cmd]: http status code OK 07:10:10 DEBUG --- status code --- 07:10:10 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:10:10 DEBUG --- http response --- 07:10:10 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:10:10 DEBUG ----- output ----- 07:10:10 DEBUG { 07:10:10 DEBUG "status": "success", 07:10:10 DEBUG "data": { 07:10:10 DEBUG "resultType": "vector", 07:10:10 DEBUG "result": [ 07:10:10 DEBUG { 07:10:10 DEBUG "metric": { 07:10:10 DEBUG "instance": "10.142.0.107:9100" 07:10:10 DEBUG }, 07:10:10 DEBUG "value": [ 07:10:10 DEBUG 07:10:10 DEBUG 07:10:10 DEBUG 07:10:10 DEBUG [...] 07:10:10 DEBUG 07:10:10 DEBUG 07:10:10 DEBUG 07:10:10 DEBUG "instance": "10.142.0.43:9100" 07:10:10 DEBUG }, 07:10:10 DEBUG "value": [ 07:10:10 DEBUG 1713593410, 07:10:10 DEBUG "8863571.066666666" 07:10:10 DEBUG ] 07:10:10 DEBUG } 07:10:10 DEBUG ] 07:10:10 DEBUG } 07:10:10 DEBUG } 07:10:10 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713593410" 07:10:10 INFO [http_cmd]: http status code OK 07:10:10 DEBUG --- status code --- 07:10:10 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:10:10 DEBUG --- http response --- 07:10:10 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:10:10 DEBUG ----- output ----- 07:10:10 DEBUG { 07:10:10 DEBUG "status": "success", 07:10:10 DEBUG "data": { 07:10:10 DEBUG "resultType": "vector", 07:10:10 DEBUG "result": [ 07:10:10 DEBUG { 07:10:10 DEBUG "metric": { 07:10:10 DEBUG "instance": "10.142.0.107:9100" 07:10:10 DEBUG }, 07:10:10 DEBUG "value": [ 07:10:10 DEBUG 07:10:10 DEBUG 07:10:10 DEBUG 07:10:10 DEBUG [...] 07:10:10 DEBUG 07:10:10 DEBUG 07:10:10 DEBUG 07:10:10 DEBUG "instance": "10.142.0.43:9100" 07:10:10 DEBUG }, 07:10:10 DEBUG "value": [ 07:10:10 DEBUG 1713593410, 07:10:10 DEBUG "0.010666666666666439" 07:10:10 DEBUG ] 07:10:10 DEBUG } 07:10:10 DEBUG ] 07:10:10 DEBUG } 07:10:10 DEBUG } 07:10:12 INFO 07:10:12 INFO [loop_until]: kubectl --namespace=xlou top pods 07:10:12 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 07:10:12 INFO 07:10:12 INFO [loop_until]: kubectl --namespace=xlou top node 07:10:12 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 07:10:12 INFO [loop_until]: OK (rc = 0) 07:10:12 DEBUG --- stdout --- 07:10:12 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 1282m 8610Mi am-767f6b67b5-qcqz2 1305m 8667Mi am-767f6b67b5-vbltz 1418m 8613Mi ds-cts-0 3735m 10563Mi ds-cts-1 3182m 10649Mi ds-cts-2 3700m 10226Mi ds-idrepo-0 1060m 3914Mi ds-idrepo-1 924m 3897Mi ds-idrepo-2 1008m 3728Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 5m 1178Mi idm-b85f9d978-wnhxb 4m 1236Mi lodemon-7cb5b9456d-dskfn 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 1259m 312Mi 07:10:12 DEBUG --- stderr --- 07:10:12 DEBUG 07:10:12 INFO [loop_until]: OK (rc = 0) 07:10:12 DEBUG --- stdout --- 07:10:12 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 73m 0% 6279Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 1098m 3% 7437Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 1264m 4% 11356Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 82m 0% 3397Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 3788m 12% 13714Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 136m 0% 5195Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 1361m 4% 11576Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 77m 0% 3420Mi 3% gke-xlou-cdm-ds-0f601751-hmm7 1535m 5% 10848Mi 9% gke-xlou-cdm-ds-0f601751-j488 1019m 3% 7018Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 3404m 11% 15590Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 901m 3% 7659Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 3874m 12% 12857Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 1323m 8% 4343Mi 7% 07:10:12 DEBUG --- stderr --- 07:10:12 DEBUG 07:11:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593469" 07:11:09 INFO [http_cmd]: http status code OK 07:11:09 DEBUG --- status code --- 07:11:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:11:09 DEBUG --- http response --- 07:11:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:11:09 DEBUG ----- output ----- 07:11:09 DEBUG { 07:11:09 DEBUG "status": "success", 07:11:09 DEBUG "data": { 07:11:09 DEBUG "resultType": "vector", 07:11:09 DEBUG "result": [ 07:11:09 DEBUG { 07:11:09 DEBUG "metric": { 07:11:09 DEBUG "pod": "ds-idrepo-0" 07:11:09 DEBUG }, 07:11:09 DEBUG "value": [ 07:11:09 DEBUG 07:11:09 DEBUG 07:11:09 DEBUG 07:11:09 DEBUG [...] 07:11:09 DEBUG 07:11:09 DEBUG 07:11:09 DEBUG 07:11:09 DEBUG "pod": "ds-idrepo-2" 07:11:09 DEBUG }, 07:11:09 DEBUG "value": [ 07:11:09 DEBUG 1713593469, 07:11:09 DEBUG "0" 07:11:09 DEBUG ] 07:11:09 DEBUG } 07:11:09 DEBUG ] 07:11:09 DEBUG } 07:11:09 DEBUG } 07:11:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713593469" 07:11:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593469" 07:11:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593469" 07:11:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593469" 07:11:09 INFO [http_cmd]: http status code OK 07:11:09 DEBUG --- status code --- 07:11:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:11:09 DEBUG --- http response --- 07:11:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:11:09 DEBUG ----- output ----- 07:11:09 DEBUG { 07:11:09 DEBUG "status": "success", 07:11:09 DEBUG "data": { 07:11:09 DEBUG "resultType": "vector", 07:11:09 DEBUG "result": [ 07:11:09 DEBUG { 07:11:09 DEBUG "metric": { 07:11:09 DEBUG "pod": "am-767f6b67b5-9zw22" 07:11:09 DEBUG }, 07:11:09 DEBUG "value": [ 07:11:09 DEBUG 07:11:09 DEBUG 07:11:09 DEBUG 07:11:09 DEBUG [...] 07:11:09 DEBUG 07:11:09 DEBUG 07:11:09 DEBUG 07:11:09 DEBUG "pod": "am-767f6b67b5-vbltz" 07:11:09 DEBUG }, 07:11:09 DEBUG "value": [ 07:11:09 DEBUG 1713593469, 07:11:09 DEBUG "52745" 07:11:09 DEBUG ] 07:11:09 DEBUG } 07:11:09 DEBUG ] 07:11:09 DEBUG } 07:11:09 DEBUG } 07:11:09 INFO [http_cmd]: http status code OK 07:11:09 DEBUG --- status code --- 07:11:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:11:09 DEBUG --- http response --- 07:11:09 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 07:11:09 ERROR [http_cmd]: ERROR 07:11:09 ERROR --- status code --- 07:11:09 ERROR http status code is 400 (expected [200, 401, 403, 404]) 07:11:09 ERROR --- http response --- 07:11:09 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 07:11:09 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 07:11:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593469" 07:11:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713593469" 07:11:09 INFO [http_cmd]: http status code OK 07:11:09 DEBUG --- status code --- 07:11:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:11:09 DEBUG --- http response --- 07:11:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:11:09 DEBUG ----- output ----- 07:11:09 DEBUG { 07:11:09 DEBUG "status": "success", 07:11:09 DEBUG "data": { 07:11:09 DEBUG "resultType": "vector", 07:11:09 DEBUG "result": [ 07:11:09 DEBUG { 07:11:09 DEBUG "metric": { 07:11:09 DEBUG "pod": "ds-idrepo-0" 07:11:09 DEBUG }, 07:11:09 DEBUG "value": [ 07:11:09 DEBUG 07:11:09 DEBUG 07:11:09 DEBUG 07:11:09 DEBUG [...] 07:11:09 DEBUG 07:11:09 DEBUG 07:11:09 DEBUG 07:11:09 DEBUG "pod": "ds-idrepo-2" 07:11:09 DEBUG }, 07:11:09 DEBUG "value": [ 07:11:09 DEBUG 1713593469, 07:11:09 DEBUG "0" 07:11:09 DEBUG ] 07:11:09 DEBUG } 07:11:09 DEBUG ] 07:11:09 DEBUG } 07:11:09 DEBUG } 07:11:09 INFO [http_cmd]: http status code OK 07:11:09 DEBUG --- status code --- 07:11:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:11:09 DEBUG --- http response --- 07:11:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 07:11:09 INFO [http_cmd]: http status code OK 07:11:09 DEBUG ----- output ----- 07:11:09 DEBUG --- status code --- 07:11:09 DEBUG { 07:11:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:11:09 DEBUG "status": "success", 07:11:09 DEBUG --- http response --- 07:11:09 DEBUG "data": { 07:11:09 DEBUG "resultType": "vector", 07:11:09 DEBUG "result": [ 07:11:09 DEBUG { 07:11:09 DEBUG "metric": { 07:11:09 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 07:11:09 DEBUG }, 07:11:09 DEBUG "value": [ 07:11:09 DEBUG 07:11:09 DEBUG 07:11:09 DEBUG 07:11:09 DEBUG [...] 07:11:09 DEBUG 07:11:09 DEBUG 07:11:09 DEBUG 07:11:09 DEBUG "pod": "idm-b85f9d978-wnhxb" 07:11:09 DEBUG }, 07:11:09 DEBUG "value": [ 07:11:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:11:09 DEBUG 1713593469, 07:11:09 DEBUG ----- output ----- 07:11:09 DEBUG "422.2141085386" 07:11:09 DEBUG { 07:11:09 DEBUG ] 07:11:09 DEBUG "status": "success", 07:11:09 DEBUG } 07:11:09 DEBUG "data": { 07:11:09 DEBUG ] 07:11:09 DEBUG "resultType": "vector", 07:11:09 DEBUG } 07:11:09 DEBUG "result": [ 07:11:09 DEBUG } 07:11:09 DEBUG { 07:11:09 DEBUG "metric": { 07:11:09 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 07:11:09 DEBUG }, 07:11:09 DEBUG "value": [ 07:11:09 DEBUG 07:11:09 DEBUG 07:11:09 DEBUG 07:11:09 DEBUG [...] 07:11:09 DEBUG 07:11:09 DEBUG 07:11:09 DEBUG 07:11:09 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 07:11:09 DEBUG }, 07:11:09 DEBUG "value": [ 07:11:09 DEBUG 1713593469, 07:11:09 DEBUG "1356525568" 07:11:09 DEBUG ] 07:11:09 DEBUG } 07:11:09 DEBUG ] 07:11:09 DEBUG } 07:11:09 DEBUG } 07:11:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593469" 07:11:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593469" 07:11:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713593469" 07:11:09 INFO [http_cmd]: http status code OK 07:11:09 DEBUG --- status code --- 07:11:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:11:09 DEBUG --- http response --- 07:11:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 07:11:09 DEBUG ----- output ----- 07:11:09 DEBUG { 07:11:09 DEBUG "status": "success", 07:11:09 DEBUG "data": { 07:11:09 DEBUG "resultType": "vector", 07:11:09 DEBUG "result": [ 07:11:09 DEBUG { 07:11:09 DEBUG "metric": { 07:11:09 DEBUG "pod": "ds-cts-0" 07:11:09 DEBUG }, 07:11:09 DEBUG "value": [ 07:11:09 DEBUG 07:11:09 DEBUG 07:11:09 DEBUG 07:11:09 DEBUG [...] 07:11:09 DEBUG 07:11:09 DEBUG 07:11:09 DEBUG 07:11:09 DEBUG "pod": "ds-cts-2" 07:11:09 DEBUG }, 07:11:09 DEBUG "value": [ 07:11:09 DEBUG 1713593469, 07:11:09 DEBUG "30" 07:11:09 DEBUG ] 07:11:09 DEBUG } 07:11:09 DEBUG ] 07:11:09 DEBUG } 07:11:09 DEBUG } 07:11:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713593469" 07:11:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593469" 07:11:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713593469" 07:11:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593469" 07:11:09 INFO [http_cmd]: http status code OK 07:11:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593469" 07:11:09 DEBUG --- status code --- 07:11:09 INFO [http_cmd]: http status code OK 07:11:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:11:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713593469" 07:11:09 DEBUG --- status code --- 07:11:09 DEBUG --- http response --- 07:11:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:11:09 DEBUG --- http response --- 07:11:09 INFO [http_cmd]: http status code OK 07:11:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 07:11:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:11:09 DEBUG --- status code --- 07:11:09 DEBUG ----- output ----- 07:11:09 DEBUG ----- output ----- 07:11:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:11:09 DEBUG { 07:11:09 DEBUG { 07:11:09 DEBUG --- http response --- 07:11:09 DEBUG "status": "success", 07:11:09 DEBUG "status": "success", 07:11:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:11:09 INFO [http_cmd]: http status code OK 07:11:09 DEBUG "data": { 07:11:09 INFO [http_cmd]: http status code OK 07:11:09 DEBUG "data": { 07:11:09 DEBUG ----- output ----- 07:11:09 DEBUG --- status code --- 07:11:09 DEBUG "resultType": "vector", 07:11:09 DEBUG --- status code --- 07:11:09 DEBUG "resultType": "vector", 07:11:09 DEBUG { 07:11:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:11:09 DEBUG "result": [ 07:11:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:11:09 DEBUG "result": [ 07:11:09 DEBUG "status": "success", 07:11:09 DEBUG --- http response --- 07:11:09 DEBUG { 07:11:09 DEBUG --- http response --- 07:11:09 DEBUG { 07:11:09 DEBUG "data": { 07:11:09 DEBUG "metric": { 07:11:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:11:09 DEBUG "metric": { 07:11:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 07:11:09 DEBUG "resultType": "vector", 07:11:09 DEBUG "pod": "ds-cts-0" 07:11:09 INFO [http_cmd]: http status code OK 07:11:09 DEBUG ----- output ----- 07:11:09 DEBUG "instance": "10.142.0.107:9100" 07:11:09 DEBUG ----- output ----- 07:11:09 DEBUG "result": [ 07:11:09 DEBUG }, 07:11:09 DEBUG --- status code --- 07:11:09 DEBUG { 07:11:09 DEBUG }, 07:11:09 DEBUG { 07:11:09 DEBUG { 07:11:09 DEBUG "value": [ 07:11:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:11:09 DEBUG "status": "success", 07:11:09 INFO [http_cmd]: http status code OK 07:11:09 DEBUG "value": [ 07:11:09 DEBUG "status": "success", 07:11:09 DEBUG "metric": { 07:11:09 DEBUG 07:11:09 DEBUG --- http response --- 07:11:09 DEBUG "data": { 07:11:09 DEBUG --- status code --- 07:11:09 DEBUG 07:11:09 DEBUG "data": { 07:11:09 DEBUG "instance": "10.142.0.107:9100" 07:11:09 DEBUG 07:11:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 07:11:09 DEBUG "resultType": "vector", 07:11:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:11:09 DEBUG 07:11:09 DEBUG "resultType": "vector", 07:11:09 DEBUG }, 07:11:09 DEBUG 07:11:09 DEBUG ----- output ----- 07:11:09 DEBUG "result": [ 07:11:09 DEBUG --- http response --- 07:11:09 DEBUG 07:11:09 DEBUG "result": [ 07:11:09 DEBUG "value": [ 07:11:09 DEBUG [...] 07:11:09 DEBUG { 07:11:09 DEBUG { 07:11:09 DEBUG [...] 07:11:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:11:09 DEBUG { 07:11:09 DEBUG 07:11:09 DEBUG 07:11:09 DEBUG "status": "success", 07:11:09 DEBUG "metric": { 07:11:09 DEBUG 07:11:09 DEBUG ----- output ----- 07:11:09 DEBUG "metric": { 07:11:09 DEBUG 07:11:09 INFO [http_cmd]: http status code OK 07:11:09 DEBUG 07:11:09 DEBUG "data": { 07:11:09 DEBUG "pod": "am-767f6b67b5-9zw22" 07:11:09 DEBUG 07:11:09 DEBUG { 07:11:09 DEBUG "pod": "ds-idrepo-0" 07:11:09 DEBUG 07:11:09 DEBUG --- status code --- 07:11:09 DEBUG 07:11:09 DEBUG "resultType": "vector", 07:11:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593469" 07:11:09 DEBUG }, 07:11:09 DEBUG 07:11:09 DEBUG "status": "success", 07:11:09 DEBUG }, 07:11:09 DEBUG [...] 07:11:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:11:09 DEBUG "pod": "ds-cts-2" 07:11:09 DEBUG "result": [ 07:11:09 DEBUG "value": [ 07:11:09 DEBUG "instance": "10.142.0.43:9100" 07:11:09 DEBUG "data": { 07:11:09 DEBUG "value": [ 07:11:09 DEBUG 07:11:09 DEBUG --- http response --- 07:11:09 DEBUG }, 07:11:09 DEBUG { 07:11:09 DEBUG 07:11:09 DEBUG }, 07:11:09 DEBUG "resultType": "vector", 07:11:09 DEBUG 07:11:09 DEBUG 07:11:09 DEBUG "value": [ 07:11:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:11:09 DEBUG "metric": { 07:11:09 DEBUG 07:11:09 DEBUG "value": [ 07:11:09 DEBUG "result": [ 07:11:09 DEBUG 07:11:09 DEBUG 07:11:09 DEBUG 1713593469, 07:11:09 DEBUG ----- output ----- 07:11:09 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 07:11:09 DEBUG 07:11:09 DEBUG 1713593469, 07:11:09 DEBUG { 07:11:09 DEBUG 07:11:09 DEBUG "instance": "10.142.0.43:9100" 07:11:09 DEBUG "0" 07:11:09 DEBUG { 07:11:09 DEBUG }, 07:11:09 DEBUG [...] 07:11:09 DEBUG "3295442076672" 07:11:09 DEBUG "metric": { 07:11:09 DEBUG [...] 07:11:09 DEBUG }, 07:11:09 DEBUG ] 07:11:09 DEBUG "status": "success", 07:11:09 DEBUG "value": [ 07:11:09 DEBUG 07:11:09 DEBUG ] 07:11:09 DEBUG "pod": "ds-idrepo-0" 07:11:09 DEBUG 07:11:09 DEBUG "value": [ 07:11:09 DEBUG } 07:11:09 DEBUG "data": { 07:11:09 DEBUG 07:11:09 DEBUG 07:11:09 DEBUG } 07:11:09 DEBUG }, 07:11:09 DEBUG 07:11:09 DEBUG 1713593469, 07:11:09 DEBUG ] 07:11:09 DEBUG "resultType": "vector", 07:11:09 DEBUG 07:11:09 DEBUG 07:11:09 DEBUG ] 07:11:09 DEBUG "value": [ 07:11:09 DEBUG 07:11:09 DEBUG "0.015366666666596984" 07:11:09 DEBUG } 07:11:09 DEBUG "result": [ 07:11:09 DEBUG 07:11:09 DEBUG "pod": "am-767f6b67b5-vbltz" 07:11:09 DEBUG } 07:11:09 DEBUG 07:11:09 DEBUG "pod": "ds-cts-2" 07:11:09 DEBUG ] 07:11:09 DEBUG } 07:11:09 DEBUG { 07:11:09 DEBUG [...] 07:11:09 DEBUG }, 07:11:09 DEBUG } 07:11:09 DEBUG 07:11:09 DEBUG }, 07:11:09 DEBUG } 07:11:09 DEBUG "metric": { 07:11:09 DEBUG 07:11:09 DEBUG "value": [ 07:11:09 DEBUG 07:11:09 DEBUG "value": [ 07:11:09 DEBUG ] 07:11:09 DEBUG "instance": "10.142.0.107:9100" 07:11:09 DEBUG 07:11:09 DEBUG 1713593469, 07:11:09 DEBUG [...] 07:11:09 DEBUG 1713593469, 07:11:09 DEBUG } 07:11:09 DEBUG }, 07:11:09 DEBUG 07:11:09 DEBUG "1307.6666666666667" 07:11:09 DEBUG 07:11:09 DEBUG "0" 07:11:09 DEBUG } 07:11:09 DEBUG "value": [ 07:11:09 DEBUG "pod": "idm-b85f9d978-wnhxb" 07:11:09 DEBUG ] 07:11:09 DEBUG 07:11:09 DEBUG ] 07:11:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593469" 07:11:09 DEBUG 07:11:09 DEBUG }, 07:11:09 DEBUG } 07:11:09 DEBUG 07:11:09 DEBUG } 07:11:09 DEBUG 07:11:09 DEBUG "value": [ 07:11:09 DEBUG ] 07:11:09 DEBUG "pod": "ds-idrepo-2" 07:11:09 DEBUG ] 07:11:09 DEBUG } 07:11:09 DEBUG 07:11:09 DEBUG } 07:11:09 DEBUG }, 07:11:09 DEBUG 1713593469, 07:11:09 DEBUG } 07:11:09 DEBUG [...] 07:11:09 INFO [http_cmd]: http status code OK 07:11:09 DEBUG } 07:11:09 DEBUG "value": [ 07:11:09 DEBUG "0" 07:11:09 DEBUG 07:11:09 DEBUG --- status code --- 07:11:09 DEBUG 1713593469, 07:11:09 DEBUG ] 07:11:09 DEBUG 07:11:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:11:09 DEBUG "0" 07:11:09 DEBUG } 07:11:09 DEBUG 07:11:09 DEBUG --- http response --- 07:11:09 DEBUG ] 07:11:09 DEBUG ] 07:11:09 DEBUG "instance": "10.142.0.43:9100" 07:11:09 DEBUG } 07:11:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:11:09 DEBUG } 07:11:09 DEBUG }, 07:11:09 DEBUG ] 07:11:09 DEBUG ----- output ----- 07:11:09 DEBUG } 07:11:09 DEBUG "value": [ 07:11:09 DEBUG } 07:11:09 DEBUG { 07:11:09 DEBUG 1713593469, 07:11:09 DEBUG } 07:11:09 DEBUG "status": "success", 07:11:09 DEBUG "8495314.233333332" 07:11:09 DEBUG "data": { 07:11:09 DEBUG ] 07:11:09 DEBUG "resultType": "vector", 07:11:09 DEBUG } 07:11:09 DEBUG "result": [ 07:11:09 DEBUG ] 07:11:09 DEBUG { 07:11:09 DEBUG } 07:11:09 DEBUG "metric": { 07:11:09 DEBUG } 07:11:09 DEBUG "pod": "am-767f6b67b5-9zw22" 07:11:09 DEBUG }, 07:11:09 DEBUG "value": [ 07:11:09 DEBUG 07:11:09 DEBUG 07:11:09 DEBUG 07:11:09 DEBUG [...] 07:11:09 DEBUG 07:11:09 DEBUG 07:11:09 DEBUG 07:11:09 DEBUG "pod": "am-767f6b67b5-vbltz" 07:11:09 DEBUG }, 07:11:09 DEBUG "value": [ 07:11:09 DEBUG 1713593469, 07:11:09 DEBUG "0" 07:11:09 DEBUG ] 07:11:09 DEBUG } 07:11:09 DEBUG ] 07:11:09 DEBUG } 07:11:09 DEBUG } 07:11:10 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713593470" 07:11:10 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593470" 07:11:10 INFO [http_cmd]: http status code OK 07:11:10 DEBUG --- status code --- 07:11:10 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:11:10 DEBUG --- http response --- 07:11:10 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:11:10 DEBUG ----- output ----- 07:11:10 DEBUG { 07:11:10 DEBUG "status": "success", 07:11:10 DEBUG "data": { 07:11:10 DEBUG "resultType": "vector", 07:11:10 DEBUG "result": [ 07:11:10 DEBUG { 07:11:10 DEBUG "metric": { 07:11:10 DEBUG "pod": "am-767f6b67b5-9zw22" 07:11:10 DEBUG }, 07:11:10 DEBUG "value": [ 07:11:10 DEBUG 07:11:10 DEBUG 07:11:10 DEBUG 07:11:10 DEBUG [...] 07:11:10 DEBUG 07:11:10 DEBUG 07:11:10 DEBUG 07:11:10 DEBUG "pod": "am-767f6b67b5-vbltz" 07:11:10 DEBUG }, 07:11:10 DEBUG "value": [ 07:11:10 DEBUG 1713593469, 07:11:10 DEBUG "0" 07:11:10 DEBUG ] 07:11:10 DEBUG } 07:11:10 DEBUG ] 07:11:10 DEBUG } 07:11:10 DEBUG } 07:11:10 INFO [http_cmd]: http status code OK 07:11:10 DEBUG --- status code --- 07:11:10 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:11:10 DEBUG --- http response --- 07:11:10 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:11:10 DEBUG ----- output ----- 07:11:10 DEBUG { 07:11:10 DEBUG "status": "success", 07:11:10 DEBUG "data": { 07:11:10 DEBUG "resultType": "vector", 07:11:10 DEBUG "result": [ 07:11:10 DEBUG { 07:11:10 DEBUG "metric": { 07:11:10 DEBUG "pod": "am-767f6b67b5-9zw22" 07:11:10 DEBUG }, 07:11:10 DEBUG "value": [ 07:11:10 DEBUG 07:11:10 DEBUG 07:11:10 DEBUG 07:11:10 DEBUG [...] 07:11:10 DEBUG 07:11:10 DEBUG 07:11:10 DEBUG 07:11:10 DEBUG "pod": "am-767f6b67b5-vbltz" 07:11:10 DEBUG }, 07:11:10 DEBUG "value": [ 07:11:10 DEBUG 1713593470, 07:11:10 DEBUG "0.097" 07:11:10 DEBUG ] 07:11:10 DEBUG } 07:11:10 DEBUG ] 07:11:10 DEBUG } 07:11:10 DEBUG } 07:11:10 INFO [http_cmd]: http status code OK 07:11:10 DEBUG --- status code --- 07:11:10 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:11:10 DEBUG --- http response --- 07:11:10 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 07:11:10 DEBUG ----- output ----- 07:11:10 DEBUG { 07:11:10 DEBUG "status": "success", 07:11:10 DEBUG "data": { 07:11:10 DEBUG "resultType": "vector", 07:11:10 DEBUG "result": [ 07:11:10 DEBUG { 07:11:10 DEBUG "metric": { 07:11:10 DEBUG "pod": "ds-cts-0" 07:11:10 DEBUG }, 07:11:10 DEBUG "value": [ 07:11:10 DEBUG 07:11:10 DEBUG 07:11:10 DEBUG 07:11:10 DEBUG [...] 07:11:10 DEBUG 07:11:10 DEBUG 07:11:10 DEBUG 07:11:10 DEBUG "pod": "ds-cts-2" 07:11:10 DEBUG }, 07:11:10 DEBUG "value": [ 07:11:10 DEBUG 1713593470, 07:11:10 DEBUG "0" 07:11:10 DEBUG ] 07:11:10 DEBUG } 07:11:10 DEBUG ] 07:11:10 DEBUG } 07:11:10 DEBUG } 07:11:10 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713593470" 07:11:10 INFO [http_cmd]: http status code OK 07:11:10 DEBUG --- status code --- 07:11:10 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:11:10 DEBUG --- http response --- 07:11:10 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:11:10 DEBUG ----- output ----- 07:11:10 DEBUG { 07:11:10 DEBUG "status": "success", 07:11:10 DEBUG "data": { 07:11:10 DEBUG "resultType": "vector", 07:11:10 DEBUG "result": [ 07:11:10 DEBUG { 07:11:10 DEBUG "metric": { 07:11:10 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 07:11:10 DEBUG }, 07:11:10 DEBUG "value": [ 07:11:10 DEBUG 07:11:10 DEBUG 07:11:10 DEBUG 07:11:10 DEBUG [...] 07:11:10 DEBUG 07:11:10 DEBUG 07:11:10 DEBUG 07:11:10 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 07:11:10 DEBUG }, 07:11:10 DEBUG "value": [ 07:11:10 DEBUG 1713593470, 07:11:10 DEBUG "1.037889877191899" 07:11:10 DEBUG ] 07:11:10 DEBUG } 07:11:10 DEBUG ] 07:11:10 DEBUG } 07:11:10 DEBUG } 07:11:10 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593470" 07:11:10 INFO [http_cmd]: http status code OK 07:11:10 DEBUG --- status code --- 07:11:10 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:11:10 DEBUG --- http response --- 07:11:10 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:11:10 DEBUG ----- output ----- 07:11:10 DEBUG { 07:11:10 DEBUG "status": "success", 07:11:10 DEBUG "data": { 07:11:10 DEBUG "resultType": "vector", 07:11:10 DEBUG "result": [ 07:11:10 DEBUG { 07:11:10 DEBUG "metric": { 07:11:10 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 07:11:10 DEBUG }, 07:11:10 DEBUG "value": [ 07:11:10 DEBUG 07:11:10 DEBUG 07:11:10 DEBUG 07:11:10 DEBUG [...] 07:11:10 DEBUG 07:11:10 DEBUG 07:11:10 DEBUG 07:11:10 DEBUG "pod": "idm-b85f9d978-wnhxb" 07:11:10 DEBUG }, 07:11:10 DEBUG "value": [ 07:11:10 DEBUG 1713593470, 07:11:10 DEBUG "0.0020035004475186234" 07:11:10 DEBUG ] 07:11:10 DEBUG } 07:11:10 DEBUG ] 07:11:10 DEBUG } 07:11:10 DEBUG } 07:11:10 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593470" 07:11:10 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593470" 07:11:10 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593470" 07:11:10 INFO [http_cmd]: http status code OK 07:11:10 DEBUG --- status code --- 07:11:10 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:11:10 DEBUG --- http response --- 07:11:10 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:11:10 DEBUG ----- output ----- 07:11:10 DEBUG { 07:11:10 DEBUG "status": "success", 07:11:10 DEBUG "data": { 07:11:10 DEBUG "resultType": "vector", 07:11:10 DEBUG "result": [ 07:11:10 DEBUG { 07:11:10 DEBUG "metric": { 07:11:10 DEBUG "pod": "am-767f6b67b5-9zw22" 07:11:10 DEBUG }, 07:11:10 DEBUG "value": [ 07:11:10 DEBUG 07:11:10 DEBUG 07:11:10 DEBUG 07:11:10 DEBUG [...] 07:11:10 DEBUG 07:11:10 DEBUG 07:11:10 DEBUG 07:11:10 DEBUG "pod": "am-767f6b67b5-vbltz" 07:11:10 DEBUG }, 07:11:10 DEBUG "value": [ 07:11:10 DEBUG 1713593470, 07:11:10 DEBUG "0" 07:11:10 DEBUG ] 07:11:10 DEBUG } 07:11:10 DEBUG ] 07:11:10 DEBUG } 07:11:10 DEBUG } 07:11:10 INFO [http_cmd]: http status code OK 07:11:10 DEBUG --- status code --- 07:11:10 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:11:10 DEBUG --- http response --- 07:11:10 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:11:10 DEBUG ----- output ----- 07:11:10 DEBUG { 07:11:10 DEBUG "status": "success", 07:11:10 DEBUG "data": { 07:11:10 DEBUG "resultType": "vector", 07:11:10 DEBUG "result": [ 07:11:10 DEBUG { 07:11:10 DEBUG "metric": { 07:11:10 DEBUG "pod": "am-767f6b67b5-9zw22" 07:11:10 DEBUG }, 07:11:10 DEBUG "value": [ 07:11:10 DEBUG 07:11:10 DEBUG 07:11:10 DEBUG 07:11:10 DEBUG [...] 07:11:10 DEBUG 07:11:10 DEBUG 07:11:10 DEBUG 07:11:10 DEBUG "pod": "am-767f6b67b5-vbltz" 07:11:10 DEBUG }, 07:11:10 DEBUG "value": [ 07:11:10 DEBUG 1713593470, 07:11:10 DEBUG "0" 07:11:10 DEBUG ] 07:11:10 DEBUG } 07:11:10 DEBUG ] 07:11:10 DEBUG } 07:11:10 DEBUG } 07:11:10 INFO [http_cmd]: http status code OK 07:11:10 DEBUG --- status code --- 07:11:10 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:11:10 DEBUG --- http response --- 07:11:10 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:11:10 DEBUG ----- output ----- 07:11:10 DEBUG { 07:11:10 DEBUG "status": "success", 07:11:10 DEBUG "data": { 07:11:10 DEBUG "resultType": "vector", 07:11:10 DEBUG "result": [ 07:11:10 DEBUG { 07:11:10 DEBUG "metric": { 07:11:10 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 07:11:10 DEBUG }, 07:11:10 DEBUG "value": [ 07:11:10 DEBUG 07:11:10 DEBUG 07:11:10 DEBUG 07:11:10 DEBUG [...] 07:11:10 DEBUG 07:11:10 DEBUG 07:11:10 DEBUG 07:11:10 DEBUG "pod": "idm-b85f9d978-wnhxb" 07:11:10 DEBUG }, 07:11:10 DEBUG "value": [ 07:11:10 DEBUG 1713593470, 07:11:10 DEBUG "2575.71713884163" 07:11:10 DEBUG ] 07:11:10 DEBUG } 07:11:10 DEBUG ] 07:11:10 DEBUG } 07:11:10 DEBUG } 07:11:10 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713593470" 07:11:10 INFO [http_cmd]: http status code OK 07:11:10 DEBUG --- status code --- 07:11:10 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:11:10 DEBUG --- http response --- 07:11:10 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:11:10 DEBUG ----- output ----- 07:11:10 DEBUG { 07:11:10 DEBUG "status": "success", 07:11:10 DEBUG "data": { 07:11:10 DEBUG "resultType": "vector", 07:11:10 DEBUG "result": [ 07:11:10 DEBUG { 07:11:10 DEBUG "metric": { 07:11:10 DEBUG "instance": "10.142.0.107:9100" 07:11:10 DEBUG }, 07:11:10 DEBUG "value": [ 07:11:10 DEBUG 07:11:10 DEBUG 07:11:10 DEBUG 07:11:10 DEBUG [...] 07:11:10 DEBUG 07:11:10 DEBUG 07:11:10 DEBUG 07:11:10 DEBUG "instance": "10.142.0.43:9100" 07:11:10 DEBUG }, 07:11:10 DEBUG "value": [ 07:11:10 DEBUG 1713593470, 07:11:10 DEBUG "8496466.133333333" 07:11:10 DEBUG ] 07:11:10 DEBUG } 07:11:10 DEBUG ] 07:11:10 DEBUG } 07:11:10 DEBUG } 07:11:10 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713593470" 07:11:10 INFO [http_cmd]: http status code OK 07:11:10 DEBUG --- status code --- 07:11:10 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:11:10 DEBUG --- http response --- 07:11:10 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:11:10 DEBUG ----- output ----- 07:11:10 DEBUG { 07:11:10 DEBUG "status": "success", 07:11:10 DEBUG "data": { 07:11:10 DEBUG "resultType": "vector", 07:11:10 DEBUG "result": [ 07:11:10 DEBUG { 07:11:10 DEBUG "metric": { 07:11:10 DEBUG "instance": "10.142.0.107:9100" 07:11:10 DEBUG }, 07:11:10 DEBUG "value": [ 07:11:10 DEBUG 07:11:10 DEBUG 07:11:10 DEBUG 07:11:10 DEBUG [...] 07:11:10 DEBUG 07:11:10 DEBUG 07:11:10 DEBUG 07:11:10 DEBUG "instance": "10.142.0.43:9100" 07:11:10 DEBUG }, 07:11:10 DEBUG "value": [ 07:11:10 DEBUG 1713593470, 07:11:10 DEBUG "0.011666666666663635" 07:11:10 DEBUG ] 07:11:10 DEBUG } 07:11:10 DEBUG ] 07:11:10 DEBUG } 07:11:10 DEBUG } 07:11:12 INFO 07:11:12 INFO [loop_until]: kubectl --namespace=xlou top pods 07:11:12 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 07:11:12 INFO 07:11:12 INFO [loop_until]: kubectl --namespace=xlou top node 07:11:12 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 07:11:12 INFO [loop_until]: OK (rc = 0) 07:11:12 DEBUG --- stdout --- 07:11:12 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 1280m 8609Mi am-767f6b67b5-qcqz2 1286m 8669Mi am-767f6b67b5-vbltz 1471m 8615Mi ds-cts-0 3714m 10483Mi ds-cts-1 3289m 10723Mi ds-cts-2 3726m 10250Mi ds-idrepo-0 394m 3802Mi ds-idrepo-1 370m 3789Mi ds-idrepo-2 399m 3718Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 3m 1178Mi idm-b85f9d978-wnhxb 3m 1237Mi lodemon-7cb5b9456d-dskfn 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-5c67bf94db-77mh9 1046m 1651Mi 07:11:12 DEBUG --- stderr --- 07:11:12 DEBUG 07:11:12 INFO [loop_until]: OK (rc = 0) 07:11:12 DEBUG --- stdout --- 07:11:12 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 72m 0% 6274Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 444m 1% 7474Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 1292m 4% 11349Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 86m 0% 3376Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 3669m 12% 13625Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 149m 0% 5187Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 1352m 4% 11577Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 77m 0% 3342Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 1544m 5% 10859Mi 9% gke-xlou-cdm-ds-0f601751-j488 451m 1% 7026Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 3321m 11% 15649Mi 14% gke-xlou-cdm-ds-0f601751-nj6q 473m 1% 7680Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 3742m 12% 12867Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 1079m 6% 5541Mi 9% 07:11:12 DEBUG --- stderr --- 07:11:12 DEBUG 07:12:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713593529" 07:12:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593529" 07:12:09 ERROR [http_cmd]: ERROR 07:12:09 ERROR --- status code --- 07:12:09 ERROR http status code is 400 (expected [200, 401, 403, 404]) 07:12:09 ERROR --- http response --- 07:12:09 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 07:12:09 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 07:12:09 INFO [http_cmd]: http status code OK 07:12:09 DEBUG --- status code --- 07:12:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:12:09 DEBUG --- http response --- 07:12:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:12:09 DEBUG ----- output ----- 07:12:09 DEBUG { 07:12:09 DEBUG "status": "success", 07:12:09 DEBUG "data": { 07:12:09 DEBUG "resultType": "vector", 07:12:09 DEBUG "result": [ 07:12:09 DEBUG { 07:12:09 DEBUG "metric": { 07:12:09 DEBUG "pod": "ds-idrepo-0" 07:12:09 DEBUG }, 07:12:09 DEBUG "value": [ 07:12:09 DEBUG 07:12:09 DEBUG 07:12:09 DEBUG 07:12:09 DEBUG [...] 07:12:09 DEBUG 07:12:09 DEBUG 07:12:09 DEBUG 07:12:09 DEBUG "pod": "ds-idrepo-2" 07:12:09 DEBUG }, 07:12:09 DEBUG "value": [ 07:12:09 DEBUG 1713593529, 07:12:09 DEBUG "0" 07:12:09 DEBUG ] 07:12:09 DEBUG } 07:12:09 DEBUG ] 07:12:09 DEBUG } 07:12:09 DEBUG } 07:12:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593529" 07:12:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593529" 07:12:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593529" 07:12:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593529" 07:12:09 INFO [http_cmd]: http status code OK 07:12:09 DEBUG --- status code --- 07:12:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:12:09 DEBUG --- http response --- 07:12:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 07:12:09 DEBUG ----- output ----- 07:12:09 DEBUG { 07:12:09 DEBUG "status": "success", 07:12:09 DEBUG "data": { 07:12:09 DEBUG "resultType": "vector", 07:12:09 DEBUG "result": [ 07:12:09 DEBUG { 07:12:09 DEBUG "metric": { 07:12:09 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 07:12:09 DEBUG }, 07:12:09 DEBUG "value": [ 07:12:09 DEBUG 07:12:09 DEBUG 07:12:09 DEBUG 07:12:09 DEBUG [...] 07:12:09 DEBUG 07:12:09 DEBUG 07:12:09 DEBUG 07:12:09 DEBUG "pod": "am-767f6b67b5-qcqz2" 07:12:09 DEBUG }, 07:12:09 DEBUG "value": [ 07:12:09 DEBUG 1713593529, 07:12:09 DEBUG "455.9627906976745" 07:12:09 DEBUG ] 07:12:09 DEBUG } 07:12:09 DEBUG ] 07:12:09 DEBUG } 07:12:09 DEBUG } 07:12:09 INFO [http_cmd]: http status code OK 07:12:09 DEBUG --- status code --- 07:12:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:12:09 DEBUG --- http response --- 07:12:09 INFO [http_cmd]: http status code OK 07:12:09 INFO [http_cmd]: http status code OK 07:12:09 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 07:12:09 DEBUG --- status code --- 07:12:09 DEBUG --- status code --- 07:12:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:12:09 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:12:09 DEBUG --- http response --- 07:12:09 DEBUG --- http response --- 07:12:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:12:09 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:12:09 DEBUG ----- output ----- 07:12:09 DEBUG ----- output ----- 07:12:09 DEBUG { 07:12:09 DEBUG { 07:12:09 DEBUG "status": "success", 07:12:09 DEBUG "status": "success", 07:12:09 DEBUG "data": { 07:12:09 DEBUG "data": { 07:12:09 DEBUG "resultType": "vector", 07:12:09 DEBUG "resultType": "vector", 07:12:09 DEBUG "result": [ 07:12:09 DEBUG "result": [ 07:12:09 DEBUG { 07:12:09 DEBUG { 07:12:09 DEBUG "metric": { 07:12:09 DEBUG "metric": { 07:12:09 DEBUG "pod": "am-767f6b67b5-9zw22" 07:12:09 DEBUG "pod": "ds-idrepo-0" 07:12:09 DEBUG }, 07:12:09 DEBUG }, 07:12:09 DEBUG "value": [ 07:12:09 DEBUG "value": [ 07:12:09 DEBUG 07:12:09 DEBUG 07:12:09 DEBUG 07:12:09 DEBUG 07:12:09 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713593529" 07:12:09 DEBUG 07:12:09 DEBUG 07:12:09 DEBUG [...] 07:12:09 DEBUG [...] 07:12:09 DEBUG 07:12:09 DEBUG 07:12:09 DEBUG 07:12:09 DEBUG 07:12:09 DEBUG 07:12:09 DEBUG 07:12:09 DEBUG "pod": "am-767f6b67b5-vbltz" 07:12:09 DEBUG "pod": "ds-idrepo-2" 07:12:09 DEBUG }, 07:12:09 DEBUG }, 07:12:09 DEBUG "value": [ 07:12:09 DEBUG "value": [ 07:12:09 DEBUG 1713593529, 07:12:09 DEBUG 1713593529, 07:12:09 DEBUG "2" 07:12:09 DEBUG "0" 07:12:09 DEBUG ] 07:12:09 DEBUG ] 07:12:09 DEBUG } 07:12:09 DEBUG } 07:12:09 DEBUG ] 07:12:09 DEBUG ] 07:12:09 DEBUG } 07:12:09 DEBUG } 07:12:09 DEBUG } 07:12:09 DEBUG } 07:12:10 INFO [http_cmd]: http status code OK 07:12:10 DEBUG --- status code --- 07:12:10 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:12:10 DEBUG --- http response --- 07:12:10 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:12:10 DEBUG ----- output ----- 07:12:10 DEBUG { 07:12:10 DEBUG "status": "success", 07:12:10 DEBUG "data": { 07:12:10 DEBUG "resultType": "vector", 07:12:10 DEBUG "result": [ 07:12:10 DEBUG { 07:12:10 DEBUG "metric": { 07:12:10 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 07:12:10 DEBUG }, 07:12:10 DEBUG "value": [ 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG [...] 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 07:12:10 DEBUG }, 07:12:10 DEBUG "value": [ 07:12:10 DEBUG 1713593529, 07:12:10 DEBUG "1947189248" 07:12:10 DEBUG ] 07:12:10 DEBUG } 07:12:10 DEBUG ] 07:12:10 DEBUG } 07:12:10 DEBUG } 07:12:10 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713593530" 07:12:10 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593530" 07:12:10 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713593530" 07:12:10 INFO [http_cmd]: http status code OK 07:12:10 DEBUG --- status code --- 07:12:10 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:12:10 DEBUG --- http response --- 07:12:10 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:12:10 DEBUG ----- output ----- 07:12:10 DEBUG { 07:12:10 DEBUG "status": "success", 07:12:10 DEBUG "data": { 07:12:10 DEBUG "resultType": "vector", 07:12:10 DEBUG "result": [ 07:12:10 DEBUG { 07:12:10 DEBUG "metric": { 07:12:10 DEBUG "instance": "10.142.0.107:9100" 07:12:10 DEBUG }, 07:12:10 DEBUG "value": [ 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG [...] 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG "instance": "10.142.0.43:9100" 07:12:10 DEBUG }, 07:12:10 DEBUG "value": [ 07:12:10 DEBUG 1713593530, 07:12:10 DEBUG "3295518278656" 07:12:10 DEBUG ] 07:12:10 DEBUG } 07:12:10 DEBUG ] 07:12:10 DEBUG } 07:12:10 DEBUG } 07:12:10 INFO [http_cmd]: http status code OK 07:12:10 DEBUG --- status code --- 07:12:10 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:12:10 DEBUG --- http response --- 07:12:10 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 07:12:10 DEBUG ----- output ----- 07:12:10 DEBUG { 07:12:10 DEBUG "status": "success", 07:12:10 DEBUG "data": { 07:12:10 DEBUG "resultType": "vector", 07:12:10 DEBUG "result": [ 07:12:10 DEBUG { 07:12:10 DEBUG "metric": { 07:12:10 DEBUG "pod": "ds-cts-0" 07:12:10 DEBUG }, 07:12:10 DEBUG "value": [ 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG [...] 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG "pod": "ds-cts-2" 07:12:10 DEBUG }, 07:12:10 DEBUG "value": [ 07:12:10 DEBUG 1713593530, 07:12:10 DEBUG "29.933333333333334" 07:12:10 DEBUG ] 07:12:10 DEBUG } 07:12:10 DEBUG ] 07:12:10 DEBUG } 07:12:10 DEBUG } 07:12:10 INFO [http_cmd]: http status code OK 07:12:10 DEBUG --- status code --- 07:12:10 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:12:10 DEBUG --- http response --- 07:12:10 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:12:10 DEBUG ----- output ----- 07:12:10 DEBUG { 07:12:10 DEBUG "status": "success", 07:12:10 DEBUG "data": { 07:12:10 DEBUG "resultType": "vector", 07:12:10 DEBUG "result": [ 07:12:10 DEBUG { 07:12:10 DEBUG "metric": { 07:12:10 DEBUG "instance": "10.142.0.107:9100" 07:12:10 DEBUG }, 07:12:10 DEBUG "value": [ 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG [...] 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG "instance": "10.142.0.43:9100" 07:12:10 DEBUG }, 07:12:10 DEBUG "value": [ 07:12:10 DEBUG 1713593530, 07:12:10 DEBUG "0.0012666666666281876" 07:12:10 DEBUG ] 07:12:10 DEBUG } 07:12:10 DEBUG ] 07:12:10 DEBUG } 07:12:10 DEBUG } 07:12:10 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713593530" 07:12:10 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593530" 07:12:10 INFO [http_cmd]: http status code OK 07:12:10 DEBUG --- status code --- 07:12:10 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:12:10 DEBUG --- http response --- 07:12:10 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593530" 07:12:10 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:12:10 DEBUG ----- output ----- 07:12:10 DEBUG { 07:12:10 DEBUG "status": "success", 07:12:10 DEBUG "data": { 07:12:10 DEBUG "resultType": "vector", 07:12:10 DEBUG "result": [ 07:12:10 DEBUG { 07:12:10 DEBUG "metric": { 07:12:10 DEBUG "pod": "am-767f6b67b5-9zw22" 07:12:10 DEBUG }, 07:12:10 DEBUG "value": [ 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG [...] 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG "pod": "am-767f6b67b5-vbltz" 07:12:10 DEBUG }, 07:12:10 DEBUG "value": [ 07:12:10 DEBUG 1713593530, 07:12:10 DEBUG "0.097" 07:12:10 DEBUG ] 07:12:10 DEBUG } 07:12:10 DEBUG ] 07:12:10 DEBUG } 07:12:10 DEBUG } 07:12:10 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593530" 07:12:10 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713593530" 07:12:10 INFO [http_cmd]: http status code OK 07:12:10 DEBUG --- status code --- 07:12:10 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:12:10 DEBUG --- http response --- 07:12:10 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:12:10 DEBUG ----- output ----- 07:12:10 DEBUG { 07:12:10 DEBUG "status": "success", 07:12:10 DEBUG "data": { 07:12:10 DEBUG "resultType": "vector", 07:12:10 DEBUG "result": [ 07:12:10 DEBUG { 07:12:10 DEBUG "metric": { 07:12:10 DEBUG "pod": "am-767f6b67b5-9zw22" 07:12:10 DEBUG }, 07:12:10 DEBUG "value": [ 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG [...] 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG "pod": "am-767f6b67b5-vbltz" 07:12:10 DEBUG }, 07:12:10 DEBUG "value": [ 07:12:10 DEBUG 1713593530, 07:12:10 DEBUG "0" 07:12:10 DEBUG ] 07:12:10 DEBUG } 07:12:10 DEBUG ] 07:12:10 DEBUG } 07:12:10 DEBUG } 07:12:10 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593530" 07:12:10 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713593530" 07:12:10 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593530" 07:12:10 INFO [http_cmd]: http status code OK 07:12:10 DEBUG --- status code --- 07:12:10 INFO [http_cmd]: http status code OK 07:12:10 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:12:10 DEBUG --- status code --- 07:12:10 DEBUG --- http response --- 07:12:10 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:12:10 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 07:12:10 DEBUG --- http response --- 07:12:10 DEBUG ----- output ----- 07:12:10 DEBUG { 07:12:10 DEBUG "status": "success", 07:12:10 DEBUG "data": { 07:12:10 DEBUG "resultType": "vector", 07:12:10 DEBUG "result": [ 07:12:10 DEBUG { 07:12:10 DEBUG "metric": { 07:12:10 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 07:12:10 DEBUG }, 07:12:10 DEBUG "value": [ 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG [...] 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG "pod": "am-767f6b67b5-qcqz2" 07:12:10 DEBUG }, 07:12:10 DEBUG "value": [ 07:12:10 DEBUG 1713593530, 07:12:10 DEBUG "0" 07:12:10 DEBUG ] 07:12:10 DEBUG } 07:12:10 DEBUG ] 07:12:10 DEBUG } 07:12:10 DEBUG } 07:12:10 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 07:12:10 INFO [http_cmd]: http status code OK 07:12:10 DEBUG ----- output ----- 07:12:10 DEBUG --- status code --- 07:12:10 DEBUG { 07:12:10 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:12:10 DEBUG "status": "success", 07:12:10 DEBUG --- http response --- 07:12:10 DEBUG "data": { 07:12:10 DEBUG "resultType": "vector", 07:12:10 DEBUG "result": [ 07:12:10 DEBUG { 07:12:10 DEBUG "metric": { 07:12:10 DEBUG "pod": "ds-cts-0" 07:12:10 DEBUG }, 07:12:10 DEBUG "value": [ 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG [...] 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG "pod": "ds-cts-2" 07:12:10 DEBUG }, 07:12:10 DEBUG "value": [ 07:12:10 DEBUG 1713593530, 07:12:10 DEBUG "0" 07:12:10 DEBUG ] 07:12:10 DEBUG } 07:12:10 DEBUG ] 07:12:10 DEBUG } 07:12:10 DEBUG } 07:12:10 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 07:12:10 INFO [http_cmd]: http status code OK 07:12:10 DEBUG ----- output ----- 07:12:10 DEBUG --- status code --- 07:12:10 DEBUG { 07:12:10 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:12:10 DEBUG "status": "success", 07:12:10 DEBUG --- http response --- 07:12:10 DEBUG "data": { 07:12:10 DEBUG "resultType": "vector", 07:12:10 DEBUG "result": [ 07:12:10 DEBUG { 07:12:10 DEBUG "metric": { 07:12:10 DEBUG "pod": "ds-idrepo-0" 07:12:10 DEBUG }, 07:12:10 DEBUG "value": [ 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG [...] 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG "pod": "ds-cts-2" 07:12:10 DEBUG }, 07:12:10 DEBUG "value": [ 07:12:10 DEBUG 1713593530, 07:12:10 DEBUG "0" 07:12:10 DEBUG ] 07:12:10 DEBUG } 07:12:10 DEBUG ] 07:12:10 DEBUG } 07:12:10 DEBUG } 07:12:10 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:12:10 DEBUG ----- output ----- 07:12:10 DEBUG { 07:12:10 DEBUG "status": "success", 07:12:10 DEBUG "data": { 07:12:10 DEBUG "resultType": "vector", 07:12:10 DEBUG "result": [ 07:12:10 DEBUG { 07:12:10 DEBUG "metric": { 07:12:10 DEBUG "pod": "am-767f6b67b5-9zw22" 07:12:10 DEBUG }, 07:12:10 DEBUG "value": [ 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG [...] 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG "pod": "am-767f6b67b5-vbltz" 07:12:10 DEBUG }, 07:12:10 DEBUG "value": [ 07:12:10 DEBUG 1713593530, 07:12:10 DEBUG "0" 07:12:10 DEBUG ] 07:12:10 DEBUG } 07:12:10 DEBUG ] 07:12:10 DEBUG } 07:12:10 DEBUG } 07:12:10 INFO [http_cmd]: http status code OK 07:12:10 DEBUG --- status code --- 07:12:10 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:12:10 DEBUG --- http response --- 07:12:10 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:12:10 DEBUG ----- output ----- 07:12:10 DEBUG { 07:12:10 DEBUG "status": "success", 07:12:10 DEBUG "data": { 07:12:10 DEBUG "resultType": "vector", 07:12:10 DEBUG "result": [ 07:12:10 DEBUG { 07:12:10 DEBUG "metric": { 07:12:10 DEBUG "pod": "ds-idrepo-0" 07:12:10 DEBUG }, 07:12:10 DEBUG "value": [ 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG [...] 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG "pod": "ds-idrepo-2" 07:12:10 DEBUG }, 07:12:10 DEBUG "value": [ 07:12:10 DEBUG 1713593530, 07:12:10 DEBUG "0" 07:12:10 DEBUG ] 07:12:10 DEBUG } 07:12:10 DEBUG ] 07:12:10 DEBUG } 07:12:10 DEBUG } 07:12:10 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593530" 07:12:10 INFO [http_cmd]: http status code OK 07:12:10 DEBUG --- status code --- 07:12:10 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:12:10 DEBUG --- http response --- 07:12:10 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:12:10 DEBUG ----- output ----- 07:12:10 DEBUG { 07:12:10 DEBUG "status": "success", 07:12:10 DEBUG "data": { 07:12:10 DEBUG "resultType": "vector", 07:12:10 DEBUG "result": [ 07:12:10 DEBUG { 07:12:10 DEBUG "metric": { 07:12:10 DEBUG "instance": "10.142.0.107:9100" 07:12:10 DEBUG }, 07:12:10 DEBUG "value": [ 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG [...] 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG "instance": "10.142.0.43:9100" 07:12:10 DEBUG }, 07:12:10 DEBUG "value": [ 07:12:10 DEBUG 1713593530, 07:12:10 DEBUG "26839.966666666667" 07:12:10 DEBUG ] 07:12:10 DEBUG } 07:12:10 DEBUG ] 07:12:10 DEBUG } 07:12:10 DEBUG } 07:12:10 INFO [http_cmd]: http status code OK 07:12:10 DEBUG --- status code --- 07:12:10 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:12:10 DEBUG --- http response --- 07:12:10 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:12:10 DEBUG ----- output ----- 07:12:10 DEBUG { 07:12:10 DEBUG "status": "success", 07:12:10 DEBUG "data": { 07:12:10 DEBUG "resultType": "vector", 07:12:10 DEBUG "result": [ 07:12:10 DEBUG { 07:12:10 DEBUG "metric": { 07:12:10 DEBUG "pod": "am-767f6b67b5-9zw22" 07:12:10 DEBUG }, 07:12:10 DEBUG "value": [ 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG [...] 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG "pod": "am-767f6b67b5-vbltz" 07:12:10 DEBUG }, 07:12:10 DEBUG "value": [ 07:12:10 DEBUG 1713593530, 07:12:10 DEBUG "0" 07:12:10 DEBUG ] 07:12:10 DEBUG } 07:12:10 DEBUG ] 07:12:10 DEBUG } 07:12:10 DEBUG } 07:12:10 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593530" 07:12:10 INFO [http_cmd]: http status code OK 07:12:10 DEBUG --- status code --- 07:12:10 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:12:10 DEBUG --- http response --- 07:12:10 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 07:12:10 DEBUG ----- output ----- 07:12:10 DEBUG { 07:12:10 DEBUG "status": "success", 07:12:10 DEBUG "data": { 07:12:10 DEBUG "resultType": "vector", 07:12:10 DEBUG "result": [ 07:12:10 DEBUG { 07:12:10 DEBUG "metric": { 07:12:10 DEBUG "pod": "ds-cts-0" 07:12:10 DEBUG }, 07:12:10 DEBUG "value": [ 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG [...] 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG "pod": "ds-cts-2" 07:12:10 DEBUG }, 07:12:10 DEBUG "value": [ 07:12:10 DEBUG 1713593530, 07:12:10 DEBUG "0" 07:12:10 DEBUG ] 07:12:10 DEBUG } 07:12:10 DEBUG ] 07:12:10 DEBUG } 07:12:10 DEBUG } 07:12:10 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713593530" 07:12:10 INFO [http_cmd]: http status code OK 07:12:10 DEBUG --- status code --- 07:12:10 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:12:10 DEBUG --- http response --- 07:12:10 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:12:10 DEBUG ----- output ----- 07:12:10 DEBUG { 07:12:10 DEBUG "status": "success", 07:12:10 DEBUG "data": { 07:12:10 DEBUG "resultType": "vector", 07:12:10 DEBUG "result": [ 07:12:10 DEBUG { 07:12:10 DEBUG "metric": { 07:12:10 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 07:12:10 DEBUG }, 07:12:10 DEBUG "value": [ 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG [...] 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 07:12:10 DEBUG }, 07:12:10 DEBUG "value": [ 07:12:10 DEBUG 1713593530, 07:12:10 DEBUG "0.7281777679654924" 07:12:10 DEBUG ] 07:12:10 DEBUG } 07:12:10 DEBUG ] 07:12:10 DEBUG } 07:12:10 DEBUG } 07:12:10 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593530" 07:12:10 INFO [http_cmd]: http status code OK 07:12:10 DEBUG --- status code --- 07:12:10 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:12:10 DEBUG --- http response --- 07:12:10 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 07:12:10 DEBUG ----- output ----- 07:12:10 DEBUG { 07:12:10 DEBUG "status": "success", 07:12:10 DEBUG "data": { 07:12:10 DEBUG "resultType": "vector", 07:12:10 DEBUG "result": [ 07:12:10 DEBUG { 07:12:10 DEBUG "metric": { 07:12:10 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 07:12:10 DEBUG }, 07:12:10 DEBUG "value": [ 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG [...] 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG "pod": "am-767f6b67b5-qcqz2" 07:12:10 DEBUG }, 07:12:10 DEBUG "value": [ 07:12:10 DEBUG 1713593530, 07:12:10 DEBUG "0.09563519435590831" 07:12:10 DEBUG ] 07:12:10 DEBUG } 07:12:10 DEBUG ] 07:12:10 DEBUG } 07:12:10 DEBUG } 07:12:10 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593530" 07:12:10 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593530" 07:12:10 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713593530" 07:12:10 INFO [http_cmd]: http status code OK 07:12:10 DEBUG --- status code --- 07:12:10 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:12:10 DEBUG --- http response --- 07:12:10 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:12:10 DEBUG ----- output ----- 07:12:10 DEBUG { 07:12:10 DEBUG "status": "success", 07:12:10 DEBUG "data": { 07:12:10 DEBUG "resultType": "vector", 07:12:10 DEBUG "result": [ 07:12:10 DEBUG { 07:12:10 DEBUG "metric": { 07:12:10 DEBUG "pod": "am-767f6b67b5-9zw22" 07:12:10 DEBUG }, 07:12:10 DEBUG "value": [ 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG [...] 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG "pod": "am-767f6b67b5-vbltz" 07:12:10 DEBUG }, 07:12:10 DEBUG "value": [ 07:12:10 DEBUG 1713593530, 07:12:10 DEBUG "0" 07:12:10 DEBUG ] 07:12:10 DEBUG } 07:12:10 DEBUG ] 07:12:10 DEBUG } 07:12:10 DEBUG } 07:12:10 INFO [http_cmd]: http status code OK 07:12:10 DEBUG --- status code --- 07:12:10 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:12:10 DEBUG --- http response --- 07:12:10 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 07:12:10 DEBUG ----- output ----- 07:12:10 DEBUG { 07:12:10 DEBUG "status": "success", 07:12:10 DEBUG "data": { 07:12:10 DEBUG "resultType": "vector", 07:12:10 DEBUG "result": [ 07:12:10 DEBUG { 07:12:10 DEBUG "metric": { 07:12:10 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 07:12:10 DEBUG }, 07:12:10 DEBUG "value": [ 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG [...] 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG "pod": "am-767f6b67b5-qcqz2" 07:12:10 DEBUG }, 07:12:10 DEBUG "value": [ 07:12:10 DEBUG 1713593530, 07:12:10 DEBUG "246005190.09612405" 07:12:10 DEBUG ] 07:12:10 DEBUG } 07:12:10 DEBUG ] 07:12:10 DEBUG } 07:12:10 DEBUG } 07:12:10 INFO [http_cmd]: http status code OK 07:12:10 DEBUG --- status code --- 07:12:10 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:12:10 DEBUG --- http response --- 07:12:10 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 07:12:10 DEBUG ----- output ----- 07:12:10 DEBUG { 07:12:10 DEBUG "status": "success", 07:12:10 DEBUG "data": { 07:12:10 DEBUG "resultType": "vector", 07:12:10 DEBUG "result": [ 07:12:10 DEBUG { 07:12:10 DEBUG "metric": { 07:12:10 DEBUG "pod": "am-767f6b67b5-9zw22" 07:12:10 DEBUG }, 07:12:10 DEBUG "value": [ 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG [...] 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG "pod": "am-767f6b67b5-vbltz" 07:12:10 DEBUG }, 07:12:10 DEBUG "value": [ 07:12:10 DEBUG 1713593530, 07:12:10 DEBUG "0" 07:12:10 DEBUG ] 07:12:10 DEBUG } 07:12:10 DEBUG ] 07:12:10 DEBUG } 07:12:10 DEBUG } 07:12:10 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713593530" 07:12:10 INFO [http_cmd]: http status code OK 07:12:10 DEBUG --- status code --- 07:12:10 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:12:10 DEBUG --- http response --- 07:12:10 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713593530" 07:12:10 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:12:10 DEBUG ----- output ----- 07:12:10 DEBUG { 07:12:10 DEBUG "status": "success", 07:12:10 DEBUG "data": { 07:12:10 DEBUG "resultType": "vector", 07:12:10 DEBUG "result": [ 07:12:10 DEBUG { 07:12:10 DEBUG "metric": { 07:12:10 DEBUG "instance": "10.142.0.107:9100" 07:12:10 DEBUG }, 07:12:10 DEBUG "value": [ 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG [...] 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG "instance": "10.142.0.43:9100" 07:12:10 DEBUG }, 07:12:10 DEBUG "value": [ 07:12:10 DEBUG 1713593530, 07:12:10 DEBUG "0.0006666666666660604" 07:12:10 DEBUG ] 07:12:10 DEBUG } 07:12:10 DEBUG ] 07:12:10 DEBUG } 07:12:10 DEBUG } 07:12:10 INFO [http_cmd]: http status code OK 07:12:10 DEBUG --- status code --- 07:12:10 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 07:12:10 DEBUG --- http response --- 07:12:10 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 07:12:10 DEBUG ----- output ----- 07:12:10 DEBUG { 07:12:10 DEBUG "status": "success", 07:12:10 DEBUG "data": { 07:12:10 DEBUG "resultType": "vector", 07:12:10 DEBUG "result": [ 07:12:10 DEBUG { 07:12:10 DEBUG "metric": { 07:12:10 DEBUG "instance": "10.142.0.107:9100" 07:12:10 DEBUG }, 07:12:10 DEBUG "value": [ 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG [...] 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG 07:12:10 DEBUG "instance": "10.142.0.43:9100" 07:12:10 DEBUG }, 07:12:10 DEBUG "value": [ 07:12:10 DEBUG 1713593530, 07:12:10 DEBUG "26623" 07:12:10 DEBUG ] 07:12:10 DEBUG } 07:12:10 DEBUG ] 07:12:10 DEBUG } 07:12:10 DEBUG } 07:12:11 INFO Finished: True 07:12:11 INFO Waiting for threads to register finish flag 07:13:10 INFO Done. Have a nice day! :) 127.0.0.1 - - [20/Apr/2024 07:13:10] "GET /monitoring/stop HTTP/1.1" 200 - 07:13:12 WARNING File /tmp/lodemon_data-20240420_063158/node_disk_read_bytes_total.json does not exist. Skipping graph generation... WARNING:bokeh.core.validation.check:W-1000 (MISSING_RENDERERS): Plot has no renderers: figure(id='p6501', ...) 127.0.0.1 - - [20/Apr/2024 07:13:16] "GET /monitoring/process HTTP/1.1" 200 -