==================================================================================================== ========================================= Pod describe ========================================= ==================================================================================================== Name: lodemon-54c87f8cbb-jzrjs Namespace: xlou Priority: 0 Service Account: k8s-svc-acct-lodemon Node: gke-xlou-cdm-ds-0f601751-mc5d/10.142.0.43 Start Time: Sun, 21 Apr 2024 01:59:40 +0000 Labels: app=lodemon app.kubernetes.io/name=lodemon pod-template-hash=54c87f8cbb skaffold.dev/run-id=0eafa3e1-e644-445b-8f3e-b85747a6cbe3 Annotations: Status: Running IP: 10.106.46.174 IPs: IP: 10.106.46.174 Controlled By: ReplicaSet/lodemon-54c87f8cbb Containers: lodemon: Container ID: containerd://1386b49a978bfbd573da1cb7231e413a71cf134f7505f3dc99853789e49a4381 Image: gcr.io/engineeringpit/lodestar-images/lodestarbox:master-stable Image ID: gcr.io/engineeringpit/lodestar-images/lodestarbox@sha256:03f76e6326d713f4d73dc474e0e25fbbbaecc7c1d8c0590bfd94d68ac0d98c97 Port: 8080/TCP Host Port: 0/TCP Command: python3 Args: /lodestar/scripts/lodemon_run.py -W default State: Running Started: Sun, 21 Apr 2024 01:59:41 +0000 Ready: True Restart Count: 0 Limits: cpu: 2 memory: 2Gi Requests: cpu: 1 memory: 1Gi Liveness: exec [cat /tmp/lodemon_alive] delay=20s timeout=1s period=10s #success=1 #failure=3 Readiness: exec [cat /tmp/lodemon_alive] delay=20s timeout=1s period=10s #success=1 #failure=3 Environment: SKAFFOLD_PROFILE: medium Mounts: /lodestar/config/config.yaml from config (rw,path="config.yaml") /lodestar/config/deployments_to_monitor.yaml from deployments-config (rw,path="deployments_to_monitor.yaml") /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-crb7f (ro) Conditions: Type Status Initialized True Ready True ContainersReady True PodScheduled True Volumes: config: Type: ConfigMap (a volume populated by a ConfigMap) Name: lodemon-config Optional: false deployments-config: Type: ConfigMap (a volume populated by a ConfigMap) Name: lodemon-deployments Optional: false kube-api-access-crb7f: Type: Projected (a volume that contains injected data from multiple sources) TokenExpirationSeconds: 3607 ConfigMapName: kube-root-ca.crt ConfigMapOptional: DownwardAPI: true QoS Class: Burstable Node-Selectors: Tolerations: node.kubernetes.io/not-ready:NoExecute op=Exists for 300s node.kubernetes.io/unreachable:NoExecute op=Exists for 300s Events: Type Reason Age From Message ---- ------ ---- ---- ------- Normal Scheduled 42m default-scheduler Successfully assigned xlou/lodemon-54c87f8cbb-jzrjs to gke-xlou-cdm-ds-0f601751-mc5d Normal Pulling 42m kubelet Pulling image "gcr.io/engineeringpit/lodestar-images/lodestarbox:master-stable" Normal Pulled 42m kubelet Successfully pulled image "gcr.io/engineeringpit/lodestar-images/lodestarbox:master-stable" in 258.801259ms (258.806528ms including waiting) Normal Created 42m kubelet Created container lodemon Normal Started 42m kubelet Started container lodemon ==================================================================================================== =========================================== Pod logs =========================================== ==================================================================================================== * Serving Flask app 'lodemon_run' * Debug mode: off WARNING: This is a development server. Do not use it in a production deployment. Use a production WSGI server instead. * Running on all addresses (0.0.0.0) * Running on http://127.0.0.1:8080 * Running on http://10.106.46.174:8080 Press CTRL+C to quit 03:00:47 INFO 03:00:47 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 03:00:47 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:00:47 INFO [loop_until]: OK (rc = 0) 03:00:47 DEBUG --- stdout --- 03:00:47 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 03:00:47 DEBUG --- stderr --- 03:00:47 DEBUG 03:00:47 INFO 03:00:47 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 03:00:47 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:00:48 INFO [loop_until]: OK (rc = 0) 03:00:48 DEBUG --- stdout --- 03:00:48 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 03:00:48 DEBUG --- stderr --- 03:00:48 DEBUG 03:00:48 INFO 03:00:48 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 03:00:48 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:00:48 INFO [loop_until]: OK (rc = 0) 03:00:48 DEBUG --- stdout --- 03:00:48 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 03:00:48 DEBUG --- stderr --- 03:00:48 DEBUG 03:00:48 INFO 03:00:48 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 03:00:48 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:00:48 INFO [loop_until]: OK (rc = 0) 03:00:48 DEBUG --- stdout --- 03:00:48 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 03:00:48 DEBUG --- stderr --- 03:00:48 DEBUG 03:00:48 INFO 03:00:48 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 03:00:48 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:00:48 INFO [loop_until]: OK (rc = 0) 03:00:48 DEBUG --- stdout --- 03:00:48 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 03:00:48 DEBUG --- stderr --- 03:00:48 DEBUG 03:00:48 INFO 03:00:48 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 03:00:48 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:00:48 INFO [loop_until]: OK (rc = 0) 03:00:48 DEBUG --- stdout --- 03:00:48 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 03:00:48 DEBUG --- stderr --- 03:00:48 DEBUG 03:00:48 INFO 03:00:48 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 03:00:48 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:00:48 INFO [loop_until]: OK (rc = 0) 03:00:48 DEBUG --- stdout --- 03:00:48 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 03:00:48 DEBUG --- stderr --- 03:00:48 DEBUG 03:00:48 INFO 03:00:48 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 03:00:48 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:00:48 INFO [loop_until]: OK (rc = 0) 03:00:48 DEBUG --- stdout --- 03:00:48 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 03:00:48 DEBUG --- stderr --- 03:00:48 DEBUG 03:00:48 INFO 03:00:48 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 03:00:48 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:00:48 INFO [loop_until]: OK (rc = 0) 03:00:48 DEBUG --- stdout --- 03:00:48 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 03:00:48 DEBUG --- stderr --- 03:00:48 DEBUG 03:00:49 INFO 03:00:49 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 03:00:49 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:00:49 INFO [loop_until]: OK (rc = 0) 03:00:49 DEBUG --- stdout --- 03:00:49 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 03:00:49 DEBUG --- stderr --- 03:00:49 DEBUG 03:00:49 INFO 03:00:49 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 03:00:49 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:00:49 INFO [loop_until]: OK (rc = 0) 03:00:49 DEBUG --- stdout --- 03:00:49 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 03:00:49 DEBUG --- stderr --- 03:00:49 DEBUG 03:00:49 INFO 03:00:49 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 03:00:49 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:00:49 INFO [loop_until]: OK (rc = 0) 03:00:49 DEBUG --- stdout --- 03:00:49 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 03:00:49 DEBUG --- stderr --- 03:00:49 DEBUG 03:00:49 INFO 03:00:49 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 03:00:49 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:00:49 INFO [loop_until]: OK (rc = 0) 03:00:49 DEBUG --- stdout --- 03:00:49 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 03:00:49 DEBUG --- stderr --- 03:00:49 DEBUG 03:00:49 INFO 03:00:49 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 03:00:49 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:00:49 INFO [loop_until]: OK (rc = 0) 03:00:49 DEBUG --- stdout --- 03:00:49 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 03:00:49 DEBUG --- stderr --- 03:00:49 DEBUG 03:00:49 INFO 03:00:49 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 03:00:49 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:00:49 INFO [loop_until]: OK (rc = 0) 03:00:49 DEBUG --- stdout --- 03:00:49 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 03:00:49 DEBUG --- stderr --- 03:00:49 DEBUG 03:00:49 INFO 03:00:49 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 03:00:49 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:00:49 INFO [loop_until]: OK (rc = 0) 03:00:49 DEBUG --- stdout --- 03:00:49 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 03:00:49 DEBUG --- stderr --- 03:00:49 DEBUG 03:00:49 INFO 03:00:49 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 03:00:49 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:00:50 INFO [loop_until]: OK (rc = 0) 03:00:50 DEBUG --- stdout --- 03:00:50 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 03:00:50 DEBUG --- stderr --- 03:00:50 DEBUG 03:00:50 INFO 03:00:50 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 03:00:50 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:00:50 INFO [loop_until]: OK (rc = 0) 03:00:50 DEBUG --- stdout --- 03:00:50 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 03:00:50 DEBUG --- stderr --- 03:00:50 DEBUG 03:00:50 INFO 03:00:50 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 03:00:50 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:00:50 INFO [loop_until]: OK (rc = 0) 03:00:50 DEBUG --- stdout --- 03:00:50 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 03:00:50 DEBUG --- stderr --- 03:00:50 DEBUG 03:00:50 INFO 03:00:50 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 03:00:50 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:00:50 INFO [loop_until]: OK (rc = 0) 03:00:50 DEBUG --- stdout --- 03:00:50 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 03:00:50 DEBUG --- stderr --- 03:00:50 DEBUG 03:00:50 INFO 03:00:50 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 03:00:50 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:00:50 INFO [loop_until]: OK (rc = 0) 03:00:50 DEBUG --- stdout --- 03:00:50 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 03:00:50 DEBUG --- stderr --- 03:00:50 DEBUG 03:00:50 INFO 03:00:50 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 03:00:50 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:00:50 INFO [loop_until]: OK (rc = 0) 03:00:50 DEBUG --- stdout --- 03:00:50 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 03:00:50 DEBUG --- stderr --- 03:00:50 DEBUG 03:00:50 INFO 03:00:50 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 03:00:50 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:00:50 INFO [loop_until]: OK (rc = 0) 03:00:50 DEBUG --- stdout --- 03:00:50 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 03:00:50 DEBUG --- stderr --- 03:00:50 DEBUG 03:00:50 INFO 03:00:50 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 03:00:50 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:00:51 INFO [loop_until]: OK (rc = 0) 03:00:51 DEBUG --- stdout --- 03:00:51 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 03:00:51 DEBUG --- stderr --- 03:00:51 DEBUG 03:00:51 INFO 03:00:51 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 03:00:51 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:00:51 INFO [loop_until]: OK (rc = 0) 03:00:51 DEBUG --- stdout --- 03:00:51 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 03:00:51 DEBUG --- stderr --- 03:00:51 DEBUG 03:00:51 INFO 03:00:51 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 03:00:51 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:00:51 INFO [loop_until]: OK (rc = 0) 03:00:51 DEBUG --- stdout --- 03:00:51 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 03:00:51 DEBUG --- stderr --- 03:00:51 DEBUG 03:00:51 INFO 03:00:51 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 03:00:51 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:00:51 INFO [loop_until]: OK (rc = 0) 03:00:51 DEBUG --- stdout --- 03:00:51 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 03:00:51 DEBUG --- stderr --- 03:00:51 DEBUG 03:00:51 INFO Initializing monitoring instance threads 03:00:51 DEBUG Monitoring instance thread list: [, , , , , , , , , , , , , , , , , , , , , , , , , , , , ] 03:00:51 INFO Starting instance threads 03:00:51 INFO 03:00:51 INFO Thread started 03:00:51 INFO [loop_until]: kubectl --namespace=xlou top node 03:00:51 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:00:51 INFO 03:00:51 INFO Thread started 03:00:51 INFO [loop_until]: kubectl --namespace=xlou top pods 03:00:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713664851" 03:00:51 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:00:51 INFO Thread started 03:00:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713664851" 03:00:51 INFO Thread started 03:00:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713664851" 03:00:51 INFO Thread started 03:00:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713664851" 03:00:51 INFO Thread started 03:00:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713664851" 03:00:51 INFO Thread started 03:00:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713664851" 03:00:51 INFO Thread started 03:00:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713664851" 03:00:51 INFO Thread started 03:00:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713664851" 03:00:51 INFO Thread started 03:00:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713664851" 03:00:51 INFO Thread started 03:00:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713664851" 03:00:51 INFO Thread started 03:00:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713664851" 03:00:51 INFO [http_cmd]: http status code OK 03:00:51 INFO [http_cmd]: http status code OK 03:00:51 INFO [http_cmd]: http status code OK 03:00:51 INFO Thread started 03:00:51 INFO [http_cmd]: http status code OK 03:00:51 DEBUG --- status code --- 03:00:51 DEBUG --- status code --- 03:00:51 DEBUG --- status code --- 03:00:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713664851" 03:00:51 DEBUG --- status code --- 03:00:51 INFO Thread started 03:00:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:00:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:00:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:00:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:00:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713664851" 03:00:51 INFO [http_cmd]: http status code OK 03:00:51 DEBUG --- http response --- 03:00:51 INFO [http_cmd]: http status code OK 03:00:51 INFO Thread started 03:00:51 INFO [http_cmd]: http status code OK 03:00:51 DEBUG --- http response --- 03:00:51 DEBUG --- http response --- 03:00:51 DEBUG --- http response --- 03:00:51 DEBUG --- status code --- 03:00:51 INFO [http_cmd]: http status code OK 03:00:51 INFO [http_cmd]: http status code OK 03:00:51 INFO [http_cmd]: http status code OK 03:00:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:00:51 DEBUG --- status code --- 03:00:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713664851" 03:00:51 DEBUG --- status code --- 03:00:51 INFO Thread started 03:00:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:00:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:00:51 INFO [http_cmd]: http status code OK 03:00:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:00:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:00:51 DEBUG --- status code --- 03:00:51 DEBUG --- status code --- 03:00:51 DEBUG --- status code --- 03:00:51 DEBUG ----- output ----- 03:00:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:00:51 INFO [http_cmd]: http status code OK 03:00:51 INFO [http_cmd]: http status code OK 03:00:51 DEBUG --- status code --- 03:00:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713664851" 03:00:51 DEBUG ----- output ----- 03:00:51 INFO Thread started 03:00:51 DEBUG ----- output ----- 03:00:51 DEBUG --- status code --- 03:00:51 DEBUG ----- output ----- 03:00:51 DEBUG --- http response --- 03:00:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:00:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:00:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:00:51 DEBUG { 03:00:51 DEBUG --- http response --- 03:00:51 DEBUG --- status code --- 03:00:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:00:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:00:51 DEBUG { 03:00:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713664851" 03:00:51 DEBUG { 03:00:51 INFO Thread started 03:00:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:00:51 DEBUG { 03:00:51 DEBUG --- http response --- 03:00:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:00:51 DEBUG --- http response --- 03:00:51 DEBUG --- http response --- 03:00:51 DEBUG "status": "success", 03:00:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:00:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:00:51 DEBUG --- http response --- 03:00:51 DEBUG --- http response --- 03:00:51 DEBUG "status": "success", 03:00:51 DEBUG "status": "success", 03:00:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713664851" 03:00:51 DEBUG --- http response --- 03:00:51 INFO Thread started 03:00:51 DEBUG "status": "success", 03:00:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:00:51 DEBUG ----- output ----- 03:00:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:00:51 DEBUG "data": { 03:00:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:00:51 DEBUG --- http response --- 03:00:51 DEBUG ----- output ----- 03:00:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:00:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:00:51 DEBUG "data": { 03:00:51 DEBUG "data": { 03:00:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:00:51 DEBUG "data": { 03:00:51 INFO [http_cmd]: http status code OK 03:00:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713664851" 03:00:51 INFO Thread started 03:00:51 DEBUG ----- output ----- 03:00:51 DEBUG { 03:00:51 DEBUG ----- output ----- 03:00:51 DEBUG "resultType": "vector", 03:00:51 DEBUG ----- output ----- 03:00:51 DEBUG { 03:00:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:00:51 DEBUG ----- output ----- 03:00:51 INFO [http_cmd]: http status code OK 03:00:51 DEBUG ----- output ----- 03:00:51 DEBUG "resultType": "vector", 03:00:51 DEBUG "resultType": "vector", 03:00:51 DEBUG ----- output ----- 03:00:51 DEBUG "resultType": "vector", 03:00:51 DEBUG --- status code --- 03:00:51 DEBUG { 03:00:51 DEBUG "status": "success", 03:00:51 INFO [http_cmd]: http status code OK 03:00:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713664851" 03:00:51 DEBUG { 03:00:51 INFO Thread started 03:00:51 DEBUG "result": [ 03:00:51 DEBUG { 03:00:51 DEBUG "status": "success", 03:00:51 DEBUG ----- output ----- 03:00:51 DEBUG { 03:00:51 DEBUG --- status code --- 03:00:51 DEBUG { 03:00:51 DEBUG "result": [ 03:00:51 DEBUG "result": [ 03:00:51 DEBUG { 03:00:51 DEBUG "result": [ 03:00:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:00:51 DEBUG --- http response --- 03:00:51 DEBUG "status": "success", 03:00:51 DEBUG --- status code --- 03:00:51 DEBUG "status": "success", 03:00:51 DEBUG { 03:00:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713664851" 03:00:51 INFO Thread started 03:00:51 DEBUG "status": "success", 03:00:51 DEBUG "data": { 03:00:51 DEBUG { 03:00:51 DEBUG "status": "success", 03:00:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:00:51 DEBUG "status": "success", 03:00:51 DEBUG { 03:00:51 DEBUG { 03:00:51 DEBUG "status": "success", 03:00:51 DEBUG { 03:00:51 DEBUG "data": { 03:00:51 DEBUG "data": { 03:00:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:00:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:00:51 DEBUG "data": { 03:00:51 DEBUG "metric": { 03:00:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713664851" 03:00:51 DEBUG "data": { 03:00:51 INFO Thread started 03:00:51 DEBUG "resultType": "vector", 03:00:51 DEBUG "status": "success", 03:00:51 DEBUG "data": { 03:00:51 DEBUG --- http response --- 03:00:51 DEBUG "data": { 03:00:51 DEBUG "metric": { 03:00:51 DEBUG "metric": { 03:00:51 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:00:51 DEBUG "metric": { 03:00:51 DEBUG "resultType": "vector", 03:00:51 DEBUG "resultType": "vector", 03:00:51 DEBUG ----- output ----- 03:00:51 DEBUG --- http response --- 03:00:51 DEBUG "resultType": "vector", 03:00:51 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:00:51 DEBUG "resultType": "vector", 03:00:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713664851" 03:00:51 DEBUG "result": [ 03:00:51 INFO Thread started 03:00:51 DEBUG "data": { 03:00:51 INFO [loop_until]: OK (rc = 0) 03:00:51 INFO [http_cmd]: http status code OK 03:00:51 INFO [http_cmd]: http status code OK 03:00:51 DEBUG "resultType": "vector", 03:00:51 INFO [http_cmd]: http status code OK 03:00:51 DEBUG "resultType": "vector", 03:00:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:00:51 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:00:51 DEBUG "data": { 03:00:51 DEBUG }, 03:00:51 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:00:51 DEBUG "result": [ 03:00:51 DEBUG "result": [ 03:00:51 DEBUG { 03:00:51 DEBUG "result": [ 03:00:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:00:51 DEBUG }, 03:00:51 DEBUG "result": [ 03:00:51 DEBUG { 03:00:51 INFO [http_cmd]: http status code OK 03:00:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713664851" 03:00:51 DEBUG "resultType": "vector", 03:00:51 INFO Thread started 03:00:51 DEBUG --- stdout --- 03:00:51 DEBUG --- status code --- 03:00:51 DEBUG --- status code --- 03:00:51 DEBUG "result": [ 03:00:51 DEBUG --- status code --- 03:00:51 DEBUG "result": [ 03:00:51 DEBUG ----- output ----- 03:00:51 DEBUG }, 03:00:51 DEBUG "resultType": "vector", 03:00:51 DEBUG "value": [ 03:00:51 DEBUG }, 03:00:51 DEBUG { 03:00:51 DEBUG { 03:00:51 DEBUG "status": "success", 03:00:51 DEBUG "data": { 03:00:51 DEBUG { 03:00:51 DEBUG "value": [ 03:00:51 DEBUG { 03:00:51 DEBUG "metric": { 03:00:51 DEBUG --- status code --- 03:00:51 DEBUG "result": [ 03:00:51 INFO [http_cmd]: http status code OK 03:00:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713664851" 03:00:51 INFO Thread started 03:00:51 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 71m 0% 6418Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 75m 0% 7740Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 58m 0% 11468Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 85m 0% 3396Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 71m 0% 5426Mi 4% gke-xlou-cdm-default-pool-1b0769c8-wpwl 163m 0% 5271Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 72m 0% 11438Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 78m 0% 3397Mi 3% gke-xlou-cdm-ds-0f601751-hmm7 65m 0% 10813Mi 9% gke-xlou-cdm-ds-0f601751-j488 64m 0% 7346Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 72m 0% 17453Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 96m 0% 7993Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 74m 0% 14592Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 69m 0% 10410Mi 17% 03:00:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:00:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:00:51 DEBUG { 03:00:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:00:51 DEBUG { 03:00:51 DEBUG { 03:00:51 DEBUG "value": [ 03:00:51 DEBUG "result": [ 03:00:51 DEBUG 03:00:51 DEBUG "value": [ 03:00:51 DEBUG "metric": { 03:00:51 DEBUG "metric": { 03:00:51 DEBUG ----- output ----- 03:00:51 DEBUG "resultType": "vector", 03:00:51 DEBUG "metric": { 03:00:51 DEBUG 03:00:51 DEBUG "metric": { 03:00:51 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:00:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:00:51 DEBUG { 03:00:51 DEBUG --- status code --- 03:00:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713664851" 03:00:51 INFO Thread started 03:00:51 DEBUG --- stderr --- 03:00:51 DEBUG --- http response --- 03:00:51 DEBUG --- http response --- 03:00:51 DEBUG "metric": { 03:00:51 DEBUG --- http response --- 03:00:51 DEBUG "metric": { 03:00:51 DEBUG "status": "success", 03:00:51 DEBUG 03:00:51 DEBUG { 03:00:51 DEBUG 03:00:51 DEBUG 03:00:51 INFO [http_cmd]: http status code OK 03:00:51 INFO [http_cmd]: http status code OK 03:00:51 DEBUG "instance": "10.142.0.107:9100" 03:00:51 DEBUG "pod": "am-767f6b67b5-9zw22" 03:00:51 DEBUG { 03:00:51 DEBUG "result": [ 03:00:51 DEBUG "instance": "10.142.0.107:9100" 03:00:51 DEBUG 03:00:51 DEBUG "instance": "10.142.0.107:9100" 03:00:51 DEBUG }, 03:00:51 DEBUG --- http response --- 03:00:51 DEBUG "metric": { 03:00:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:00:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713664851" 03:00:51 INFO Thread started 03:00:51 DEBUG 03:00:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:00:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:00:51 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:00:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:00:51 DEBUG "pod": "ds-cts-0" 03:00:51 DEBUG "data": { 03:00:51 DEBUG 03:00:51 DEBUG "metric": { 03:00:51 DEBUG 03:00:51 DEBUG 03:00:51 DEBUG --- status code --- 03:00:51 DEBUG --- status code --- 03:00:51 DEBUG }, 03:00:51 DEBUG }, 03:00:51 DEBUG "status": "success", 03:00:51 DEBUG { 03:00:51 DEBUG }, 03:00:51 DEBUG 03:00:51 DEBUG }, 03:00:51 DEBUG "value": [ 03:00:51 DEBUG "pod": "ds-idrepo-0" 03:00:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:00:51 DEBUG --- http response --- 03:00:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713664851" 03:00:51 INFO Thread started 03:00:51 DEBUG ----- output ----- 03:00:51 DEBUG ----- output ----- 03:00:51 DEBUG }, 03:00:51 DEBUG ----- output ----- 03:00:51 DEBUG }, 03:00:51 DEBUG "resultType": "vector", 03:00:51 DEBUG "result": [ 03:00:51 DEBUG 03:00:51 DEBUG [...] 03:00:51 DEBUG 03:00:51 ERROR [http_cmd]: ERROR 03:00:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:00:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:00:51 DEBUG "value": [ 03:00:51 DEBUG "value": [ 03:00:51 DEBUG "data": { 03:00:51 DEBUG "metric": { 03:00:51 DEBUG "value": [ 03:00:51 DEBUG [...] 03:00:51 DEBUG "value": [ 03:00:51 DEBUG 03:00:51 DEBUG }, 03:00:51 DEBUG ----- output ----- 03:00:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:00:51 INFO All threads has been started 127.0.0.1 - - [21/Apr/2024 03:00:51] "GET /monitoring/start HTTP/1.1" 200 - 03:00:51 DEBUG { 03:00:51 DEBUG { 03:00:51 INFO [http_cmd]: http status code OK 03:00:51 DEBUG "value": [ 03:00:51 DEBUG { 03:00:51 DEBUG "value": [ 03:00:51 DEBUG "pod": "am-767f6b67b5-9zw22" 03:00:51 DEBUG { 03:00:51 DEBUG [...] 03:00:51 DEBUG 03:00:51 DEBUG [...] 03:00:51 ERROR --- status code --- 03:00:51 DEBUG --- http response --- 03:00:51 DEBUG --- http response --- 03:00:51 DEBUG 03:00:51 DEBUG 03:00:51 DEBUG "resultType": "vector", 03:00:51 DEBUG "pod": "ds-idrepo-0" 03:00:51 DEBUG }, 03:00:51 DEBUG "value": [ 03:00:51 DEBUG 03:00:51 DEBUG 03:00:51 DEBUG "value": [ 03:00:51 DEBUG { 03:00:51 DEBUG ----- output ----- 03:00:51 DEBUG "status": "success", 03:00:51 DEBUG "status": "success", 03:00:51 DEBUG --- status code --- 03:00:51 DEBUG 03:00:51 DEBUG "status": "success", 03:00:51 DEBUG 03:00:51 DEBUG }, 03:00:51 DEBUG "metric": { 03:00:51 DEBUG "pod": "ds-idrepo-0" 03:00:51 DEBUG 03:00:51 DEBUG 03:00:51 ERROR http status code is 400 (expected [200, 401, 403, 404]) 03:00:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:00:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:00:51 DEBUG 03:00:51 DEBUG 03:00:51 DEBUG "result": [ 03:00:51 DEBUG 03:00:51 DEBUG 03:00:51 DEBUG 03:00:51 DEBUG 03:00:51 INFO [http_cmd]: http status code OK 03:00:51 DEBUG 03:00:51 DEBUG 03:00:51 DEBUG "status": "success", 03:00:51 DEBUG { 03:00:51 DEBUG "status": "success", 03:00:51 DEBUG "data": { 03:00:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:00:51 DEBUG 03:00:51 DEBUG 03:00:51 DEBUG 03:00:51 DEBUG "value": [ 03:00:51 DEBUG }, 03:00:51 DEBUG 03:00:51 DEBUG 03:00:51 DEBUG 03:00:51 ERROR --- http response --- 03:00:51 DEBUG ----- output ----- 03:00:51 DEBUG ----- output ----- 03:00:51 DEBUG 03:00:51 DEBUG 03:00:51 DEBUG { 03:00:51 DEBUG 03:00:51 DEBUG 03:00:51 DEBUG 03:00:51 DEBUG 03:00:51 DEBUG --- status code --- 03:00:51 DEBUG [...] 03:00:51 DEBUG 03:00:51 DEBUG "data": { 03:00:51 DEBUG "data": { 03:00:51 DEBUG "resultType": "vector", 03:00:51 DEBUG "resultType": "vector", 03:00:51 DEBUG --- http response --- 03:00:51 DEBUG "data": { 03:00:51 DEBUG [...] 03:00:51 DEBUG 03:00:51 DEBUG 03:00:51 DEBUG "value": [ 03:00:51 DEBUG 03:00:51 DEBUG 03:00:51 DEBUG 03:00:51 DEBUG { 03:00:51 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 03:00:51 DEBUG { 03:00:51 DEBUG [...] 03:00:51 DEBUG [...] 03:00:51 DEBUG "metric": { 03:00:51 DEBUG 03:00:51 DEBUG 03:00:51 DEBUG 03:00:51 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:00:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:00:51 DEBUG 03:00:51 DEBUG 03:00:51 DEBUG "resultType": "vector", 03:00:51 INFO [http_cmd]: http status code OK 03:00:51 DEBUG "data": { 03:00:51 DEBUG "result": [ 03:00:51 DEBUG "result": [ 03:00:51 DEBUG "resultType": "vector", 03:00:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:00:51 DEBUG 03:00:51 DEBUG [...] 03:00:51 DEBUG 03:00:51 DEBUG 03:00:51 DEBUG 03:00:51 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:00:51 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:00:51 DEBUG "status": "success", 03:00:51 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 03:00:51 DEBUG "status": "success", 03:00:51 DEBUG "data": { 03:00:51 DEBUG 03:00:51 DEBUG "pod": "am-767f6b67b5-9zw22" 03:00:51 DEBUG [...] 03:00:51 DEBUG [...] 03:00:51 DEBUG [...] 03:00:51 DEBUG }, 03:00:51 DEBUG --- http response --- 03:00:51 DEBUG 03:00:51 DEBUG [...] 03:00:51 DEBUG "result": [ 03:00:51 DEBUG --- status code --- 03:00:51 DEBUG "resultType": "vector", 03:00:51 DEBUG { 03:00:51 DEBUG { 03:00:51 DEBUG "metric": { 03:00:51 DEBUG "result": [ 03:00:51 DEBUG 03:00:51 DEBUG 03:00:51 DEBUG 03:00:51 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:00:51 DEBUG 03:00:51 DEBUG }, 03:00:51 DEBUG }, 03:00:51 DEBUG "data": { 03:00:51 DEBUG 03:00:51 DEBUG "resultType": "vector", 03:00:51 DEBUG 03:00:51 DEBUG }, 03:00:51 DEBUG 03:00:51 DEBUG 03:00:51 DEBUG 03:00:51 DEBUG "value": [ 03:00:51 DEBUG 03:00:51 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 03:00:51 DEBUG 03:00:51 DEBUG { 03:00:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:00:51 DEBUG --- http response --- 03:00:51 DEBUG "metric": { 03:00:51 DEBUG "pod": "am-767f6b67b5-9zw22" 03:00:51 DEBUG }, 03:00:51 DEBUG { 03:00:51 DEBUG 03:00:51 DEBUG 03:00:51 DEBUG [...] 03:00:51 DEBUG 03:00:51 DEBUG [...] 03:00:51 DEBUG "value": [ 03:00:51 DEBUG "value": [ 03:00:51 DEBUG "resultType": "vector", 03:00:51 DEBUG 03:00:51 DEBUG "result": [ 03:00:51 DEBUG 03:00:51 DEBUG "value": [ 03:00:51 DEBUG 03:00:51 DEBUG 03:00:51 DEBUG 03:00:51 DEBUG 1713664851, 03:00:51 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:00:51 DEBUG 03:00:51 DEBUG "metric": { 03:00:51 DEBUG "result": [ 03:00:51 DEBUG ----- output ----- 03:00:51 DEBUG "pod": "am-767f6b67b5-9zw22" 03:00:51 DEBUG "value": [ 03:00:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:00:51 DEBUG "metric": { 03:00:51 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:00:51 DEBUG 03:00:51 DEBUG }, 03:00:51 DEBUG 03:00:51 DEBUG 03:00:51 DEBUG 1713664851, 03:00:51 DEBUG 1713664851, 03:00:51 DEBUG "result": [ 03:00:51 DEBUG 03:00:51 DEBUG { 03:00:51 DEBUG "pod": "am-767f6b67b5-vbltz" 03:00:51 DEBUG 03:00:51 DEBUG 03:00:51 DEBUG 03:00:51 DEBUG 03:00:51 DEBUG "0" 03:00:51 DEBUG }, 03:00:51 DEBUG 03:00:51 DEBUG "pod": "ds-cts-0" 03:00:51 DEBUG { 03:00:51 DEBUG "metric": { 03:00:51 DEBUG { 03:00:51 DEBUG 03:00:51 DEBUG ----- output ----- 03:00:51 DEBUG "pod": "ds-cts-0" 03:00:51 DEBUG }, 03:00:51 DEBUG "pod": "ds-cts-2" 03:00:51 DEBUG "value": [ 03:00:51 DEBUG 03:00:51 DEBUG 03:00:51 DEBUG "0" 03:00:51 DEBUG "0" 03:00:51 DEBUG { 03:00:51 DEBUG "instance": "10.142.0.43:9100" 03:00:51 DEBUG "metric": { 03:00:51 DEBUG "pod": "ds-idrepo-0" 03:00:51 DEBUG 03:00:51 DEBUG "instance": "10.142.0.43:9100" 03:00:51 DEBUG "instance": "10.142.0.43:9100" 03:00:51 DEBUG "pod": "ds-idrepo-2" 03:00:51 DEBUG ] 03:00:51 DEBUG "value": [ 03:00:51 DEBUG "pod": "ds-idrepo-2" 03:00:51 DEBUG }, 03:00:51 DEBUG }, 03:00:51 DEBUG "pod": "am-767f6b67b5-9zw22" 03:00:51 DEBUG "status": "success", 03:00:51 DEBUG 03:00:51 DEBUG { 03:00:51 DEBUG }, 03:00:51 DEBUG "value": [ 03:00:51 DEBUG }, 03:00:51 DEBUG 1713664851, 03:00:51 DEBUG "pod": "am-767f6b67b5-vbltz" 03:00:51 DEBUG 03:00:51 DEBUG "pod": "ds-idrepo-2" 03:00:51 DEBUG ] 03:00:51 DEBUG "metric": { 03:00:51 DEBUG }, 03:00:51 DEBUG }, 03:00:51 DEBUG }, 03:00:51 DEBUG 03:00:51 DEBUG }, 03:00:51 DEBUG }, 03:00:51 DEBUG }, 03:00:51 DEBUG } 03:00:51 DEBUG 1713664851, 03:00:51 DEBUG }, 03:00:51 DEBUG "value": [ 03:00:51 DEBUG "value": [ 03:00:51 DEBUG }, 03:00:51 DEBUG "data": { 03:00:51 DEBUG 03:00:51 DEBUG "status": "success", 03:00:51 DEBUG "value": [ 03:00:51 DEBUG 1713664851, 03:00:51 DEBUG "value": [ 03:00:51 DEBUG "0.0003136137516254878" 03:00:51 DEBUG }, 03:00:51 DEBUG ] 03:00:51 DEBUG }, 03:00:51 DEBUG "value": [ 03:00:51 DEBUG 1713664851, 03:00:51 DEBUG "value": [ 03:00:51 DEBUG "value": [ 03:00:51 DEBUG "value": [ 03:00:51 DEBUG [...] 03:00:51 DEBUG "value": [ 03:00:51 DEBUG "value": [ 03:00:51 DEBUG "value": [ 03:00:51 DEBUG ] 03:00:51 DEBUG "115855360" 03:00:51 DEBUG "value": [ 03:00:51 DEBUG 03:00:51 DEBUG 03:00:51 DEBUG 03:00:51 DEBUG "value": [ 03:00:51 DEBUG [...] 03:00:51 DEBUG "data": { 03:00:51 DEBUG 03:00:51 DEBUG "0.036520981674393126" 03:00:51 DEBUG 1713664851, 03:00:51 DEBUG ] 03:00:51 DEBUG "value": [ 03:00:51 DEBUG } 03:00:51 DEBUG } 03:00:51 DEBUG "pod": "am-767f6b67b5-9zw22" 03:00:51 DEBUG "0" 03:00:51 DEBUG 1713664851, 03:00:51 DEBUG 1713664851, 03:00:51 DEBUG 03:00:51 DEBUG 03:00:51 DEBUG 1713664851, 03:00:51 DEBUG 1713664851, 03:00:51 DEBUG 1713664851, 03:00:51 DEBUG } 03:00:51 DEBUG ] 03:00:51 DEBUG 1713664851, 03:00:51 DEBUG 03:00:51 DEBUG 03:00:51 DEBUG [...] 03:00:51 DEBUG 03:00:51 DEBUG 03:00:51 DEBUG "resultType": "vector", 03:00:51 DEBUG 03:00:51 DEBUG ] 03:00:51 DEBUG "0" 03:00:51 DEBUG } 03:00:51 DEBUG 1713664851, 03:00:51 DEBUG ] 03:00:51 DEBUG ] 03:00:51 DEBUG }, 03:00:51 DEBUG ] 03:00:51 DEBUG "0.0003333333333330302" 03:00:51 DEBUG "0" 03:00:51 DEBUG 03:00:51 DEBUG 03:00:51 DEBUG "28612.333333333336" 03:00:51 DEBUG "34268.933333333334" 03:00:51 DEBUG "0" 03:00:51 DEBUG } 03:00:51 DEBUG } 03:00:51 DEBUG "0" 03:00:51 DEBUG "resultType": "vector", 03:00:51 DEBUG 03:00:51 DEBUG 03:00:51 DEBUG 03:00:51 DEBUG 03:00:51 DEBUG "result": [ 03:00:51 DEBUG 03:00:51 DEBUG } 03:00:51 DEBUG ] 03:00:51 DEBUG ] 03:00:51 DEBUG "0" 03:00:51 DEBUG } 03:00:51 DEBUG } 03:00:51 DEBUG "value": [ 03:00:51 DEBUG } 03:00:51 DEBUG ] 03:00:51 DEBUG } 03:00:51 DEBUG 03:00:51 DEBUG 03:00:51 DEBUG ] 03:00:51 DEBUG ] 03:00:51 DEBUG ] 03:00:51 DEBUG ] 03:00:51 DEBUG ] 03:00:51 DEBUG "result": [ 03:00:51 DEBUG [...] 03:00:51 DEBUG 03:00:51 DEBUG 03:00:51 DEBUG 03:00:51 DEBUG { 03:00:51 DEBUG [...] 03:00:51 DEBUG ] 03:00:51 DEBUG } 03:00:51 DEBUG } 03:00:51 DEBUG ] 03:00:51 DEBUG } 03:00:51 DEBUG } 03:00:51 DEBUG 03:00:51 DEBUG ] 03:00:51 DEBUG ] 03:00:51 DEBUG ] 03:00:51 DEBUG [...] 03:00:51 DEBUG "pod": "am-767f6b67b5-vbltz" 03:00:51 DEBUG } 03:00:51 DEBUG } 03:00:51 DEBUG } 03:00:51 DEBUG } 03:00:51 DEBUG } 03:00:51 DEBUG { 03:00:51 DEBUG 03:00:51 DEBUG 03:00:51 DEBUG "pod": "ds-cts-2" 03:00:51 DEBUG [...] 03:00:51 DEBUG "metric": { 03:00:51 DEBUG 03:00:51 DEBUG 03:00:51 DEBUG ] 03:00:51 DEBUG } 03:00:51 DEBUG } 03:00:51 DEBUG 03:00:51 DEBUG } 03:00:51 DEBUG } 03:00:51 DEBUG } 03:00:51 DEBUG 03:00:51 DEBUG }, 03:00:51 DEBUG ] 03:00:51 DEBUG ] 03:00:51 DEBUG ] 03:00:51 DEBUG } 03:00:51 DEBUG ] 03:00:51 DEBUG "metric": { 03:00:51 DEBUG 03:00:51 DEBUG "pod": "am-767f6b67b5-vbltz" 03:00:51 DEBUG }, 03:00:51 DEBUG 03:00:51 DEBUG "instance": "10.142.0.107:9100" 03:00:51 DEBUG } 03:00:51 DEBUG 03:00:51 DEBUG } 03:00:51 DEBUG ] 03:00:51 DEBUG 03:00:51 DEBUG } 03:00:51 DEBUG ] 03:00:51 DEBUG } 03:00:51 DEBUG 03:00:51 DEBUG "value": [ 03:00:51 DEBUG } 03:00:51 DEBUG } 03:00:51 DEBUG } 03:00:51 DEBUG } 03:00:51 DEBUG } 03:00:51 DEBUG 03:00:51 DEBUG }, 03:00:51 DEBUG "value": [ 03:00:51 DEBUG 03:00:51 DEBUG }, 03:00:51 DEBUG } 03:00:51 DEBUG "pod": "ds-cts-2" 03:00:51 DEBUG } 03:00:51 DEBUG } 03:00:51 DEBUG [...] 03:00:51 DEBUG } 03:00:51 DEBUG 03:00:51 DEBUG "pod": "ds-cts-2" 03:00:51 DEBUG } 03:00:51 DEBUG } 03:00:51 DEBUG } 03:00:51 DEBUG "instance": "10.142.0.107:9100" 03:00:51 DEBUG "pod": "am-767f6b67b5-vbltz" 03:00:51 DEBUG "value": [ 03:00:51 DEBUG 1713664851, 03:00:51 DEBUG 03:00:51 DEBUG "value": [ 03:00:51 DEBUG }, 03:00:51 DEBUG } 03:00:51 DEBUG 03:00:51 DEBUG } 03:00:51 DEBUG 1713664851, 03:00:51 DEBUG }, 03:00:51 DEBUG }, 03:00:51 DEBUG }, 03:00:51 DEBUG 1713664851, 03:00:51 DEBUG "0" 03:00:51 DEBUG "pod": "am-767f6b67b5-vbltz" 03:00:51 DEBUG 03:00:51 DEBUG "value": [ 03:00:51 DEBUG 03:00:51 DEBUG "0" 03:00:51 DEBUG "value": [ 03:00:51 DEBUG "value": [ 03:00:51 DEBUG "value": [ 03:00:51 DEBUG "1" 03:00:51 DEBUG ] 03:00:51 DEBUG }, 03:00:51 DEBUG 03:00:51 DEBUG 1713664851, 03:00:51 DEBUG 03:00:51 DEBUG ] 03:00:51 DEBUG 1713664851, 03:00:51 DEBUG 03:00:51 DEBUG 1713664851, 03:00:51 DEBUG ] 03:00:51 DEBUG } 03:00:51 DEBUG "value": [ 03:00:51 DEBUG 03:00:51 DEBUG "0.03333333333333333" 03:00:51 DEBUG "pod": "am-767f6b67b5-vbltz" 03:00:51 DEBUG } 03:00:51 DEBUG "0" 03:00:51 DEBUG 03:00:51 DEBUG "0" 03:00:51 DEBUG } 03:00:51 DEBUG ] 03:00:51 DEBUG 1713664851, 03:00:51 DEBUG [...] 03:00:51 DEBUG ] 03:00:51 DEBUG }, 03:00:51 DEBUG ] 03:00:51 DEBUG ] 03:00:51 DEBUG 03:00:51 DEBUG ] 03:00:51 DEBUG ] 03:00:51 DEBUG } 03:00:51 DEBUG "0" 03:00:51 DEBUG 03:00:51 DEBUG } 03:00:51 DEBUG "value": [ 03:00:51 DEBUG } 03:00:51 DEBUG } 03:00:51 DEBUG [...] 03:00:51 DEBUG } 03:00:51 DEBUG } 03:00:51 DEBUG } 03:00:51 DEBUG ] 03:00:51 DEBUG 03:00:51 DEBUG ] 03:00:51 DEBUG 1713664851, 03:00:51 DEBUG } 03:00:51 DEBUG ] 03:00:51 DEBUG 03:00:51 DEBUG ] 03:00:51 DEBUG } 03:00:51 DEBUG } 03:00:51 DEBUG 03:00:51 DEBUG } 03:00:51 DEBUG "149.887" 03:00:51 DEBUG } 03:00:51 DEBUG } 03:00:51 DEBUG } 03:00:51 DEBUG ] 03:00:51 DEBUG "instance": "10.142.0.43:9100" 03:00:51 DEBUG } 03:00:51 DEBUG ] 03:00:51 DEBUG 03:00:51 DEBUG } 03:00:51 DEBUG } 03:00:51 DEBUG }, 03:00:51 DEBUG } 03:00:51 DEBUG 03:00:51 DEBUG } 03:00:51 DEBUG "value": [ 03:00:51 DEBUG ] 03:00:51 DEBUG "instance": "10.142.0.43:9100" 03:00:51 DEBUG 1713664851, 03:00:51 DEBUG } 03:00:51 DEBUG }, 03:00:51 DEBUG "0.0007333333333463089" 03:00:51 DEBUG } 03:00:51 DEBUG "value": [ 03:00:51 DEBUG ] 03:00:51 DEBUG 1713664851, 03:00:51 DEBUG } 03:00:51 DEBUG "4312442399744" 03:00:51 DEBUG ] 03:00:51 DEBUG ] 03:00:51 DEBUG } 03:00:51 DEBUG } 03:00:51 DEBUG } 03:00:51 DEBUG ] 03:00:51 DEBUG } 03:00:51 DEBUG } 03:00:51 INFO [loop_until]: OK (rc = 0) 03:00:51 DEBUG --- stdout --- 03:00:51 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 6m 8708Mi am-767f6b67b5-qcqz2 7m 8533Mi am-767f6b67b5-vbltz 8m 8608Mi ds-cts-0 11m 2142Mi ds-cts-1 13m 12329Mi ds-cts-2 12m 11880Mi ds-idrepo-0 13m 3928Mi ds-idrepo-1 13m 3953Mi ds-idrepo-2 13m 3913Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1179Mi idm-b85f9d978-wnhxb 3m 1240Mi lodemon-54c87f8cbb-jzrjs 1m 54Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-68c5d5b884-p97w5 1m 15Mi 03:00:51 DEBUG --- stderr --- 03:00:51 DEBUG 03:01:51 INFO 03:01:51 INFO [loop_until]: kubectl --namespace=xlou top node 03:01:51 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:01:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713664911" 03:01:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713664911" 03:01:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713664911" 03:01:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713664911" 03:01:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713664911" 03:01:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713664911" 03:01:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713664911" 03:01:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713664911" 03:01:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713664911" 03:01:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713664911" 03:01:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713664911" 03:01:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713664911" 03:01:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713664911" 03:01:51 INFO [http_cmd]: http status code OK 03:01:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713664911" 03:01:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713664911" 03:01:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713664911" 03:01:51 INFO [http_cmd]: http status code OK 03:01:51 DEBUG --- status code --- 03:01:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713664911" 03:01:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713664911" 03:01:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713664911" 03:01:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713664911" 03:01:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713664911" 03:01:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713664911" 03:01:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713664911" 03:01:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713664911" 03:01:51 ERROR [http_cmd]: ERROR 03:01:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713664911" 03:01:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713664911" 03:01:51 INFO [http_cmd]: http status code OK 03:01:51 DEBUG --- status code --- 03:01:51 INFO [http_cmd]: http status code OK 03:01:51 INFO [http_cmd]: http status code OK 03:01:51 INFO [http_cmd]: http status code OK 03:01:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713664911" 03:01:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:01:51 INFO [http_cmd]: http status code OK 03:01:51 INFO [http_cmd]: http status code OK 03:01:51 INFO [http_cmd]: http status code OK 03:01:51 INFO [http_cmd]: http status code OK 03:01:51 INFO [http_cmd]: http status code OK 03:01:51 INFO [http_cmd]: http status code OK 03:01:51 INFO [http_cmd]: http status code OK 03:01:51 INFO [http_cmd]: http status code OK 03:01:51 ERROR --- status code --- 03:01:51 INFO [http_cmd]: http status code OK 03:01:51 INFO [http_cmd]: http status code OK 03:01:51 INFO [http_cmd]: http status code OK 03:01:51 INFO [http_cmd]: http status code OK 03:01:51 INFO [http_cmd]: http status code OK 03:01:51 INFO [http_cmd]: http status code OK 03:01:51 DEBUG --- status code --- 03:01:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:01:51 INFO [http_cmd]: http status code OK 03:01:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:01:51 INFO [loop_until]: OK (rc = 0) 03:01:51 DEBUG --- status code --- 03:01:51 INFO [http_cmd]: http status code OK 03:01:51 DEBUG --- status code --- 03:01:51 DEBUG --- status code --- 03:01:51 DEBUG --- http response --- 03:01:51 DEBUG --- status code --- 03:01:51 DEBUG --- status code --- 03:01:51 DEBUG --- status code --- 03:01:51 DEBUG --- status code --- 03:01:51 DEBUG --- status code --- 03:01:51 DEBUG --- status code --- 03:01:51 DEBUG --- status code --- 03:01:51 DEBUG --- status code --- 03:01:51 ERROR http status code is 400 (expected [200, 401, 403, 404]) 03:01:51 DEBUG --- status code --- 03:01:51 DEBUG --- status code --- 03:01:51 DEBUG --- status code --- 03:01:51 DEBUG --- status code --- 03:01:51 DEBUG --- status code --- 03:01:51 DEBUG --- status code --- 03:01:51 INFO [http_cmd]: http status code OK 03:01:51 DEBUG --- http response --- 03:01:51 DEBUG --- status code --- 03:01:51 DEBUG --- http response --- 03:01:51 DEBUG --- stdout --- 03:01:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:01:51 DEBUG --- status code --- 03:01:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:01:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:01:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:01:51 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 03:01:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:01:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:01:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:01:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:01:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:01:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:01:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:01:51 ERROR --- http response --- 03:01:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:01:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:01:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:01:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:01:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:01:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:01:51 DEBUG --- status code --- 03:01:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:01:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:01:51 INFO [http_cmd]: http status code OK 03:01:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:01:51 INFO [http_cmd]: http status code OK 03:01:51 INFO [http_cmd]: http status code OK 03:01:51 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 67m 0% 6305Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 83m 0% 7757Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 633m 2% 11496Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 84m 0% 3384Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 229m 0% 5426Mi 4% gke-xlou-cdm-default-pool-1b0769c8-wpwl 159m 0% 5272Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 323m 1% 11453Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 586m 1% 3270Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 555m 1% 10843Mi 9% gke-xlou-cdm-ds-0f601751-j488 277m 0% 7370Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 228m 0% 17405Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 406m 1% 8017Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 426m 1% 14549Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 2449m 15% 10939Mi 18% 03:01:51 DEBUG --- http response --- 03:01:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:01:51 DEBUG --- http response --- 03:01:51 DEBUG --- http response --- 03:01:51 DEBUG --- http response --- 03:01:51 DEBUG --- http response --- 03:01:51 DEBUG --- http response --- 03:01:51 DEBUG --- http response --- 03:01:51 DEBUG --- http response --- 03:01:51 DEBUG --- http response --- 03:01:51 DEBUG --- http response --- 03:01:51 DEBUG --- http response --- 03:01:51 DEBUG --- http response --- 03:01:51 DEBUG --- http response --- 03:01:51 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 03:01:51 DEBUG --- http response --- 03:01:51 DEBUG --- http response --- 03:01:51 DEBUG --- http response --- 03:01:51 DEBUG --- http response --- 03:01:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:01:51 DEBUG --- http response --- 03:01:51 DEBUG ----- output ----- 03:01:51 DEBUG --- status code --- 03:01:51 DEBUG ----- output ----- 03:01:51 DEBUG --- status code --- 03:01:51 DEBUG --- status code --- 03:01:51 DEBUG --- stderr --- 03:01:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:01:51 DEBUG --- http response --- 03:01:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:01:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:01:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:01:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:01:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:01:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:01:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:01:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:01:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:01:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:01:51 DEBUG ----- output ----- 03:01:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:01:51 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 03:01:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:01:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:01:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:01:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:01:51 DEBUG --- http response --- 03:01:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:01:51 DEBUG { 03:01:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:01:51 DEBUG { 03:01:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:01:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:01:51 DEBUG 03:01:51 DEBUG ----- output ----- 03:01:51 DEBUG ----- output ----- 03:01:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:01:51 DEBUG ----- output ----- 03:01:51 DEBUG ----- output ----- 03:01:51 DEBUG ----- output ----- 03:01:51 DEBUG ----- output ----- 03:01:51 DEBUG ----- output ----- 03:01:51 DEBUG ----- output ----- 03:01:51 DEBUG ----- output ----- 03:01:51 DEBUG ----- output ----- 03:01:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:01:51 DEBUG { 03:01:51 DEBUG ----- output ----- 03:01:51 DEBUG ----- output ----- 03:01:51 DEBUG ----- output ----- 03:01:51 DEBUG ----- output ----- 03:01:51 DEBUG ----- output ----- 03:01:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:01:51 DEBUG ----- output ----- 03:01:51 DEBUG "status": "success", 03:01:51 DEBUG --- http response --- 03:01:51 DEBUG "status": "success", 03:01:51 DEBUG --- http response --- 03:01:51 DEBUG --- http response --- 03:01:51 DEBUG { 03:01:51 DEBUG { 03:01:51 DEBUG ----- output ----- 03:01:51 DEBUG { 03:01:51 DEBUG { 03:01:51 DEBUG { 03:01:51 DEBUG { 03:01:51 DEBUG { 03:01:51 DEBUG { 03:01:51 DEBUG { 03:01:51 DEBUG { 03:01:51 DEBUG ----- output ----- 03:01:51 DEBUG "status": "success", 03:01:51 DEBUG { 03:01:51 DEBUG { 03:01:51 DEBUG { 03:01:51 DEBUG { 03:01:51 DEBUG { 03:01:51 DEBUG ----- output ----- 03:01:51 DEBUG { 03:01:51 DEBUG "data": { 03:01:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:01:51 DEBUG "data": { 03:01:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:01:51 DEBUG "status": "success", 03:01:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:01:51 DEBUG "status": "success", 03:01:51 DEBUG { 03:01:51 DEBUG "status": "success", 03:01:51 DEBUG "status": "success", 03:01:51 DEBUG "status": "success", 03:01:51 DEBUG "status": "success", 03:01:51 DEBUG "status": "success", 03:01:51 DEBUG "status": "success", 03:01:51 DEBUG "status": "success", 03:01:51 DEBUG "status": "success", 03:01:51 DEBUG { 03:01:51 DEBUG "data": { 03:01:51 DEBUG "status": "success", 03:01:51 DEBUG "status": "success", 03:01:51 DEBUG "status": "success", 03:01:51 DEBUG "status": "success", 03:01:51 DEBUG "status": "success", 03:01:51 DEBUG { 03:01:51 DEBUG "status": "success", 03:01:51 DEBUG "resultType": "vector", 03:01:51 DEBUG ----- output ----- 03:01:51 DEBUG "resultType": "vector", 03:01:51 DEBUG ----- output ----- 03:01:51 DEBUG "data": { 03:01:51 DEBUG ----- output ----- 03:01:51 DEBUG "data": { 03:01:51 DEBUG "status": "success", 03:01:51 DEBUG "data": { 03:01:51 DEBUG "data": { 03:01:51 DEBUG "data": { 03:01:51 DEBUG "data": { 03:01:51 DEBUG "data": { 03:01:51 DEBUG "data": { 03:01:51 DEBUG "data": { 03:01:51 DEBUG "data": { 03:01:51 DEBUG "status": "success", 03:01:51 DEBUG "resultType": "vector", 03:01:51 DEBUG "data": { 03:01:51 DEBUG "resultType": "vector", 03:01:51 DEBUG "result": [ 03:01:51 DEBUG "data": { 03:01:51 DEBUG "data": { 03:01:51 DEBUG "status": "success", 03:01:51 DEBUG "data": { 03:01:51 DEBUG "result": [ 03:01:51 DEBUG { 03:01:51 DEBUG "result": [ 03:01:51 DEBUG { 03:01:51 DEBUG "resultType": "vector", 03:01:51 DEBUG { 03:01:51 DEBUG "resultType": "vector", 03:01:51 DEBUG "data": { 03:01:51 DEBUG "resultType": "vector", 03:01:51 DEBUG "resultType": "vector", 03:01:51 DEBUG "resultType": "vector", 03:01:51 DEBUG "resultType": "vector", 03:01:51 DEBUG "resultType": "vector", 03:01:51 DEBUG "resultType": "vector", 03:01:51 DEBUG "resultType": "vector", 03:01:51 DEBUG "resultType": "vector", 03:01:51 DEBUG "data": { 03:01:51 DEBUG "result": [ 03:01:51 DEBUG { 03:01:51 DEBUG "data": { 03:01:51 DEBUG "data": { 03:01:51 DEBUG "resultType": "vector", 03:01:51 DEBUG "resultType": "vector", 03:01:51 DEBUG "data": { 03:01:51 DEBUG "resultType": "vector", 03:01:51 DEBUG { 03:01:51 DEBUG "status": "success", 03:01:51 DEBUG { 03:01:51 DEBUG "status": "success", 03:01:51 DEBUG "result": [ 03:01:51 DEBUG "status": "success", 03:01:51 DEBUG "result": [ 03:01:51 DEBUG "resultType": "vector", 03:01:51 DEBUG "result": [ 03:01:51 DEBUG "result": [ 03:01:51 DEBUG "result": [ 03:01:51 DEBUG "result": [ 03:01:51 DEBUG "result": [ 03:01:51 DEBUG "result": [ 03:01:51 DEBUG "result": [ 03:01:51 DEBUG "result": [ 03:01:51 DEBUG "resultType": "vector", 03:01:51 DEBUG { 03:01:51 DEBUG "metric": { 03:01:51 DEBUG "resultType": "vector", 03:01:51 DEBUG "resultType": "vector", 03:01:51 DEBUG "result": [ 03:01:51 DEBUG "result": [ 03:01:51 DEBUG "resultType": "vector", 03:01:51 DEBUG "result": [ 03:01:51 DEBUG "metric": { 03:01:51 DEBUG "data": { 03:01:51 DEBUG "metric": { 03:01:51 DEBUG "data": { 03:01:51 DEBUG { 03:01:51 DEBUG "data": { 03:01:51 DEBUG { 03:01:51 DEBUG "result": [ 03:01:51 DEBUG { 03:01:51 DEBUG { 03:01:51 DEBUG { 03:01:51 DEBUG { 03:01:51 DEBUG { 03:01:51 DEBUG { 03:01:51 DEBUG { 03:01:51 DEBUG { 03:01:51 DEBUG "result": [ 03:01:51 DEBUG "metric": { 03:01:51 DEBUG "pod": "am-767f6b67b5-9zw22" 03:01:51 DEBUG "result": [ 03:01:51 DEBUG "result": [ 03:01:51 DEBUG { 03:01:51 DEBUG { 03:01:51 DEBUG "metric": { 03:01:51 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:01:51 DEBUG "resultType": "vector", 03:01:51 DEBUG "result": [ 03:01:51 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:01:51 DEBUG "resultType": "vector", 03:01:51 DEBUG "metric": { 03:01:51 DEBUG "resultType": "vector", 03:01:51 DEBUG "metric": { 03:01:51 DEBUG { 03:01:51 DEBUG "metric": { 03:01:51 DEBUG "metric": { 03:01:51 DEBUG "metric": { 03:01:51 DEBUG "metric": { 03:01:51 DEBUG "metric": { 03:01:51 DEBUG "metric": { 03:01:51 DEBUG "metric": { 03:01:51 DEBUG "metric": { 03:01:51 DEBUG { 03:01:51 DEBUG "instance": "10.142.0.107:9100" 03:01:51 DEBUG }, 03:01:51 DEBUG { 03:01:51 DEBUG { 03:01:51 DEBUG "metric": { 03:01:51 DEBUG "pod": "ds-cts-0" 03:01:51 DEBUG }, 03:01:51 DEBUG "result": [ 03:01:51 DEBUG { 03:01:51 DEBUG { 03:01:51 DEBUG }, 03:01:51 DEBUG "result": [ 03:01:51 DEBUG "pod": "ds-idrepo-0" 03:01:51 DEBUG "result": [ 03:01:51 DEBUG { 03:01:51 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:01:51 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:01:51 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:01:51 DEBUG "pod": "ds-idrepo-0" 03:01:51 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:01:51 DEBUG "pod": "ds-cts-0" 03:01:51 DEBUG "instance": "10.142.0.107:9100" 03:01:51 DEBUG "pod": "ds-idrepo-0" 03:01:51 DEBUG "pod": "am-767f6b67b5-9zw22" 03:01:51 DEBUG "metric": { 03:01:51 DEBUG }, 03:01:51 DEBUG "value": [ 03:01:51 DEBUG "metric": { 03:01:51 DEBUG "metric": { 03:01:51 DEBUG "pod": "am-767f6b67b5-9zw22" 03:01:51 DEBUG }, 03:01:51 DEBUG "value": [ 03:01:51 DEBUG { 03:01:51 DEBUG "metric": { 03:01:51 DEBUG "metric": { 03:01:51 DEBUG "value": [ 03:01:51 DEBUG { 03:01:51 DEBUG }, 03:01:51 DEBUG "metric": { 03:01:51 DEBUG "metric": { 03:01:51 DEBUG }, 03:01:51 DEBUG }, 03:01:51 DEBUG }, 03:01:51 DEBUG }, 03:01:51 DEBUG }, 03:01:51 DEBUG }, 03:01:51 DEBUG }, 03:01:51 DEBUG }, 03:01:51 DEBUG }, 03:01:51 DEBUG "pod": "am-767f6b67b5-9zw22" 03:01:51 DEBUG "value": [ 03:01:51 DEBUG 03:01:51 DEBUG "instance": "10.142.0.107:9100" 03:01:51 DEBUG }, 03:01:51 DEBUG }, 03:01:51 DEBUG "value": [ 03:01:51 DEBUG 03:01:51 DEBUG "metric": { 03:01:51 DEBUG "instance": "10.142.0.107:9100" 03:01:51 DEBUG "instance": "10.142.0.107:9100" 03:01:51 DEBUG 03:01:51 DEBUG "metric": { 03:01:51 DEBUG "value": [ 03:01:51 DEBUG "pod": "am-767f6b67b5-9zw22" 03:01:51 DEBUG }, 03:01:51 DEBUG "value": [ 03:01:51 DEBUG "value": [ 03:01:51 DEBUG "value": [ 03:01:51 DEBUG "value": [ 03:01:51 DEBUG "value": [ 03:01:51 DEBUG "value": [ 03:01:51 DEBUG "value": [ 03:01:51 DEBUG "value": [ 03:01:51 DEBUG "value": [ 03:01:51 DEBUG }, 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG "pod": "am-767f6b67b5-9zw22" 03:01:51 DEBUG "value": [ 03:01:51 DEBUG "value": [ 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG "pod": "am-767f6b67b5-9zw22" 03:01:51 DEBUG }, 03:01:51 DEBUG }, 03:01:51 DEBUG 03:01:51 DEBUG "pod": "ds-cts-0" 03:01:51 DEBUG 03:01:51 DEBUG "value": [ 03:01:51 DEBUG "pod": "ds-idrepo-0" 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG "value": [ 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG }, 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG }, 03:01:51 DEBUG "value": [ 03:01:51 DEBUG "value": [ 03:01:51 DEBUG 03:01:51 DEBUG }, 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG }, 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG [...] 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG [...] 03:01:51 DEBUG "value": [ 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG [...] 03:01:51 DEBUG "value": [ 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG [...] 03:01:51 DEBUG "value": [ 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG "value": [ 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG [...] 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG [...] 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG [...] 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG [...] 03:01:51 DEBUG [...] 03:01:51 DEBUG [...] 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG [...] 03:01:51 DEBUG [...] 03:01:51 DEBUG [...] 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG [...] 03:01:51 DEBUG [...] 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG [...] 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG [...] 03:01:51 DEBUG [...] 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG [...] 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG [...] 03:01:51 DEBUG [...] 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG "pod": "am-767f6b67b5-vbltz" 03:01:51 DEBUG [...] 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:01:51 DEBUG [...] 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:01:51 DEBUG [...] 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG [...] 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG "pod": "ds-idrepo-2" 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG "instance": "10.142.0.43:9100" 03:01:51 DEBUG }, 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG "pod": "ds-cts-2" 03:01:51 DEBUG }, 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG }, 03:01:51 DEBUG 03:01:51 DEBUG "pod": "ds-idrepo-2" 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:01:51 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:01:51 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:01:51 DEBUG }, 03:01:51 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:01:51 DEBUG 03:01:51 DEBUG "instance": "10.142.0.43:9100" 03:01:51 DEBUG "pod": "ds-idrepo-2" 03:01:51 DEBUG "pod": "am-767f6b67b5-vbltz" 03:01:51 DEBUG 03:01:51 DEBUG }, 03:01:51 DEBUG "value": [ 03:01:51 DEBUG 03:01:51 DEBUG "instance": "10.142.0.43:9100" 03:01:51 DEBUG "pod": "am-767f6b67b5-vbltz" 03:01:51 DEBUG }, 03:01:51 DEBUG "value": [ 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG 03:01:51 DEBUG "value": [ 03:01:51 DEBUG 03:01:51 DEBUG }, 03:01:51 DEBUG "pod": "am-767f6b67b5-vbltz" 03:01:51 DEBUG 03:01:51 DEBUG }, 03:01:51 DEBUG }, 03:01:51 DEBUG }, 03:01:51 DEBUG "value": [ 03:01:51 DEBUG }, 03:01:51 DEBUG "pod": "ds-cts-2" 03:01:51 DEBUG }, 03:01:51 DEBUG }, 03:01:51 DEBUG }, 03:01:51 DEBUG "pod": "am-767f6b67b5-vbltz" 03:01:51 DEBUG "value": [ 03:01:51 DEBUG 1713664911, 03:01:51 DEBUG 03:01:51 DEBUG }, 03:01:51 DEBUG }, 03:01:51 DEBUG "value": [ 03:01:51 DEBUG "value": [ 03:01:51 DEBUG 03:01:51 DEBUG "instance": "10.142.0.43:9100" 03:01:51 DEBUG "instance": "10.142.0.43:9100" 03:01:51 DEBUG 1713664911, 03:01:51 DEBUG 03:01:51 DEBUG "value": [ 03:01:51 DEBUG }, 03:01:51 DEBUG 03:01:51 DEBUG "value": [ 03:01:51 DEBUG "value": [ 03:01:51 DEBUG "value": [ 03:01:51 DEBUG 1713664911, 03:01:51 DEBUG 1713664911, 03:01:51 DEBUG }, 03:01:51 DEBUG "value": [ 03:01:51 DEBUG "value": [ 03:01:51 DEBUG "value": [ 03:01:51 DEBUG }, 03:01:51 DEBUG 1713664911, 03:01:51 DEBUG "59.833333333333336" 03:01:51 DEBUG "pod": "am-767f6b67b5-vbltz" 03:01:51 DEBUG "value": [ 03:01:51 DEBUG 1713664911, 03:01:51 DEBUG 1713664911, 03:01:51 DEBUG 1713664911, 03:01:51 DEBUG "pod": "am-767f6b67b5-vbltz" 03:01:51 DEBUG }, 03:01:51 DEBUG }, 03:01:51 DEBUG "169329.68492301478" 03:01:51 DEBUG "pod": "ds-cts-2" 03:01:51 DEBUG 1713664911, 03:01:51 DEBUG "value": [ 03:01:51 DEBUG "pod": "ds-cts-2" 03:01:51 DEBUG 1713664911, 03:01:51 DEBUG 1713664911, 03:01:51 DEBUG "value": [ 03:01:51 DEBUG "6.12580643905958" 03:01:51 DEBUG "0" 03:01:51 DEBUG "value": [ 03:01:51 DEBUG 1713664911, 03:01:51 DEBUG 1713664911, 03:01:51 DEBUG 1713664911, 03:01:51 DEBUG "value": [ 03:01:51 DEBUG "23754833.9" 03:01:51 DEBUG ] 03:01:51 DEBUG }, 03:01:51 DEBUG 1713664911, 03:01:51 DEBUG "34374891.87617126" 03:01:51 DEBUG "149.888" 03:01:51 DEBUG "0" 03:01:51 DEBUG }, 03:01:51 DEBUG "value": [ 03:01:51 DEBUG 1713664911, 03:01:51 DEBUG ] 03:01:51 DEBUG }, 03:01:51 DEBUG "0" 03:01:51 DEBUG 1713664911, 03:01:51 DEBUG }, 03:01:51 DEBUG "4950.407823283789" 03:01:51 DEBUG "727089152" 03:01:51 DEBUG 1713664911, 03:01:51 DEBUG "3.1587397635415897" 03:01:51 DEBUG ] 03:01:51 DEBUG 1713664911, 03:01:51 DEBUG "0.017333333333325148" 03:01:51 DEBUG "0" 03:01:51 DEBUG "59.63333333333333" 03:01:51 DEBUG 1713664911, 03:01:51 DEBUG ] 03:01:51 DEBUG } 03:01:51 DEBUG "value": [ 03:01:51 DEBUG "23758305.333333332" 03:01:51 DEBUG ] 03:01:51 DEBUG ] 03:01:51 DEBUG ] 03:01:51 DEBUG "value": [ 03:01:51 DEBUG "value": [ 03:01:51 DEBUG "0.024000000000036912" 03:01:51 DEBUG } 03:01:51 DEBUG ] 03:01:51 DEBUG "value": [ 03:01:51 DEBUG "892.5" 03:01:51 DEBUG "value": [ 03:01:51 DEBUG ] 03:01:51 DEBUG ] 03:01:51 DEBUG ] 03:01:51 DEBUG ] 03:01:51 DEBUG } 03:01:51 DEBUG "0" 03:01:51 DEBUG ] 03:01:51 DEBUG ] 03:01:51 DEBUG ] 03:01:51 DEBUG "0" 03:01:51 DEBUG } 03:01:51 DEBUG ] 03:01:51 DEBUG 1713664911, 03:01:51 DEBUG ] 03:01:51 DEBUG } 03:01:51 DEBUG } 03:01:51 DEBUG } 03:01:51 DEBUG 1713664911, 03:01:51 DEBUG 1713664911, 03:01:51 DEBUG ] 03:01:51 DEBUG ] 03:01:51 DEBUG } 03:01:51 DEBUG 1713664911, 03:01:51 DEBUG ] 03:01:51 DEBUG 1713664911, 03:01:51 DEBUG } 03:01:51 DEBUG } 03:01:51 DEBUG } 03:01:51 DEBUG } 03:01:51 DEBUG ] 03:01:51 DEBUG ] 03:01:51 DEBUG } 03:01:51 DEBUG } 03:01:51 DEBUG } 03:01:51 DEBUG ] 03:01:51 DEBUG ] 03:01:51 DEBUG } 03:01:51 DEBUG "0" 03:01:51 DEBUG } 03:01:51 DEBUG ] 03:01:51 DEBUG ] 03:01:51 DEBUG ] 03:01:51 DEBUG "59.8" 03:01:51 DEBUG "4312684241920" 03:01:51 DEBUG } 03:01:51 DEBUG } 03:01:51 DEBUG } 03:01:51 DEBUG "0.000032104444444444446" 03:01:51 DEBUG } 03:01:51 DEBUG "0" 03:01:51 DEBUG ] 03:01:51 DEBUG ] 03:01:51 DEBUG ] 03:01:51 DEBUG ] 03:01:51 DEBUG } 03:01:51 DEBUG } 03:01:51 DEBUG ] 03:01:51 DEBUG } 03:01:51 DEBUG ] 03:01:51 DEBUG } 03:01:51 DEBUG } 03:01:51 DEBUG } 03:01:51 DEBUG ] 03:01:51 DEBUG ] 03:01:51 DEBUG } 03:01:51 DEBUG } 03:01:51 DEBUG } 03:01:51 DEBUG ] 03:01:51 DEBUG ] 03:01:51 DEBUG ] 03:01:51 DEBUG ] 03:01:51 DEBUG ] 03:01:51 DEBUG ] 03:01:51 DEBUG ] 03:01:51 DEBUG } 03:01:51 DEBUG } 03:01:51 DEBUG } 03:01:51 DEBUG } 03:01:51 DEBUG } 03:01:51 DEBUG ] 03:01:51 DEBUG ] 03:01:51 DEBUG } 03:01:51 DEBUG } 03:01:51 DEBUG ] 03:01:51 DEBUG } 03:01:51 DEBUG } 03:01:51 DEBUG ] 03:01:51 DEBUG } 03:01:51 DEBUG } 03:01:51 DEBUG } 03:01:51 DEBUG } 03:01:51 DEBUG } 03:01:51 DEBUG } 03:01:51 DEBUG } 03:01:51 DEBUG } 03:01:51 DEBUG } 03:01:51 DEBUG } 03:01:51 DEBUG } 03:01:51 DEBUG } 03:01:51 DEBUG } 03:01:51 DEBUG } 03:01:51 DEBUG } 03:01:51 DEBUG } 03:01:51 DEBUG } 03:01:51 DEBUG } 03:01:51 DEBUG } 03:01:51 DEBUG } 03:01:51 DEBUG } 03:01:51 DEBUG ] 03:01:51 DEBUG ] 03:01:51 DEBUG } 03:01:51 DEBUG } 03:01:51 DEBUG ] 03:01:51 DEBUG } 03:01:51 DEBUG ] 03:01:51 INFO 03:01:51 DEBUG } 03:01:51 DEBUG } 03:01:51 DEBUG } 03:01:51 DEBUG } 03:01:51 DEBUG } 03:01:51 DEBUG } 03:01:51 DEBUG } 03:01:51 DEBUG } 03:01:51 INFO [loop_until]: kubectl --namespace=xlou top pods 03:01:51 DEBUG } 03:01:51 DEBUG } 03:01:51 DEBUG } 03:01:51 DEBUG } 03:01:51 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:01:51 INFO [loop_until]: OK (rc = 0) 03:01:51 DEBUG --- stdout --- 03:01:51 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 121m 8710Mi am-767f6b67b5-qcqz2 232m 8535Mi am-767f6b67b5-vbltz 843m 8611Mi ds-cts-0 266m 2146Mi ds-cts-1 173m 12331Mi ds-cts-2 86m 11882Mi ds-idrepo-0 381m 3929Mi ds-idrepo-1 76m 3953Mi ds-idrepo-2 10m 3913Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1179Mi idm-b85f9d978-wnhxb 4m 1239Mi lodemon-54c87f8cbb-jzrjs 1m 61Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-68c5d5b884-p97w5 2664m 550Mi 03:01:51 DEBUG --- stderr --- 03:01:51 DEBUG 03:02:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713664971" 03:02:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713664971" 03:02:51 INFO 03:02:51 INFO [loop_until]: kubectl --namespace=xlou top node 03:02:51 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:02:51 INFO [http_cmd]: http status code OK 03:02:51 ERROR [http_cmd]: ERROR 03:02:51 DEBUG --- status code --- 03:02:51 ERROR --- status code --- 03:02:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:02:51 ERROR http status code is 400 (expected [200, 401, 403, 404]) 03:02:51 DEBUG --- http response --- 03:02:51 ERROR --- http response --- 03:02:51 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 03:02:51 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 03:02:51 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 03:02:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713664971" 03:02:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713664971" 03:02:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713664971" 03:02:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713664971" 03:02:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713664971" 03:02:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713664971" 03:02:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713664971" 03:02:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713664971" 03:02:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713664971" 03:02:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713664971" 03:02:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713664971" 03:02:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713664971" 03:02:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713664971" 03:02:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713664971" 03:02:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713664971" 03:02:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713664971" 03:02:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713664971" 03:02:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713664971" 03:02:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713664971" 03:02:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713664971" 03:02:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713664971" 03:02:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713664971" 03:02:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713664971" 03:02:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713664971" 03:02:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713664971" 03:02:51 INFO [loop_until]: OK (rc = 0) 03:02:51 INFO [http_cmd]: http status code OK 03:02:51 INFO [http_cmd]: http status code OK 03:02:51 INFO [http_cmd]: http status code OK 03:02:51 INFO [http_cmd]: http status code OK 03:02:51 INFO [http_cmd]: http status code OK 03:02:51 INFO [http_cmd]: http status code OK 03:02:51 INFO [http_cmd]: http status code OK 03:02:51 INFO [http_cmd]: http status code OK 03:02:51 INFO [http_cmd]: http status code OK 03:02:51 INFO [http_cmd]: http status code OK 03:02:51 INFO [http_cmd]: http status code OK 03:02:51 INFO [http_cmd]: http status code OK 03:02:51 INFO [http_cmd]: http status code OK 03:02:51 INFO [http_cmd]: http status code OK 03:02:51 INFO [http_cmd]: http status code OK 03:02:51 INFO [http_cmd]: http status code OK 03:02:51 INFO [http_cmd]: http status code OK 03:02:51 INFO [http_cmd]: http status code OK 03:02:51 DEBUG --- stdout --- 03:02:51 INFO [http_cmd]: http status code OK 03:02:51 DEBUG --- status code --- 03:02:51 INFO [http_cmd]: http status code OK 03:02:51 INFO [http_cmd]: http status code OK 03:02:51 DEBUG --- status code --- 03:02:51 DEBUG --- status code --- 03:02:51 INFO [http_cmd]: http status code OK 03:02:51 DEBUG --- status code --- 03:02:51 INFO [http_cmd]: http status code OK 03:02:51 DEBUG --- status code --- 03:02:51 DEBUG --- status code --- 03:02:51 DEBUG --- status code --- 03:02:51 DEBUG --- status code --- 03:02:51 DEBUG --- status code --- 03:02:51 DEBUG --- status code --- 03:02:51 DEBUG --- status code --- 03:02:51 INFO [http_cmd]: http status code OK 03:02:51 DEBUG --- status code --- 03:02:51 DEBUG --- status code --- 03:02:51 DEBUG --- status code --- 03:02:51 DEBUG --- status code --- 03:02:51 DEBUG --- status code --- 03:02:51 DEBUG --- status code --- 03:02:51 INFO [http_cmd]: http status code OK 03:02:51 DEBUG --- status code --- 03:02:51 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 60m 0% 6294Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 8687m 29% 7775Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 17610m 58% 11510Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 87m 0% 3389Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 10845m 36% 7823Mi 7% gke-xlou-cdm-default-pool-1b0769c8-wpwl 143m 0% 5279Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 17964m 60% 11460Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 19398m 64% 3222Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 20232m 67% 10856Mi 9% gke-xlou-cdm-ds-0f601751-j488 8956m 29% 7369Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 9288m 31% 14532Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 9184m 30% 8016Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 10030m 33% 12112Mi 10% gke-xlou-cdm-frontend-a8771548-i82n 11848m 74% 11026Mi 18% 03:02:51 DEBUG --- status code --- 03:02:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:02:51 DEBUG --- status code --- 03:02:51 DEBUG --- status code --- 03:02:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:02:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:02:51 DEBUG --- status code --- 03:02:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:02:51 DEBUG --- status code --- 03:02:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:02:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:02:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:02:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:02:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:02:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:02:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:02:51 DEBUG --- status code --- 03:02:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:02:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:02:51 DEBUG --- http response --- 03:02:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:02:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:02:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:02:51 DEBUG --- status code --- 03:02:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:02:51 DEBUG --- stderr --- 03:02:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:02:51 DEBUG --- http response --- 03:02:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:02:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:02:51 DEBUG --- http response --- 03:02:51 DEBUG --- http response --- 03:02:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:02:51 DEBUG --- http response --- 03:02:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:02:51 DEBUG --- http response --- 03:02:51 DEBUG --- http response --- 03:02:51 DEBUG --- http response --- 03:02:51 DEBUG --- http response --- 03:02:51 DEBUG --- http response --- 03:02:51 DEBUG --- http response --- 03:02:51 DEBUG --- http response --- 03:02:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:02:51 DEBUG --- http response --- 03:02:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:02:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:02:51 DEBUG --- http response --- 03:02:51 DEBUG --- http response --- 03:02:51 DEBUG --- http response --- 03:02:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:02:51 DEBUG --- http response --- 03:02:51 DEBUG 03:02:51 DEBUG --- http response --- 03:02:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:02:51 DEBUG --- http response --- 03:02:51 DEBUG --- http response --- 03:02:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:02:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:02:51 DEBUG --- http response --- 03:02:51 DEBUG --- http response --- 03:02:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:02:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:02:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:02:51 INFO 03:02:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:02:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:02:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:02:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:02:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:02:51 DEBUG --- http response --- 03:02:51 DEBUG --- http response --- 03:02:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:02:51 DEBUG ----- output ----- 03:02:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:02:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:02:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:02:51 DEBUG --- http response --- 03:02:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:02:51 DEBUG ----- output ----- 03:02:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:02:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:02:51 DEBUG ----- output ----- 03:02:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:02:51 DEBUG ----- output ----- 03:02:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:02:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:02:51 DEBUG ----- output ----- 03:02:51 DEBUG ----- output ----- 03:02:51 DEBUG ----- output ----- 03:02:51 INFO [loop_until]: kubectl --namespace=xlou top pods 03:02:51 DEBUG ----- output ----- 03:02:51 DEBUG ----- output ----- 03:02:51 DEBUG ----- output ----- 03:02:51 DEBUG ----- output ----- 03:02:51 DEBUG ----- output ----- 03:02:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:02:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:02:51 DEBUG ----- output ----- 03:02:51 DEBUG { 03:02:51 DEBUG ----- output ----- 03:02:51 DEBUG ----- output ----- 03:02:51 DEBUG ----- output ----- 03:02:51 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:02:51 DEBUG ----- output ----- 03:02:51 DEBUG { 03:02:51 DEBUG ----- output ----- 03:02:51 DEBUG ----- output ----- 03:02:51 DEBUG { 03:02:51 DEBUG ----- output ----- 03:02:51 DEBUG { 03:02:51 DEBUG ----- output ----- 03:02:51 DEBUG ----- output ----- 03:02:51 DEBUG { 03:02:51 DEBUG { 03:02:51 DEBUG { 03:02:51 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:02:51 DEBUG { 03:02:51 DEBUG { 03:02:51 DEBUG { 03:02:51 DEBUG { 03:02:51 DEBUG { 03:02:51 DEBUG ----- output ----- 03:02:51 DEBUG ----- output ----- 03:02:51 DEBUG { 03:02:51 DEBUG "status": "success", 03:02:51 DEBUG { 03:02:51 DEBUG { 03:02:51 DEBUG { 03:02:51 DEBUG ----- output ----- 03:02:51 DEBUG { 03:02:51 DEBUG "status": "success", 03:02:51 DEBUG { 03:02:51 DEBUG { 03:02:51 DEBUG "status": "success", 03:02:51 DEBUG { 03:02:51 DEBUG "status": "success", 03:02:51 DEBUG { 03:02:51 DEBUG { 03:02:51 DEBUG "status": "success", 03:02:51 DEBUG "status": "success", 03:02:51 DEBUG "status": "success", 03:02:51 DEBUG "status": "success", 03:02:51 DEBUG "status": "success", 03:02:51 DEBUG "status": "success", 03:02:51 DEBUG "status": "success", 03:02:51 DEBUG "status": "success", 03:02:51 DEBUG { 03:02:51 DEBUG { 03:02:51 DEBUG "status": "success", 03:02:51 DEBUG "data": { 03:02:51 DEBUG "status": "success", 03:02:51 DEBUG "status": "success", 03:02:51 DEBUG "status": "success", 03:02:51 DEBUG { 03:02:52 DEBUG "status": "success", 03:02:51 DEBUG "status": "success", 03:02:51 DEBUG "status": "success", 03:02:51 DEBUG "status": "success", 03:02:52 DEBUG "data": { 03:02:51 DEBUG "status": "success", 03:02:51 DEBUG "data": { 03:02:51 DEBUG "status": "success", 03:02:51 DEBUG "status": "success", 03:02:51 DEBUG "data": { 03:02:51 DEBUG "data": { 03:02:51 DEBUG "data": { 03:02:51 DEBUG "data": { 03:02:51 DEBUG "data": { 03:02:51 DEBUG "data": { 03:02:51 DEBUG "data": { 03:02:51 DEBUG "data": { 03:02:52 DEBUG "status": "success", 03:02:52 DEBUG "status": "success", 03:02:52 DEBUG "data": { 03:02:52 DEBUG "resultType": "vector", 03:02:52 DEBUG "data": { 03:02:52 DEBUG "data": { 03:02:52 DEBUG "resultType": "vector", 03:02:51 DEBUG "data": { 03:02:52 DEBUG "data": { 03:02:52 DEBUG "data": { 03:02:52 DEBUG "resultType": "vector", 03:02:52 DEBUG "resultType": "vector", 03:02:51 DEBUG "data": { 03:02:52 DEBUG "data": { 03:02:52 DEBUG "resultType": "vector", 03:02:52 DEBUG "data": { 03:02:52 DEBUG "data": { 03:02:52 DEBUG "resultType": "vector", 03:02:52 DEBUG "resultType": "vector", 03:02:52 DEBUG "resultType": "vector", 03:02:52 DEBUG "resultType": "vector", 03:02:52 DEBUG "resultType": "vector", 03:02:52 DEBUG "resultType": "vector", 03:02:52 DEBUG "resultType": "vector", 03:02:52 DEBUG "resultType": "vector", 03:02:52 DEBUG "data": { 03:02:52 DEBUG "data": { 03:02:52 DEBUG "resultType": "vector", 03:02:52 DEBUG "result": [ 03:02:52 DEBUG "resultType": "vector", 03:02:52 DEBUG "result": [ 03:02:52 DEBUG "resultType": "vector", 03:02:52 DEBUG "resultType": "vector", 03:02:52 DEBUG "data": { 03:02:52 DEBUG "result": [ 03:02:52 DEBUG "result": [ 03:02:52 DEBUG "resultType": "vector", 03:02:52 DEBUG "data": { 03:02:52 DEBUG "resultType": "vector", 03:02:52 DEBUG "result": [ 03:02:52 DEBUG "resultType": "vector", 03:02:52 DEBUG "resultType": "vector", 03:02:52 DEBUG "result": [ 03:02:52 DEBUG "result": [ 03:02:52 DEBUG "result": [ 03:02:52 DEBUG "result": [ 03:02:52 DEBUG "result": [ 03:02:52 DEBUG "result": [ 03:02:52 DEBUG "result": [ 03:02:52 DEBUG "result": [ 03:02:52 DEBUG "resultType": "vector", 03:02:52 DEBUG "resultType": "vector", 03:02:52 DEBUG "result": [ 03:02:52 DEBUG { 03:02:52 DEBUG "result": [ 03:02:52 DEBUG { 03:02:52 DEBUG "result": [ 03:02:52 DEBUG "result": [ 03:02:52 DEBUG "resultType": "vector", 03:02:52 DEBUG { 03:02:52 DEBUG { 03:02:52 DEBUG "result": [ 03:02:52 DEBUG "resultType": "vector", 03:02:52 DEBUG "result": [ 03:02:52 DEBUG { 03:02:52 DEBUG "result": [ 03:02:52 DEBUG "result": [ 03:02:52 DEBUG { 03:02:52 DEBUG { 03:02:52 DEBUG { 03:02:52 DEBUG { 03:02:52 DEBUG { 03:02:52 DEBUG { 03:02:52 DEBUG { 03:02:52 DEBUG { 03:02:52 DEBUG "result": [ 03:02:52 DEBUG "result": [ 03:02:52 DEBUG { 03:02:52 DEBUG "metric": { 03:02:52 DEBUG { 03:02:52 DEBUG "metric": { 03:02:52 DEBUG { 03:02:52 DEBUG { 03:02:52 DEBUG "result": [ 03:02:52 DEBUG "metric": { 03:02:52 DEBUG "metric": { 03:02:52 DEBUG { 03:02:52 DEBUG "result": [ 03:02:52 DEBUG { 03:02:52 DEBUG "metric": { 03:02:52 DEBUG { 03:02:52 DEBUG { 03:02:52 DEBUG "metric": { 03:02:52 DEBUG "metric": { 03:02:52 DEBUG "metric": { 03:02:52 DEBUG "metric": { 03:02:52 DEBUG "metric": { 03:02:52 DEBUG "metric": { 03:02:52 DEBUG "metric": { 03:02:52 DEBUG "metric": { 03:02:52 DEBUG { 03:02:52 DEBUG "metric": { 03:02:52 DEBUG "metric": { 03:02:52 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:02:52 DEBUG "metric": { 03:02:52 DEBUG "instance": "10.142.0.107:9100" 03:02:52 DEBUG "metric": { 03:02:52 DEBUG "metric": { 03:02:52 DEBUG { 03:02:52 DEBUG "pod": "am-767f6b67b5-9zw22" 03:02:52 DEBUG "pod": "ds-idrepo-0" 03:02:52 DEBUG "metric": { 03:02:52 DEBUG { 03:02:52 DEBUG "metric": { 03:02:52 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:02:52 DEBUG "metric": { 03:02:52 DEBUG "metric": { 03:02:52 DEBUG "pod": "ds-idrepo-0" 03:02:52 DEBUG "instance": "10.142.0.107:9100" 03:02:52 DEBUG "instance": "10.142.0.107:9100" 03:02:52 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:02:52 DEBUG "pod": "am-767f6b67b5-9zw22" 03:02:52 DEBUG "pod": "ds-cts-0" 03:02:52 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:02:52 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:02:52 DEBUG { 03:02:52 DEBUG "instance": "10.142.0.107:9100" 03:02:52 DEBUG "pod": "am-767f6b67b5-9zw22" 03:02:52 DEBUG }, 03:02:52 DEBUG "pod": "ds-idrepo-0" 03:02:52 DEBUG }, 03:02:52 DEBUG "pod": "am-767f6b67b5-9zw22" 03:02:52 DEBUG "pod": "ds-idrepo-0" 03:02:52 DEBUG "metric": { 03:02:52 DEBUG }, 03:02:52 DEBUG }, 03:02:52 DEBUG "pod": "am-767f6b67b5-9zw22" 03:02:52 DEBUG "metric": { 03:02:52 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:02:52 DEBUG }, 03:02:52 DEBUG "pod": "am-767f6b67b5-9zw22" 03:02:52 DEBUG "pod": "ds-cts-0" 03:02:52 DEBUG }, 03:02:52 DEBUG }, 03:02:52 DEBUG }, 03:02:52 DEBUG }, 03:02:52 DEBUG }, 03:02:52 DEBUG }, 03:02:52 DEBUG }, 03:02:52 DEBUG }, 03:02:52 DEBUG "metric": { 03:02:52 DEBUG }, 03:02:52 DEBUG }, 03:02:52 DEBUG "value": [ 03:02:52 DEBUG }, 03:02:52 DEBUG "value": [ 03:02:52 DEBUG }, 03:02:52 DEBUG }, 03:02:52 DEBUG "pod": "ds-cts-0" 03:02:52 DEBUG "value": [ 03:02:52 DEBUG "value": [ 03:02:52 DEBUG }, 03:02:52 DEBUG "pod": "am-767f6b67b5-9zw22" 03:02:52 DEBUG }, 03:02:52 DEBUG "value": [ 03:02:52 DEBUG }, 03:02:52 DEBUG }, 03:02:52 DEBUG "value": [ 03:02:52 DEBUG "value": [ 03:02:52 DEBUG "value": [ 03:02:52 DEBUG "value": [ 03:02:52 DEBUG "value": [ 03:02:52 DEBUG "value": [ 03:02:52 DEBUG "value": [ 03:02:52 DEBUG "value": [ 03:02:52 DEBUG "instance": "10.142.0.107:9100" 03:02:52 DEBUG "value": [ 03:02:52 DEBUG "value": [ 03:02:52 DEBUG 03:02:52 DEBUG "value": [ 03:02:52 DEBUG 03:02:52 DEBUG "value": [ 03:02:52 DEBUG "value": [ 03:02:52 DEBUG }, 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG "value": [ 03:02:52 DEBUG }, 03:02:52 DEBUG "value": [ 03:02:52 DEBUG 03:02:52 DEBUG "value": [ 03:02:52 DEBUG "value": [ 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG }, 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG "value": [ 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG "value": [ 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG "value": [ 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG [...] 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG [...] 03:02:52 DEBUG 03:02:52 DEBUG [...] 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG [...] 03:02:52 DEBUG [...] 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG [...] 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG [...] 03:02:52 DEBUG [...] 03:02:52 DEBUG [...] 03:02:52 DEBUG [...] 03:02:52 DEBUG [...] 03:02:52 DEBUG [...] 03:02:52 DEBUG [...] 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG [...] 03:02:52 DEBUG [...] 03:02:52 DEBUG 03:02:52 DEBUG [...] 03:02:52 DEBUG 03:02:52 DEBUG [...] 03:02:52 DEBUG [...] 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG [...] 03:02:52 DEBUG 03:02:52 DEBUG [...] 03:02:52 DEBUG 03:02:52 DEBUG [...] 03:02:52 DEBUG [...] 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG [...] 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG [...] 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG "pod": "ds-idrepo-2" 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:02:52 DEBUG [...] 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:02:52 DEBUG 03:02:52 DEBUG "instance": "10.142.0.43:9100" 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG "pod": "ds-idrepo-2" 03:02:52 DEBUG "pod": "am-767f6b67b5-vbltz" 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG 03:02:52 DEBUG }, 03:02:52 DEBUG "instance": "10.142.0.43:9100" 03:02:52 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:02:52 DEBUG "pod": "am-767f6b67b5-vbltz" 03:02:52 DEBUG "pod": "ds-cts-2" 03:02:52 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:02:52 DEBUG }, 03:02:52 DEBUG 03:02:52 DEBUG "instance": "10.142.0.43:9100" 03:02:52 DEBUG }, 03:02:52 DEBUG }, 03:02:52 DEBUG "pod": "ds-idrepo-2" 03:02:52 DEBUG }, 03:02:52 DEBUG "pod": "am-767f6b67b5-vbltz" 03:02:52 DEBUG "pod": "ds-cts-2" 03:02:52 DEBUG 03:02:52 DEBUG }, 03:02:52 DEBUG }, 03:02:52 DEBUG "pod": "am-767f6b67b5-vbltz" 03:02:52 DEBUG 03:02:52 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:02:52 DEBUG }, 03:02:52 DEBUG "value": [ 03:02:52 DEBUG "pod": "ds-cts-2" 03:02:52 DEBUG "instance": "10.142.0.43:9100" 03:02:52 DEBUG "value": [ 03:02:52 DEBUG }, 03:02:52 DEBUG }, 03:02:52 DEBUG }, 03:02:52 DEBUG }, 03:02:52 DEBUG }, 03:02:52 DEBUG "value": [ 03:02:52 DEBUG 03:02:52 DEBUG "pod": "am-767f6b67b5-vbltz" 03:02:52 DEBUG "value": [ 03:02:52 DEBUG "value": [ 03:02:52 DEBUG }, 03:02:52 DEBUG "value": [ 03:02:52 DEBUG }, 03:02:52 DEBUG }, 03:02:52 DEBUG "pod": "ds-cts-2" 03:02:52 DEBUG "value": [ 03:02:52 DEBUG "value": [ 03:02:52 DEBUG }, 03:02:52 DEBUG "pod": "am-767f6b67b5-vbltz" 03:02:52 DEBUG }, 03:02:52 DEBUG 1713664971, 03:02:52 DEBUG "pod": "am-767f6b67b5-vbltz" 03:02:52 DEBUG }, 03:02:52 DEBUG }, 03:02:52 DEBUG 1713664971, 03:02:52 DEBUG "value": [ 03:02:52 DEBUG "value": [ 03:02:52 DEBUG "value": [ 03:02:52 DEBUG "value": [ 03:02:52 DEBUG "value": [ 03:02:52 DEBUG 1713664971, 03:02:52 DEBUG 03:02:52 DEBUG }, 03:02:52 DEBUG 1713664971, 03:02:52 DEBUG 1713664971, 03:02:52 DEBUG "value": [ 03:02:52 DEBUG 1713664971, 03:02:52 DEBUG "value": [ 03:02:52 DEBUG 1713664971, 03:02:52 DEBUG }, 03:02:52 DEBUG 1713664971, 03:02:52 DEBUG 1713664971, 03:02:52 DEBUG "value": [ 03:02:52 DEBUG }, 03:02:52 DEBUG "value": [ 03:02:52 DEBUG "0" 03:02:52 DEBUG }, 03:02:52 DEBUG "value": [ 03:02:52 DEBUG "value": [ 03:02:52 DEBUG "0" 03:02:52 DEBUG 1713664971, 03:02:52 DEBUG "79877600.16666666" 03:02:52 DEBUG 1713664971, 03:02:52 DEBUG 1713664971, 03:02:52 DEBUG 1713664971, 03:02:52 DEBUG "624714.4094910802" 03:02:52 DEBUG "instance": "10.142.0.43:9100" 03:02:52 DEBUG "value": [ 03:02:52 DEBUG "4315529370624" 03:02:52 DEBUG "12.059731640516803" 03:02:52 DEBUG 1713664971, 03:02:52 DEBUG "0.05870000000000459" 03:02:52 DEBUG "value": [ 03:02:52 DEBUG "3346.5" 03:02:52 DEBUG "value": [ 03:02:52 DEBUG "0" 03:02:52 DEBUG ] 03:02:52 DEBUG 1713664971, 03:02:52 DEBUG "value": [ 03:02:52 DEBUG 1713664971, 03:02:52 DEBUG ] 03:02:52 DEBUG "value": [ 03:02:52 DEBUG 1713664971, 03:02:52 DEBUG 1713664971, 03:02:52 DEBUG ] 03:02:52 DEBUG 1713664971, 03:02:52 DEBUG ] 03:02:52 DEBUG "774115328" 03:02:52 DEBUG "0" 03:02:52 DEBUG "0" 03:02:52 DEBUG ] 03:02:52 DEBUG }, 03:02:52 DEBUG 1713664971, 03:02:52 DEBUG ] 03:02:52 DEBUG ] 03:02:52 DEBUG "0" 03:02:52 DEBUG ] 03:02:52 DEBUG 1713664971, 03:02:52 DEBUG ] 03:02:52 DEBUG 1713664971, 03:02:52 DEBUG } 03:02:52 DEBUG "118975.5" 03:02:52 DEBUG "0" 03:02:52 DEBUG 1713664971, 03:02:52 DEBUG "20.109731062798048" 03:02:52 DEBUG } 03:02:52 DEBUG 1713664971, 03:02:52 DEBUG "0" 03:02:52 DEBUG ] 03:02:52 DEBUG } 03:02:52 DEBUG "149.888" 03:02:52 DEBUG } 03:02:52 DEBUG ] 03:02:52 DEBUG ] 03:02:52 DEBUG ] 03:02:52 DEBUG } 03:02:52 DEBUG "value": [ 03:02:52 DEBUG "3346.633333333333" 03:02:52 DEBUG } 03:02:52 DEBUG } 03:02:52 DEBUG ] 03:02:52 DEBUG } 03:02:52 DEBUG "0" 03:02:52 DEBUG } 03:02:52 DEBUG "0" 03:02:52 DEBUG ] 03:02:52 DEBUG ] 03:02:52 DEBUG ] 03:02:52 DEBUG "0" 03:02:52 DEBUG ] 03:02:52 DEBUG ] 03:02:52 DEBUG "3346.633333333333" 03:02:52 DEBUG } 03:02:52 DEBUG "0.03766666666666463" 03:02:52 DEBUG ] 03:02:52 DEBUG ] 03:02:52 DEBUG ] 03:02:52 DEBUG } 03:02:52 DEBUG } 03:02:52 DEBUG } 03:02:52 DEBUG ] 03:02:52 DEBUG 1713664971, 03:02:52 DEBUG ] 03:02:52 DEBUG ] 03:02:52 DEBUG ] 03:02:52 DEBUG } 03:02:52 DEBUG ] 03:02:52 DEBUG ] 03:02:52 DEBUG ] 03:02:52 DEBUG ] 03:02:52 DEBUG } 03:02:52 DEBUG } 03:02:52 DEBUG } 03:02:52 DEBUG ] 03:02:52 DEBUG } 03:02:52 DEBUG } 03:02:52 DEBUG ] 03:02:52 DEBUG ] 03:02:52 DEBUG ] 03:02:52 DEBUG } 03:02:52 DEBUG } 03:02:52 DEBUG } 03:02:52 DEBUG ] 03:02:52 DEBUG ] 03:02:52 DEBUG ] 03:02:52 DEBUG } 03:02:52 DEBUG "79881405.73333333" 03:02:52 DEBUG } 03:02:52 DEBUG } 03:02:52 DEBUG } 03:02:52 DEBUG ] 03:02:52 DEBUG } 03:02:52 DEBUG } 03:02:52 DEBUG } 03:02:52 DEBUG } 03:02:52 DEBUG } 03:02:52 DEBUG ] 03:02:52 DEBUG ] 03:02:52 DEBUG } 03:02:52 DEBUG ] 03:02:52 DEBUG } 03:02:52 DEBUG } 03:02:52 DEBUG } 03:02:52 DEBUG } 03:02:52 DEBUG } 03:02:52 DEBUG ] 03:02:52 DEBUG } 03:02:52 DEBUG } 03:02:52 DEBUG } 03:02:52 DEBUG } 03:02:52 DEBUG } 03:02:52 DEBUG ] 03:02:52 DEBUG ] 03:02:52 DEBUG } 03:02:52 DEBUG } 03:02:52 DEBUG } 03:02:52 DEBUG } 03:02:52 DEBUG ] 03:02:52 DEBUG } 03:02:52 DEBUG ] 03:02:52 DEBUG } 03:02:52 DEBUG } 03:02:52 DEBUG ] 03:02:52 DEBUG } 03:02:52 DEBUG ] 03:02:52 DEBUG } 03:02:52 DEBUG ] 03:02:52 DEBUG } 03:02:52 DEBUG } 03:02:52 DEBUG } 03:02:52 INFO [loop_until]: OK (rc = 0) 03:02:52 DEBUG } 03:02:52 DEBUG } 03:02:52 DEBUG } 03:02:52 DEBUG } 03:02:52 DEBUG } 03:02:52 DEBUG } 03:02:52 DEBUG } 03:02:52 DEBUG } 03:02:52 DEBUG } 03:02:52 DEBUG } 03:02:52 DEBUG } 03:02:52 DEBUG } 03:02:52 DEBUG } 03:02:52 DEBUG --- stdout --- 03:02:52 DEBUG ] 03:02:52 DEBUG } 03:02:52 DEBUG } 03:02:52 DEBUG } 03:02:52 DEBUG } 03:02:52 DEBUG } 03:02:52 DEBUG } 03:02:52 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 17529m 8730Mi am-767f6b67b5-qcqz2 17956m 8553Mi am-767f6b67b5-vbltz 18268m 8626Mi ds-cts-0 10991m 4428Mi ds-cts-1 9306m 9393Mi ds-cts-2 10403m 9381Mi ds-idrepo-0 8194m 3931Mi ds-idrepo-1 8148m 3954Mi ds-idrepo-2 8351m 3915Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1179Mi idm-b85f9d978-wnhxb 3m 1239Mi lodemon-54c87f8cbb-jzrjs 12m 61Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-68c5d5b884-p97w5 11921m 644Mi 03:02:52 DEBUG } 03:02:52 DEBUG --- stderr --- 03:02:52 DEBUG } 03:02:52 DEBUG 03:03:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713665031" 03:03:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665031" 03:03:51 ERROR [http_cmd]: ERROR 03:03:51 INFO [http_cmd]: http status code OK 03:03:51 ERROR --- status code --- 03:03:51 DEBUG --- status code --- 03:03:51 ERROR http status code is 400 (expected [200, 401, 403, 404]) 03:03:51 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:03:51 ERROR --- http response --- 03:03:51 DEBUG --- http response --- 03:03:51 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 03:03:51 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 03:03:51 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 03:03:52 INFO 03:03:52 INFO [loop_until]: kubectl --namespace=xlou top node 03:03:52 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:03:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665032" 03:03:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665032" 03:03:52 INFO [http_cmd]: http status code OK 03:03:52 INFO [http_cmd]: http status code OK 03:03:52 DEBUG --- status code --- 03:03:52 DEBUG --- status code --- 03:03:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:03:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:03:52 DEBUG --- http response --- 03:03:52 DEBUG --- http response --- 03:03:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:03:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:03:52 DEBUG ----- output ----- 03:03:52 DEBUG ----- output ----- 03:03:52 DEBUG { 03:03:52 DEBUG { 03:03:52 DEBUG "status": "success", 03:03:52 DEBUG "status": "success", 03:03:52 DEBUG "data": { 03:03:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665032" 03:03:52 DEBUG "data": { 03:03:52 DEBUG "resultType": "vector", 03:03:52 DEBUG "resultType": "vector", 03:03:52 DEBUG "result": [ 03:03:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665032" 03:03:52 DEBUG "result": [ 03:03:52 DEBUG { 03:03:52 DEBUG { 03:03:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713665032" 03:03:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665032" 03:03:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665032" 03:03:52 DEBUG "metric": { 03:03:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665032" 03:03:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665032" 03:03:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665032" 03:03:52 DEBUG "metric": { 03:03:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665032" 03:03:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713665032" 03:03:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665032" 03:03:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665032" 03:03:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665032" 03:03:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713665032" 03:03:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713665032" 03:03:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665032" 03:03:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665032" 03:03:52 DEBUG "instance": "10.142.0.107:9100" 03:03:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665032" 03:03:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665032" 03:03:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665032" 03:03:52 INFO 03:03:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665032" 03:03:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665032" 03:03:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665032" 03:03:52 INFO [http_cmd]: http status code OK 03:03:52 INFO [loop_until]: OK (rc = 0) 03:03:52 INFO [http_cmd]: http status code OK 03:03:52 INFO [http_cmd]: http status code OK 03:03:52 INFO [http_cmd]: http status code OK 03:03:52 INFO [http_cmd]: http status code OK 03:03:52 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:03:52 INFO [http_cmd]: http status code OK 03:03:52 INFO [http_cmd]: http status code OK 03:03:52 INFO [http_cmd]: http status code OK 03:03:52 DEBUG }, 03:03:52 INFO [http_cmd]: http status code OK 03:03:52 INFO [http_cmd]: http status code OK 03:03:52 INFO [http_cmd]: http status code OK 03:03:52 INFO [http_cmd]: http status code OK 03:03:52 INFO [http_cmd]: http status code OK 03:03:52 INFO [loop_until]: kubectl --namespace=xlou top pods 03:03:52 INFO [http_cmd]: http status code OK 03:03:52 INFO [http_cmd]: http status code OK 03:03:52 INFO [http_cmd]: http status code OK 03:03:52 INFO [http_cmd]: http status code OK 03:03:52 DEBUG --- status code --- 03:03:52 DEBUG --- stdout --- 03:03:52 DEBUG --- status code --- 03:03:52 DEBUG --- status code --- 03:03:52 DEBUG --- status code --- 03:03:52 DEBUG --- status code --- 03:03:52 INFO [http_cmd]: http status code OK 03:03:52 INFO [http_cmd]: http status code OK 03:03:52 INFO [http_cmd]: http status code OK 03:03:52 DEBUG }, 03:03:52 DEBUG --- status code --- 03:03:52 DEBUG --- status code --- 03:03:52 DEBUG --- status code --- 03:03:52 DEBUG "value": [ 03:03:52 DEBUG --- status code --- 03:03:52 DEBUG --- status code --- 03:03:52 DEBUG --- status code --- 03:03:52 DEBUG --- status code --- 03:03:52 DEBUG --- status code --- 03:03:52 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:03:52 DEBUG --- status code --- 03:03:52 DEBUG --- status code --- 03:03:52 DEBUG --- status code --- 03:03:52 DEBUG --- status code --- 03:03:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:03:52 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 6284Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 8586m 28% 7779Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 17561m 58% 11516Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 81m 0% 3387Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 10556m 35% 10655Mi 9% gke-xlou-cdm-default-pool-1b0769c8-wpwl 160m 0% 5266Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 18088m 60% 11460Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 19444m 65% 3216Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 19495m 65% 10872Mi 9% gke-xlou-cdm-ds-0f601751-j488 9208m 30% 7365Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 9498m 31% 14600Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 9155m 30% 8015Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 10342m 34% 12023Mi 10% gke-xlou-cdm-frontend-a8771548-i82n 11940m 75% 11029Mi 18% 03:03:52 INFO [http_cmd]: http status code OK 03:03:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:03:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:03:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:03:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:03:52 DEBUG --- status code --- 03:03:52 DEBUG --- status code --- 03:03:52 DEBUG --- status code --- 03:03:52 DEBUG "value": [ 03:03:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:03:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:03:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:03:52 DEBUG 03:03:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:03:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:03:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:03:52 INFO [http_cmd]: http status code OK 03:03:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:03:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:03:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:03:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:03:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:03:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:03:52 INFO [http_cmd]: http status code OK 03:03:52 DEBUG --- http response --- 03:03:52 DEBUG --- stderr --- 03:03:52 DEBUG --- status code --- 03:03:52 DEBUG --- http response --- 03:03:52 DEBUG --- http response --- 03:03:52 DEBUG --- http response --- 03:03:52 DEBUG --- http response --- 03:03:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:03:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:03:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:03:52 DEBUG 03:03:52 DEBUG --- http response --- 03:03:52 DEBUG --- http response --- 03:03:52 DEBUG --- http response --- 03:03:52 DEBUG 03:03:52 DEBUG --- http response --- 03:03:52 DEBUG --- http response --- 03:03:52 DEBUG --- http response --- 03:03:52 DEBUG --- status code --- 03:03:52 DEBUG --- http response --- 03:03:52 DEBUG --- http response --- 03:03:52 DEBUG --- http response --- 03:03:52 DEBUG --- http response --- 03:03:52 DEBUG --- http response --- 03:03:52 DEBUG --- http response --- 03:03:52 DEBUG --- status code --- 03:03:52 DEBUG 03:03:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:03:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:03:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:03:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:03:52 DEBUG --- http response --- 03:03:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:03:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:03:52 DEBUG --- http response --- 03:03:52 DEBUG --- http response --- 03:03:52 DEBUG 03:03:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:03:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:03:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:03:52 DEBUG 03:03:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:03:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:03:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:03:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:03:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:03:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:03:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:03:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:03:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:03:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:03:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:03:52 DEBUG ----- output ----- 03:03:52 DEBUG --- http response --- 03:03:52 DEBUG ----- output ----- 03:03:52 DEBUG ----- output ----- 03:03:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:03:52 DEBUG ----- output ----- 03:03:52 DEBUG ----- output ----- 03:03:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:03:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:03:52 DEBUG 03:03:52 DEBUG ----- output ----- 03:03:52 DEBUG ----- output ----- 03:03:52 DEBUG ----- output ----- 03:03:52 DEBUG [...] 03:03:52 DEBUG ----- output ----- 03:03:52 DEBUG ----- output ----- 03:03:52 DEBUG ----- output ----- 03:03:52 DEBUG --- http response --- 03:03:52 DEBUG ----- output ----- 03:03:52 DEBUG ----- output ----- 03:03:52 DEBUG ----- output ----- 03:03:52 DEBUG ----- output ----- 03:03:52 DEBUG ----- output ----- 03:03:52 DEBUG --- http response --- 03:03:52 DEBUG ----- output ----- 03:03:52 DEBUG { 03:03:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:03:52 DEBUG { 03:03:52 DEBUG { 03:03:52 DEBUG ----- output ----- 03:03:52 DEBUG { 03:03:52 DEBUG { 03:03:52 DEBUG ----- output ----- 03:03:52 DEBUG ----- output ----- 03:03:52 DEBUG [...] 03:03:52 DEBUG { 03:03:52 DEBUG { 03:03:52 DEBUG { 03:03:52 DEBUG 03:03:52 DEBUG { 03:03:52 DEBUG { 03:03:52 DEBUG { 03:03:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:03:52 DEBUG { 03:03:52 DEBUG { 03:03:52 DEBUG { 03:03:52 DEBUG { 03:03:52 DEBUG { 03:03:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:03:52 DEBUG { 03:03:52 DEBUG "status": "success", 03:03:52 DEBUG ----- output ----- 03:03:52 DEBUG "status": "success", 03:03:52 DEBUG "status": "success", 03:03:52 DEBUG { 03:03:52 DEBUG "status": "success", 03:03:52 DEBUG "status": "success", 03:03:52 DEBUG { 03:03:52 DEBUG { 03:03:52 DEBUG 03:03:52 DEBUG "status": "success", 03:03:52 DEBUG "status": "success", 03:03:52 DEBUG "status": "success", 03:03:52 DEBUG 03:03:52 DEBUG "status": "success", 03:03:52 DEBUG "status": "success", 03:03:52 DEBUG "status": "success", 03:03:52 DEBUG ----- output ----- 03:03:52 DEBUG { 03:03:52 DEBUG "status": "success", 03:03:52 DEBUG "status": "success", 03:03:52 DEBUG "status": "success", 03:03:52 DEBUG "status": "success", 03:03:52 DEBUG ----- output ----- 03:03:52 DEBUG "status": "success", 03:03:52 DEBUG "data": { 03:03:52 DEBUG { 03:03:52 DEBUG "data": { 03:03:52 DEBUG "data": { 03:03:52 DEBUG "status": "success", 03:03:52 DEBUG "data": { 03:03:52 DEBUG "data": { 03:03:52 DEBUG "status": "success", 03:03:52 DEBUG "status": "success", 03:03:52 DEBUG 03:03:52 DEBUG "data": { 03:03:52 DEBUG "data": { 03:03:52 DEBUG "data": { 03:03:52 DEBUG 03:03:52 DEBUG "data": { 03:03:52 DEBUG "data": { 03:03:52 DEBUG "data": { 03:03:52 DEBUG "status": "success", 03:03:52 DEBUG "status": "success", 03:03:52 DEBUG "data": { 03:03:52 DEBUG "data": { 03:03:52 DEBUG "data": { 03:03:52 DEBUG "data": { 03:03:52 DEBUG { 03:03:52 DEBUG "data": { 03:03:52 DEBUG "resultType": "vector", 03:03:52 DEBUG "status": "success", 03:03:52 DEBUG "resultType": "vector", 03:03:52 DEBUG "resultType": "vector", 03:03:52 DEBUG "data": { 03:03:52 DEBUG "resultType": "vector", 03:03:52 DEBUG "resultType": "vector", 03:03:52 DEBUG "data": { 03:03:52 DEBUG "data": { 03:03:52 DEBUG 03:03:52 DEBUG "resultType": "vector", 03:03:52 DEBUG "resultType": "vector", 03:03:52 DEBUG "resultType": "vector", 03:03:52 DEBUG "instance": "10.142.0.43:9100" 03:03:52 DEBUG "resultType": "vector", 03:03:52 DEBUG "resultType": "vector", 03:03:52 DEBUG "resultType": "vector", 03:03:52 DEBUG "data": { 03:03:52 DEBUG "data": { 03:03:52 DEBUG "resultType": "vector", 03:03:52 DEBUG "result": [ 03:03:52 DEBUG "resultType": "vector", 03:03:52 DEBUG "resultType": "vector", 03:03:52 DEBUG "status": "success", 03:03:52 DEBUG "resultType": "vector", 03:03:52 DEBUG "result": [ 03:03:52 DEBUG "data": { 03:03:52 DEBUG "result": [ 03:03:52 DEBUG "result": [ 03:03:52 DEBUG "resultType": "vector", 03:03:52 DEBUG "result": [ 03:03:52 DEBUG "result": [ 03:03:52 DEBUG "resultType": "vector", 03:03:52 DEBUG "resultType": "vector", 03:03:52 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 03:03:52 DEBUG "result": [ 03:03:52 DEBUG "result": [ 03:03:52 DEBUG "result": [ 03:03:52 DEBUG }, 03:03:52 DEBUG "result": [ 03:03:52 DEBUG "result": [ 03:03:52 DEBUG "result": [ 03:03:52 DEBUG "resultType": "vector", 03:03:52 DEBUG "resultType": "vector", 03:03:52 DEBUG "resultType": "vector", 03:03:52 DEBUG { 03:03:52 DEBUG "result": [ 03:03:52 DEBUG "result": [ 03:03:52 DEBUG "data": { 03:03:52 DEBUG "result": [ 03:03:52 DEBUG { 03:03:52 DEBUG "resultType": "vector", 03:03:52 DEBUG { 03:03:52 DEBUG { 03:03:52 DEBUG "result": [ 03:03:52 DEBUG { 03:03:52 DEBUG { 03:03:52 DEBUG "result": [ 03:03:52 DEBUG "result": [ 03:03:52 DEBUG }, 03:03:52 DEBUG { 03:03:52 DEBUG { 03:03:52 DEBUG { 03:03:52 DEBUG "value": [ 03:03:52 DEBUG { 03:03:52 DEBUG { 03:03:52 DEBUG { 03:03:52 DEBUG "result": [ 03:03:52 DEBUG "result": [ 03:03:52 DEBUG "result": [ 03:03:52 DEBUG "metric": { 03:03:52 DEBUG { 03:03:52 DEBUG { 03:03:52 DEBUG "resultType": "vector", 03:03:52 DEBUG { 03:03:52 DEBUG "metric": { 03:03:52 DEBUG "result": [ 03:03:52 DEBUG "metric": { 03:03:52 DEBUG "metric": { 03:03:52 DEBUG { 03:03:52 DEBUG "metric": { 03:03:52 DEBUG "metric": { 03:03:52 DEBUG { 03:03:52 DEBUG { 03:03:52 DEBUG "value": [ 03:03:52 DEBUG "metric": { 03:03:52 DEBUG "metric": { 03:03:52 DEBUG "metric": { 03:03:52 DEBUG 1713665032, 03:03:52 DEBUG "metric": { 03:03:52 DEBUG "metric": { 03:03:52 DEBUG "metric": { 03:03:52 DEBUG { 03:03:52 DEBUG { 03:03:52 DEBUG { 03:03:52 DEBUG "pod": "ds-cts-0" 03:03:52 DEBUG "metric": { 03:03:52 DEBUG "metric": { 03:03:52 DEBUG "result": [ 03:03:52 DEBUG "metric": { 03:03:52 DEBUG "pod": "ds-idrepo-0" 03:03:52 DEBUG { 03:03:52 DEBUG "pod": "ds-idrepo-0" 03:03:52 DEBUG "instance": "10.142.0.107:9100" 03:03:52 DEBUG "metric": { 03:03:52 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:03:52 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:03:52 DEBUG "metric": { 03:03:52 DEBUG "metric": { 03:03:52 DEBUG 1713665032, 03:03:52 DEBUG "0" 03:03:52 DEBUG "instance": "10.142.0.107:9100" 03:03:52 DEBUG "pod": "am-767f6b67b5-9zw22" 03:03:52 DEBUG "79347261.13333333" 03:03:52 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:03:52 DEBUG "pod": "ds-cts-0" 03:03:52 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:03:52 DEBUG "metric": { 03:03:52 DEBUG "metric": { 03:03:52 DEBUG "metric": { 03:03:52 DEBUG }, 03:03:52 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:03:52 DEBUG "pod": "ds-idrepo-0" 03:03:52 DEBUG { 03:03:52 DEBUG "pod": "am-767f6b67b5-9zw22" 03:03:52 DEBUG }, 03:03:52 DEBUG "metric": { 03:03:52 DEBUG }, 03:03:52 DEBUG }, 03:03:52 DEBUG "pod": "ds-cts-0" 03:03:52 DEBUG }, 03:03:52 DEBUG }, 03:03:52 DEBUG "pod": "am-767f6b67b5-9zw22" 03:03:52 DEBUG "pod": "am-767f6b67b5-9zw22" 03:03:52 DEBUG "pod": "am-767f6b67b5-9zw22" 03:03:52 DEBUG ] 03:03:52 DEBUG }, 03:03:52 DEBUG }, 03:03:52 DEBUG ] 03:03:52 DEBUG }, 03:03:52 DEBUG }, 03:03:52 DEBUG "value": [ 03:03:52 DEBUG "pod": "ds-idrepo-0" 03:03:52 DEBUG "instance": "10.142.0.107:9100" 03:03:52 DEBUG "pod": "am-767f6b67b5-9zw22" 03:03:52 DEBUG "value": [ 03:03:52 DEBUG }, 03:03:52 DEBUG }, 03:03:52 DEBUG "metric": { 03:03:52 DEBUG }, 03:03:52 DEBUG "value": [ 03:03:52 DEBUG "pod": "am-767f6b67b5-9zw22" 03:03:52 DEBUG "value": [ 03:03:52 DEBUG "value": [ 03:03:52 DEBUG }, 03:03:52 DEBUG "value": [ 03:03:52 DEBUG "value": [ 03:03:52 DEBUG }, 03:03:52 DEBUG }, 03:03:52 DEBUG }, 03:03:52 DEBUG } 03:03:52 DEBUG "value": [ 03:03:52 DEBUG "value": [ 03:03:52 DEBUG } 03:03:52 DEBUG "value": [ 03:03:52 DEBUG 03:03:52 DEBUG }, 03:03:52 DEBUG }, 03:03:52 DEBUG }, 03:03:52 DEBUG }, 03:03:52 DEBUG 03:03:52 DEBUG "value": [ 03:03:52 DEBUG "value": [ 03:03:52 DEBUG "instance": "10.142.0.107:9100" 03:03:52 DEBUG "value": [ 03:03:52 DEBUG 03:03:52 DEBUG }, 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG "value": [ 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG "value": [ 03:03:52 DEBUG "value": [ 03:03:52 DEBUG "value": [ 03:03:52 DEBUG ] 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG ] 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG "value": [ 03:03:52 DEBUG "value": [ 03:03:52 DEBUG "value": [ 03:03:52 DEBUG "value": [ 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG }, 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG "value": [ 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG } 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG } 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG "value": [ 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG [...] 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG } 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG } 03:03:52 DEBUG 03:03:52 DEBUG [...] 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG [...] 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG [...] 03:03:52 DEBUG [...] 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG [...] 03:03:52 DEBUG [...] 03:03:52 DEBUG [...] 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG [...] 03:03:52 DEBUG [...] 03:03:52 DEBUG [...] 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG [...] 03:03:52 DEBUG 03:03:52 DEBUG [...] 03:03:52 DEBUG [...] 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG [...] 03:03:52 DEBUG [...] 03:03:52 DEBUG [...] 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG [...] 03:03:52 DEBUG 03:03:52 DEBUG [...] 03:03:52 DEBUG 03:03:52 DEBUG [...] 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG [...] 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG [...] 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG [...] 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG "instance": "10.142.0.43:9100" 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 03:03:52 DEBUG 03:03:52 DEBUG "pod": "ds-cts-2" 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG "pod": "ds-idrepo-2" 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG }, 03:03:52 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:03:52 INFO [loop_until]: OK (rc = 0) 03:03:52 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG 03:03:52 DEBUG "instance": "10.142.0.43:9100" 03:03:52 DEBUG "pod": "am-767f6b67b5-vbltz" 03:03:52 DEBUG }, 03:03:52 DEBUG 03:03:52 DEBUG }, 03:03:52 DEBUG 03:03:52 DEBUG "instance": "10.142.0.43:9100" 03:03:52 DEBUG 03:03:52 DEBUG "pod": "ds-cts-2" 03:03:52 DEBUG 03:03:52 DEBUG }, 03:03:52 DEBUG 03:03:52 DEBUG "pod": "am-767f6b67b5-vbltz" 03:03:52 DEBUG }, 03:03:52 DEBUG "pod": "ds-idrepo-2" 03:03:52 DEBUG "pod": "am-767f6b67b5-vbltz" 03:03:52 DEBUG 03:03:52 DEBUG "value": [ 03:03:52 DEBUG }, 03:03:52 DEBUG --- stdout --- 03:03:52 DEBUG }, 03:03:52 DEBUG "pod": "am-767f6b67b5-vbltz" 03:03:52 DEBUG }, 03:03:52 DEBUG "pod": "am-767f6b67b5-vbltz" 03:03:52 DEBUG }, 03:03:52 DEBUG }, 03:03:52 DEBUG "value": [ 03:03:52 DEBUG 03:03:52 DEBUG "value": [ 03:03:52 DEBUG 1713665032, 03:03:52 DEBUG }, 03:03:52 DEBUG "pod": "am-767f6b67b5-vbltz" 03:03:52 DEBUG }, 03:03:52 DEBUG "pod": "ds-cts-2" 03:03:52 DEBUG "value": [ 03:03:52 DEBUG 03:03:52 DEBUG }, 03:03:52 DEBUG "value": [ 03:03:52 DEBUG }, 03:03:52 DEBUG }, 03:03:52 DEBUG "pod": "ds-cts-2" 03:03:52 DEBUG 1713665032, 03:03:52 DEBUG "value": [ 03:03:52 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 17097m 8732Mi am-767f6b67b5-qcqz2 17392m 8558Mi am-767f6b67b5-vbltz 19448m 8636Mi ds-cts-0 10401m 7049Mi ds-cts-1 9614m 9446Mi ds-cts-2 10293m 9290Mi ds-idrepo-0 8311m 3930Mi ds-idrepo-1 8263m 3954Mi ds-idrepo-2 8143m 3916Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1179Mi idm-b85f9d978-wnhxb 3m 1239Mi lodemon-54c87f8cbb-jzrjs 19m 62Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-68c5d5b884-p97w5 11351m 651Mi 03:03:52 DEBUG "value": [ 03:03:52 DEBUG "value": [ 03:03:52 DEBUG }, 03:03:52 DEBUG "value": [ 03:03:52 DEBUG "pod": "am-767f6b67b5-vbltz" 03:03:52 DEBUG "value": [ 03:03:52 DEBUG 1713665032, 03:03:52 DEBUG "pod": "ds-idrepo-2" 03:03:52 DEBUG "0.00023266" 03:03:52 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:03:52 DEBUG "value": [ 03:03:52 DEBUG }, 03:03:52 DEBUG "value": [ 03:03:52 DEBUG }, 03:03:52 DEBUG 1713665032, 03:03:52 DEBUG "instance": "10.142.0.43:9100" 03:03:52 DEBUG "value": [ 03:03:52 DEBUG 1713665032, 03:03:52 DEBUG "value": [ 03:03:52 DEBUG "value": [ 03:03:52 DEBUG }, 03:03:52 DEBUG "4319152935936" 03:03:52 DEBUG 1713665032, 03:03:52 DEBUG --- stderr --- 03:03:52 DEBUG 1713665032, 03:03:52 DEBUG 1713665032, 03:03:52 DEBUG "value": [ 03:03:52 DEBUG 1713665032, 03:03:52 DEBUG }, 03:03:52 DEBUG 1713665032, 03:03:52 DEBUG "0" 03:03:52 DEBUG }, 03:03:52 DEBUG ] 03:03:52 DEBUG }, 03:03:52 DEBUG 1713665032, 03:03:52 DEBUG "value": [ 03:03:52 DEBUG 1713665032, 03:03:52 DEBUG "value": [ 03:03:52 DEBUG "0.00003123032003205276" 03:03:52 DEBUG }, 03:03:52 DEBUG 1713665032, 03:03:52 DEBUG "0" 03:03:52 DEBUG 1713665032, 03:03:52 DEBUG 1713665032, 03:03:52 DEBUG "value": [ 03:03:52 DEBUG ] 03:03:52 DEBUG "11.714023684995523" 03:03:52 DEBUG 03:03:52 DEBUG "0" 03:03:52 DEBUG "3329.733333333334" 03:03:52 DEBUG 1713665032, 03:03:52 DEBUG "0.06273333333327287" 03:03:52 DEBUG "value": [ 03:03:52 DEBUG "0" 03:03:52 DEBUG ] 03:03:52 DEBUG "value": [ 03:03:52 DEBUG } 03:03:52 DEBUG "value": [ 03:03:52 DEBUG "0.039333333333329765" 03:03:52 DEBUG 1713665032, 03:03:52 DEBUG "0" 03:03:52 DEBUG 1713665032, 03:03:52 DEBUG ] 03:03:52 DEBUG "value": [ 03:03:52 DEBUG "418880" 03:03:52 DEBUG ] 03:03:52 DEBUG "0" 03:03:52 DEBUG "0" 03:03:52 DEBUG 1713665032, 03:03:52 DEBUG } 03:03:52 DEBUG ] 03:03:52 DEBUG ] 03:03:52 DEBUG ] 03:03:52 DEBUG "3329.4333333333334" 03:03:52 DEBUG ] 03:03:52 DEBUG 1713665032, 03:03:52 DEBUG ] 03:03:52 DEBUG } 03:03:52 DEBUG 1713665032, 03:03:52 DEBUG ] 03:03:52 DEBUG 1713665032, 03:03:52 DEBUG ] 03:03:52 DEBUG "149.888" 03:03:52 DEBUG ] 03:03:52 DEBUG "0" 03:03:52 DEBUG } 03:03:52 DEBUG 1713665032, 03:03:52 DEBUG ] 03:03:52 DEBUG } 03:03:52 DEBUG ] 03:03:52 DEBUG ] 03:03:52 DEBUG "0" 03:03:52 DEBUG ] 03:03:52 DEBUG } 03:03:52 DEBUG } 03:03:52 DEBUG } 03:03:52 DEBUG ] 03:03:52 DEBUG } 03:03:52 DEBUG "3329.600000000001" 03:03:52 DEBUG } 03:03:52 DEBUG ] 03:03:52 DEBUG "0" 03:03:52 DEBUG } 03:03:52 DEBUG "781053952" 03:03:52 DEBUG } 03:03:52 DEBUG ] 03:03:52 DEBUG } 03:03:52 DEBUG ] 03:03:52 DEBUG ] 03:03:52 DEBUG "79350829" 03:03:52 DEBUG } 03:03:52 DEBUG ] 03:03:52 DEBUG } 03:03:52 DEBUG } 03:03:52 DEBUG ] 03:03:52 DEBUG } 03:03:52 DEBUG ] 03:03:52 DEBUG ] 03:03:52 DEBUG ] 03:03:52 DEBUG } 03:03:52 DEBUG ] 03:03:52 DEBUG ] 03:03:52 DEBUG ] 03:03:52 DEBUG } 03:03:52 DEBUG ] 03:03:52 DEBUG } 03:03:52 DEBUG ] 03:03:52 DEBUG ] 03:03:52 DEBUG } 03:03:52 DEBUG ] 03:03:52 DEBUG } 03:03:52 DEBUG } 03:03:52 DEBUG ] 03:03:52 DEBUG ] 03:03:52 DEBUG } 03:03:52 DEBUG ] 03:03:52 DEBUG ] 03:03:52 DEBUG } 03:03:52 DEBUG } 03:03:52 DEBUG } 03:03:52 DEBUG } 03:03:52 DEBUG } 03:03:52 DEBUG ] 03:03:52 DEBUG } 03:03:52 DEBUG } 03:03:52 DEBUG } 03:03:52 DEBUG } 03:03:52 DEBUG } 03:03:52 DEBUG } 03:03:52 DEBUG } 03:03:52 DEBUG ] 03:03:52 DEBUG } 03:03:52 DEBUG ] 03:03:52 DEBUG } 03:03:52 DEBUG } 03:03:52 DEBUG } 03:03:52 DEBUG } 03:03:52 DEBUG } 03:03:52 DEBUG } 03:03:52 DEBUG ] 03:03:52 DEBUG } 03:03:52 DEBUG } 03:03:52 DEBUG } 03:03:52 DEBUG } 03:03:52 DEBUG } 03:03:52 DEBUG ] 03:03:52 DEBUG } 03:03:52 DEBUG ] 03:03:52 DEBUG ] 03:03:52 DEBUG } 03:03:52 DEBUG } 03:03:52 DEBUG } 03:03:52 DEBUG } 03:03:52 DEBUG ] 03:03:52 DEBUG } 03:03:52 DEBUG } 03:03:52 DEBUG } 03:03:52 DEBUG } 03:03:52 DEBUG } 03:03:52 DEBUG } 03:03:52 DEBUG } 03:03:52 DEBUG } 03:03:52 DEBUG } 03:03:52 DEBUG } 03:03:52 DEBUG } 03:03:52 DEBUG } 03:03:52 DEBUG } 03:03:52 DEBUG } 03:03:52 DEBUG } 03:03:52 DEBUG } 03:04:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713665091" 03:04:51 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665091" 03:04:52 ERROR [http_cmd]: ERROR 03:04:52 INFO [http_cmd]: http status code OK 03:04:52 ERROR --- status code --- 03:04:52 DEBUG --- status code --- 03:04:52 ERROR http status code is 400 (expected [200, 401, 403, 404]) 03:04:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:04:52 ERROR --- http response --- 03:04:52 DEBUG --- http response --- 03:04:52 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 03:04:52 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 03:04:52 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 03:04:52 INFO 03:04:52 INFO [loop_until]: kubectl --namespace=xlou top node 03:04:52 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:04:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665092" 03:04:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665092" 03:04:52 INFO 03:04:52 INFO [loop_until]: kubectl --namespace=xlou top pods 03:04:52 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:04:52 INFO [http_cmd]: http status code OK 03:04:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665092" 03:04:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665092" 03:04:52 INFO [http_cmd]: http status code OK 03:04:52 DEBUG --- status code --- 03:04:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665092" 03:04:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665092" 03:04:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665092" 03:04:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713665092" 03:04:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665092" 03:04:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665092" 03:04:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665092" 03:04:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665092" 03:04:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665092" 03:04:52 DEBUG --- status code --- 03:04:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665092" 03:04:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665092" 03:04:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713665092" 03:04:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665092" 03:04:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665092" 03:04:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713665092" 03:04:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665092" 03:04:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665092" 03:04:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713665092" 03:04:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665092" 03:04:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:04:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665092" 03:04:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665092" 03:04:52 INFO [loop_until]: OK (rc = 0) 03:04:52 INFO [http_cmd]: http status code OK 03:04:52 INFO [http_cmd]: http status code OK 03:04:52 INFO [http_cmd]: http status code OK 03:04:52 INFO [http_cmd]: http status code OK 03:04:52 INFO [http_cmd]: http status code OK 03:04:52 INFO [http_cmd]: http status code OK 03:04:52 INFO [http_cmd]: http status code OK 03:04:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:04:52 INFO [http_cmd]: http status code OK 03:04:52 INFO [http_cmd]: http status code OK 03:04:52 INFO [http_cmd]: http status code OK 03:04:52 INFO [http_cmd]: http status code OK 03:04:52 INFO [http_cmd]: http status code OK 03:04:52 DEBUG --- http response --- 03:04:52 INFO [http_cmd]: http status code OK 03:04:52 INFO [http_cmd]: http status code OK 03:04:52 INFO [http_cmd]: http status code OK 03:04:52 INFO [loop_until]: OK (rc = 0) 03:04:52 INFO [http_cmd]: http status code OK 03:04:52 INFO [http_cmd]: http status code OK 03:04:52 DEBUG --- stdout --- 03:04:52 INFO [http_cmd]: http status code OK 03:04:52 INFO [http_cmd]: http status code OK 03:04:52 DEBUG --- status code --- 03:04:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:04:52 DEBUG --- http response --- 03:04:52 DEBUG --- status code --- 03:04:52 DEBUG --- status code --- 03:04:52 DEBUG --- status code --- 03:04:52 DEBUG --- status code --- 03:04:52 DEBUG --- http response --- 03:04:52 INFO [http_cmd]: http status code OK 03:04:52 INFO [http_cmd]: http status code OK 03:04:52 DEBUG --- status code --- 03:04:52 DEBUG --- status code --- 03:04:52 DEBUG --- status code --- 03:04:52 DEBUG --- status code --- 03:04:52 INFO [http_cmd]: http status code OK 03:04:52 DEBUG --- status code --- 03:04:52 INFO [http_cmd]: http status code OK 03:04:52 DEBUG --- status code --- 03:04:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 03:04:52 DEBUG --- status code --- 03:04:52 DEBUG --- status code --- 03:04:52 DEBUG --- stdout --- 03:04:52 DEBUG --- status code --- 03:04:52 DEBUG --- status code --- 03:04:52 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 65m 0% 6289Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 8616m 28% 7790Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 17853m 59% 11520Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 84m 0% 3364Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 11029m 36% 13479Mi 12% gke-xlou-cdm-default-pool-1b0769c8-wpwl 159m 0% 5255Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 18147m 60% 11975Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 18877m 63% 3208Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 20327m 68% 10867Mi 9% gke-xlou-cdm-ds-0f601751-j488 9291m 31% 7363Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 10215m 34% 15201Mi 13% gke-xlou-cdm-ds-0f601751-nj6q 9203m 30% 8019Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 11624m 38% 12981Mi 11% gke-xlou-cdm-frontend-a8771548-i82n 12008m 75% 11028Mi 18% 03:04:52 DEBUG --- status code --- 03:04:52 DEBUG --- status code --- 03:04:52 DEBUG --- status code --- 03:04:52 DEBUG --- status code --- 03:04:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:04:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:04:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:04:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:04:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:04:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:04:52 DEBUG --- status code --- 03:04:52 DEBUG --- status code --- 03:04:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:04:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:04:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:04:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:04:52 DEBUG --- http response --- 03:04:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:04:52 DEBUG --- status code --- 03:04:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:04:52 DEBUG ----- output ----- 03:04:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:04:52 DEBUG --- http response --- 03:04:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:04:52 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 17285m 8731Mi am-767f6b67b5-qcqz2 18105m 9068Mi am-767f6b67b5-vbltz 19509m 8639Mi ds-cts-0 11031m 10112Mi ds-cts-1 10210m 10291Mi ds-cts-2 10760m 9825Mi ds-idrepo-0 8205m 3930Mi ds-idrepo-1 8255m 3954Mi ds-idrepo-2 8527m 3915Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1179Mi idm-b85f9d978-wnhxb 3m 1239Mi lodemon-54c87f8cbb-jzrjs 15m 62Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-68c5d5b884-p97w5 11966m 657Mi 03:04:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:04:52 DEBUG --- stderr --- 03:04:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:04:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:04:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:04:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:04:52 DEBUG --- http response --- 03:04:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 03:04:52 DEBUG --- http response --- 03:04:52 DEBUG --- http response --- 03:04:52 DEBUG --- http response --- 03:04:52 DEBUG ----- output ----- 03:04:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:04:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:04:52 DEBUG --- http response --- 03:04:52 DEBUG --- http response --- 03:04:52 DEBUG --- http response --- 03:04:52 DEBUG ----- output ----- 03:04:52 DEBUG --- status code --- 03:04:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:04:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:04:52 DEBUG --- http response --- 03:04:52 DEBUG { 03:04:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:04:52 DEBUG --- http response --- 03:04:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:04:52 DEBUG --- stderr --- 03:04:52 DEBUG --- http response --- 03:04:52 DEBUG 03:04:52 DEBUG --- http response --- 03:04:52 DEBUG --- http response --- 03:04:52 DEBUG --- http response --- 03:04:52 DEBUG ----- output ----- 03:04:52 DEBUG --- http response --- 03:04:52 DEBUG ----- output ----- 03:04:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:04:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:04:52 DEBUG ----- output ----- 03:04:52 DEBUG { 03:04:52 DEBUG --- http response --- 03:04:52 DEBUG --- http response --- 03:04:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:04:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:04:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 03:04:52 DEBUG { 03:04:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:04:52 DEBUG --- http response --- 03:04:52 DEBUG --- http response --- 03:04:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:04:52 DEBUG "status": "success", 03:04:52 DEBUG --- http response --- 03:04:52 DEBUG ----- output ----- 03:04:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:04:52 DEBUG 03:04:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:04:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:04:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:04:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:04:52 DEBUG { 03:04:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:04:52 DEBUG ----- output ----- 03:04:52 DEBUG { 03:04:52 DEBUG "status": "success", 03:04:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 03:04:52 DEBUG "status": "success", 03:04:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:04:52 DEBUG ----- output ----- 03:04:52 DEBUG { 03:04:52 DEBUG ----- output ----- 03:04:52 DEBUG ----- output ----- 03:04:52 DEBUG "status": "success", 03:04:52 DEBUG --- http response --- 03:04:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:04:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:04:52 DEBUG ----- output ----- 03:04:52 DEBUG "data": { 03:04:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:04:52 DEBUG { 03:04:52 DEBUG ----- output ----- 03:04:52 DEBUG ----- output ----- 03:04:52 DEBUG ----- output ----- 03:04:52 DEBUG ----- output ----- 03:04:52 DEBUG ----- output ----- 03:04:52 DEBUG "status": "success", 03:04:52 DEBUG { 03:04:52 DEBUG { 03:04:52 DEBUG "data": { 03:04:52 DEBUG ----- output ----- 03:04:52 DEBUG ----- output ----- 03:04:52 DEBUG "data": { 03:04:52 DEBUG ----- output ----- 03:04:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:04:52 DEBUG "status": "success", 03:04:52 DEBUG { 03:04:52 DEBUG { 03:04:52 DEBUG "status": "success", 03:04:52 DEBUG ----- output ----- 03:04:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:04:52 DEBUG ----- output ----- 03:04:52 DEBUG { 03:04:52 DEBUG "resultType": "vector", 03:04:52 DEBUG ----- output ----- 03:04:52 DEBUG "status": "success", 03:04:52 DEBUG { 03:04:52 DEBUG { 03:04:52 DEBUG { 03:04:52 DEBUG { 03:04:52 DEBUG { 03:04:52 DEBUG "data": { 03:04:52 DEBUG "status": "success", 03:04:52 DEBUG "status": "success", 03:04:52 DEBUG "resultType": "vector", 03:04:52 DEBUG { 03:04:52 DEBUG { 03:04:52 DEBUG "resultType": "vector", 03:04:52 DEBUG { 03:04:52 DEBUG ----- output ----- 03:04:52 DEBUG "data": { 03:04:52 DEBUG "status": "success", 03:04:52 DEBUG "data": { 03:04:52 DEBUG { 03:04:52 DEBUG "data": { 03:04:52 DEBUG ----- output ----- 03:04:52 DEBUG { 03:04:52 DEBUG "status": "success", 03:04:52 DEBUG "result": [ 03:04:52 DEBUG { 03:04:52 DEBUG "data": { 03:04:52 DEBUG "status": "success", 03:04:52 DEBUG "status": "success", 03:04:52 DEBUG "status": "success", 03:04:52 DEBUG "status": "success", 03:04:52 DEBUG "status": "success", 03:04:52 DEBUG "resultType": "vector", 03:04:52 DEBUG "data": { 03:04:52 DEBUG "data": { 03:04:52 DEBUG "result": [ 03:04:52 DEBUG "status": "success", 03:04:52 DEBUG "status": "success", 03:04:52 DEBUG "result": [ 03:04:52 DEBUG "status": "success", 03:04:52 DEBUG { 03:04:52 DEBUG "resultType": "vector", 03:04:52 DEBUG "data": { 03:04:52 DEBUG "resultType": "vector", 03:04:52 DEBUG "status": "success", 03:04:52 DEBUG "resultType": "vector", 03:04:52 DEBUG { 03:04:52 DEBUG "status": "success", 03:04:52 DEBUG "data": { 03:04:52 DEBUG { 03:04:52 DEBUG "status": "success", 03:04:52 DEBUG "resultType": "vector", 03:04:52 DEBUG "data": { 03:04:52 DEBUG "data": { 03:04:52 DEBUG "data": { 03:04:52 DEBUG "data": { 03:04:52 DEBUG "data": { 03:04:52 DEBUG "result": [ 03:04:52 DEBUG "resultType": "vector", 03:04:52 DEBUG "resultType": "vector", 03:04:52 DEBUG { 03:04:52 DEBUG "data": { 03:04:52 DEBUG "data": { 03:04:52 DEBUG { 03:04:52 DEBUG "data": { 03:04:52 DEBUG "status": "success", 03:04:52 DEBUG "result": [ 03:04:52 DEBUG "resultType": "vector", 03:04:52 DEBUG "result": [ 03:04:52 DEBUG "data": { 03:04:52 DEBUG "result": [ 03:04:52 DEBUG "status": "success", 03:04:52 DEBUG "data": { 03:04:52 DEBUG "resultType": "vector", 03:04:52 DEBUG "metric": { 03:04:52 DEBUG "data": { 03:04:52 DEBUG "result": [ 03:04:52 DEBUG "resultType": "vector", 03:04:52 DEBUG "resultType": "vector", 03:04:52 DEBUG "resultType": "vector", 03:04:52 DEBUG "resultType": "vector", 03:04:52 DEBUG "resultType": "vector", 03:04:52 DEBUG { 03:04:52 DEBUG "result": [ 03:04:52 DEBUG "result": [ 03:04:52 DEBUG "metric": { 03:04:52 DEBUG "resultType": "vector", 03:04:52 DEBUG "resultType": "vector", 03:04:52 DEBUG "metric": { 03:04:52 DEBUG "resultType": "vector", 03:04:52 DEBUG "data": { 03:04:52 DEBUG { 03:04:52 DEBUG "result": [ 03:04:52 DEBUG { 03:04:52 DEBUG "resultType": "vector", 03:04:52 DEBUG { 03:04:52 DEBUG "data": { 03:04:52 DEBUG "result": [ 03:04:52 DEBUG "resultType": "vector", 03:04:52 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:04:52 DEBUG "resultType": "vector", 03:04:52 DEBUG { 03:04:52 DEBUG "result": [ 03:04:52 DEBUG "result": [ 03:04:52 DEBUG "result": [ 03:04:52 DEBUG "result": [ 03:04:52 DEBUG "result": [ 03:04:52 DEBUG { 03:04:52 DEBUG "metric": { 03:04:52 DEBUG { 03:04:52 DEBUG "pod": "am-767f6b67b5-9zw22" 03:04:52 DEBUG "result": [ 03:04:52 DEBUG "result": [ 03:04:52 DEBUG "instance": "10.142.0.107:9100" 03:04:52 DEBUG "result": [ 03:04:52 DEBUG "resultType": "vector", 03:04:52 DEBUG "metric": { 03:04:52 DEBUG { 03:04:52 DEBUG "metric": { 03:04:52 DEBUG "result": [ 03:04:52 DEBUG "metric": { 03:04:52 DEBUG "resultType": "vector", 03:04:52 DEBUG { 03:04:52 DEBUG "result": [ 03:04:52 DEBUG }, 03:04:52 DEBUG "result": [ 03:04:52 DEBUG "metric": { 03:04:52 DEBUG { 03:04:52 DEBUG { 03:04:52 DEBUG { 03:04:52 DEBUG { 03:04:52 DEBUG { 03:04:52 DEBUG "metric": { 03:04:52 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:04:52 DEBUG "metric": { 03:04:52 DEBUG }, 03:04:52 DEBUG { 03:04:52 DEBUG { 03:04:52 DEBUG }, 03:04:52 DEBUG { 03:04:52 DEBUG "result": [ 03:04:52 DEBUG "pod": "am-767f6b67b5-9zw22" 03:04:52 DEBUG "metric": { 03:04:52 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:04:52 DEBUG { 03:04:52 DEBUG "instance": "10.142.0.107:9100" 03:04:52 DEBUG "result": [ 03:04:52 DEBUG "metric": { 03:04:52 DEBUG { 03:04:52 DEBUG "value": [ 03:04:52 DEBUG { 03:04:52 DEBUG "pod": "ds-idrepo-0" 03:04:52 DEBUG "metric": { 03:04:52 DEBUG "metric": { 03:04:52 DEBUG "metric": { 03:04:52 DEBUG "metric": { 03:04:52 DEBUG "metric": { 03:04:52 DEBUG "instance": "10.142.0.107:9100" 03:04:52 DEBUG }, 03:04:52 DEBUG "pod": "ds-idrepo-0" 03:04:52 DEBUG "value": [ 03:04:52 DEBUG "metric": { 03:04:52 DEBUG "metric": { 03:04:52 DEBUG "value": [ 03:04:52 DEBUG "metric": { 03:04:52 DEBUG { 03:04:52 DEBUG }, 03:04:52 DEBUG "instance": "10.142.0.107:9100" 03:04:52 DEBUG }, 03:04:52 DEBUG "metric": { 03:04:52 DEBUG }, 03:04:52 DEBUG { 03:04:52 DEBUG "pod": "ds-cts-0" 03:04:52 DEBUG "metric": { 03:04:52 DEBUG 03:04:52 DEBUG "metric": { 03:04:52 DEBUG }, 03:04:52 DEBUG "pod": "am-767f6b67b5-9zw22" 03:04:52 DEBUG "pod": "am-767f6b67b5-9zw22" 03:04:52 DEBUG "pod": "am-767f6b67b5-9zw22" 03:04:52 DEBUG "pod": "ds-idrepo-0" 03:04:52 DEBUG "pod": "ds-cts-0" 03:04:52 DEBUG }, 03:04:52 DEBUG "value": [ 03:04:52 DEBUG }, 03:04:52 DEBUG 03:04:52 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:04:52 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:04:52 DEBUG 03:04:52 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:04:52 DEBUG }, 03:04:52 DEBUG "value": [ 03:04:52 DEBUG }, 03:04:52 DEBUG "value": [ 03:04:52 DEBUG "pod": "ds-cts-0" 03:04:52 DEBUG "value": [ 03:04:52 DEBUG "metric": { 03:04:52 DEBUG }, 03:04:52 DEBUG "pod": "am-767f6b67b5-9zw22" 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG "value": [ 03:04:52 DEBUG }, 03:04:52 DEBUG }, 03:04:52 DEBUG }, 03:04:52 DEBUG }, 03:04:52 DEBUG }, 03:04:52 DEBUG "value": [ 03:04:52 DEBUG 03:04:52 DEBUG "value": [ 03:04:52 DEBUG 03:04:52 DEBUG }, 03:04:52 DEBUG }, 03:04:52 DEBUG 03:04:52 DEBUG "metric": { 03:04:52 DEBUG "value": [ 03:04:52 DEBUG 03:04:52 DEBUG "value": [ 03:04:52 DEBUG 03:04:52 DEBUG }, 03:04:52 DEBUG 03:04:52 DEBUG "pod": "ds-idrepo-0" 03:04:52 DEBUG "value": [ 03:04:52 DEBUG }, 03:04:52 DEBUG "pod": "am-767f6b67b5-9zw22" 03:04:52 DEBUG [...] 03:04:52 DEBUG 03:04:52 DEBUG "value": [ 03:04:52 DEBUG "value": [ 03:04:52 DEBUG "value": [ 03:04:52 DEBUG "value": [ 03:04:52 DEBUG "value": [ 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG "value": [ 03:04:52 DEBUG "value": [ 03:04:52 DEBUG 03:04:52 DEBUG "instance": "10.142.0.107:9100" 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG "value": [ 03:04:52 DEBUG 03:04:52 DEBUG }, 03:04:52 DEBUG 03:04:52 DEBUG "value": [ 03:04:52 DEBUG }, 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG [...] 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG [...] 03:04:52 DEBUG }, 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG "value": [ 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG [...] 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG "value": [ 03:04:52 DEBUG 03:04:52 DEBUG [...] 03:04:52 DEBUG 03:04:52 DEBUG [...] 03:04:52 DEBUG "value": [ 03:04:52 DEBUG 03:04:52 DEBUG [...] 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG [...] 03:04:52 DEBUG 03:04:52 DEBUG [...] 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG [...] 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG "instance": "10.142.0.43:9100" 03:04:52 DEBUG }, 03:04:52 DEBUG [...] 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG [...] 03:04:52 DEBUG 03:04:52 DEBUG [...] 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG [...] 03:04:52 DEBUG [...] 03:04:52 DEBUG [...] 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG [...] 03:04:52 DEBUG 03:04:52 DEBUG [...] 03:04:52 DEBUG [...] 03:04:52 DEBUG [...] 03:04:52 DEBUG 03:04:52 DEBUG "value": [ 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG [...] 03:04:52 DEBUG 03:04:52 DEBUG }, 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:04:52 DEBUG 03:04:52 DEBUG "pod": "am-767f6b67b5-vbltz" 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG 1713665092, 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG [...] 03:04:52 DEBUG "value": [ 03:04:52 DEBUG 03:04:52 DEBUG [...] 03:04:52 DEBUG "pod": "am-767f6b67b5-vbltz" 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG }, 03:04:52 DEBUG 03:04:52 DEBUG }, 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG "85393494.83333334" 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:04:52 DEBUG [...] 03:04:52 DEBUG 03:04:52 DEBUG "pod": "ds-cts-2" 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG 1713665092, 03:04:52 DEBUG "pod": "ds-cts-2" 03:04:52 DEBUG 03:04:52 DEBUG }, 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG "pod": "ds-cts-2" 03:04:52 DEBUG "value": [ 03:04:52 DEBUG 03:04:52 DEBUG "value": [ 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG "pod": "am-767f6b67b5-vbltz" 03:04:52 DEBUG ] 03:04:52 DEBUG [...] 03:04:52 DEBUG "instance": "10.142.0.43:9100" 03:04:52 DEBUG }, 03:04:52 DEBUG 03:04:52 DEBUG "instance": "10.142.0.43:9100" 03:04:52 DEBUG }, 03:04:52 DEBUG "pod": "ds-cts-2" 03:04:52 DEBUG 03:04:52 DEBUG 03:04:52 DEBUG "0" 03:04:52 DEBUG }, 03:04:52 DEBUG 03:04:52 DEBUG "value": [ 03:04:52 DEBUG "pod": "am-767f6b67b5-vbltz" 03:04:52 DEBUG "pod": "ds-idrepo-2" 03:04:52 DEBUG "instance": "10.142.0.43:9100" 03:04:52 DEBUG }, 03:04:52 DEBUG 1713665092, 03:04:52 DEBUG "pod": "ds-idrepo-2" 03:04:52 DEBUG 1713665092, 03:04:52 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:04:52 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:04:52 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:04:52 DEBUG }, 03:04:52 DEBUG } 03:04:52 DEBUG 03:04:52 DEBUG }, 03:04:52 DEBUG "value": [ 03:04:52 DEBUG 03:04:52 DEBUG }, 03:04:52 DEBUG "value": [ 03:04:52 DEBUG }, 03:04:52 DEBUG "pod": "am-767f6b67b5-vbltz" 03:04:52 DEBUG 03:04:52 DEBUG ] 03:04:52 DEBUG "value": [ 03:04:52 DEBUG 03:04:52 DEBUG 1713665092, 03:04:52 DEBUG }, 03:04:52 DEBUG }, 03:04:52 DEBUG }, 03:04:52 DEBUG "value": [ 03:04:52 DEBUG "8.474407648482462" 03:04:52 DEBUG }, 03:04:52 DEBUG "3314.6666666666674" 03:04:52 DEBUG }, 03:04:52 DEBUG }, 03:04:52 DEBUG }, 03:04:52 DEBUG "value": [ 03:04:52 DEBUG ] 03:04:52 DEBUG 03:04:52 DEBUG "value": [ 03:04:52 DEBUG 1713665092, 03:04:52 DEBUG 03:04:52 DEBUG "value": [ 03:04:52 DEBUG 1713665092, 03:04:52 DEBUG "value": [ 03:04:52 DEBUG }, 03:04:52 DEBUG "pod": "am-767f6b67b5-vbltz" 03:04:52 DEBUG } 03:04:52 DEBUG 1713665092, 03:04:52 DEBUG "pod": "am-767f6b67b5-vbltz" 03:04:52 DEBUG "3321.6" 03:04:52 DEBUG "value": [ 03:04:52 DEBUG "value": [ 03:04:52 DEBUG "value": [ 03:04:52 DEBUG 1713665092, 03:04:52 DEBUG ] 03:04:52 DEBUG "value": [ 03:04:52 DEBUG ] 03:04:52 DEBUG "value": [ 03:04:52 DEBUG "value": [ 03:04:52 DEBUG "value": [ 03:04:52 DEBUG 1713665092, 03:04:52 DEBUG } 03:04:52 DEBUG 03:04:52 DEBUG 1713665092, 03:04:52 DEBUG "112906.10955414012" 03:04:52 DEBUG "pod": "ds-idrepo-2" 03:04:52 DEBUG 1713665092, 03:04:52 DEBUG "0" 03:04:52 DEBUG 1713665092, 03:04:52 DEBUG "value": [ 03:04:52 DEBUG }, 03:04:52 DEBUG ] 03:04:52 DEBUG "0" 03:04:52 DEBUG }, 03:04:52 DEBUG ] 03:04:52 DEBUG 1713665092, 03:04:52 DEBUG 1713665092, 03:04:52 DEBUG 1713665092, 03:04:52 DEBUG "4322846168064" 03:04:52 DEBUG } 03:04:52 DEBUG 1713665092, 03:04:52 DEBUG } 03:04:52 DEBUG 1713665092, 03:04:52 DEBUG 1713665092, 03:04:52 DEBUG 1713665092, 03:04:52 DEBUG "2217.433333333334" 03:04:52 DEBUG } 03:04:52 DEBUG "instance": "10.142.0.43:9100" 03:04:52 DEBUG "0.06453333333338378" 03:04:52 DEBUG ] 03:04:52 DEBUG }, 03:04:52 DEBUG "0.03833333333334016" 03:04:52 DEBUG ] 03:04:52 DEBUG "0" 03:04:52 DEBUG 1713665092, 03:04:52 DEBUG "value": [ 03:04:52 DEBUG } 03:04:52 DEBUG ] 03:04:52 DEBUG "value": [ 03:04:52 DEBUG } 03:04:52 DEBUG "3314.733333333334" 03:04:52 DEBUG "0" 03:04:52 DEBUG "0" 03:04:52 DEBUG ] 03:04:52 DEBUG ] 03:04:52 DEBUG "0" 03:04:52 DEBUG ] 03:04:52 DEBUG "11.755892832253727" 03:04:52 DEBUG "0" 03:04:52 DEBUG "784846848" 03:04:52 DEBUG ] 03:04:52 DEBUG }, 03:04:52 DEBUG ] 03:04:52 DEBUG } 03:04:52 DEBUG "value": [ 03:04:52 DEBUG ] 03:04:52 DEBUG } 03:04:52 DEBUG ] 03:04:52 DEBUG "0" 03:04:52 DEBUG 1713665092, 03:04:52 DEBUG } 03:04:52 DEBUG } 03:04:52 DEBUG 1713665092, 03:04:52 DEBUG "583280.5" 03:04:52 DEBUG ] 03:04:52 DEBUG ] 03:04:52 DEBUG ] 03:04:52 DEBUG } 03:04:52 DEBUG } 03:04:52 DEBUG ] 03:04:52 DEBUG } 03:04:52 DEBUG ] 03:04:52 DEBUG ] 03:04:52 DEBUG ] 03:04:52 DEBUG } 03:04:52 DEBUG "value": [ 03:04:52 DEBUG } 03:04:52 DEBUG ] 03:04:52 DEBUG 1713665092, 03:04:52 DEBUG } 03:04:52 DEBUG ] 03:04:52 DEBUG } 03:04:52 DEBUG ] 03:04:52 DEBUG "149.888" 03:04:52 DEBUG ] 03:04:52 DEBUG } 03:04:52 DEBUG } 03:04:52 DEBUG ] 03:04:52 DEBUG } 03:04:52 DEBUG } 03:04:52 DEBUG ] 03:04:52 DEBUG } 03:04:52 DEBUG } 03:04:52 DEBUG ] 03:04:52 DEBUG } 03:04:52 DEBUG } 03:04:52 DEBUG } 03:04:52 DEBUG ] 03:04:52 DEBUG } 03:04:52 DEBUG ] 03:04:52 DEBUG } 03:04:52 DEBUG "0" 03:04:52 DEBUG ] 03:04:52 DEBUG } 03:04:52 DEBUG ] 03:04:52 DEBUG } 03:04:52 DEBUG ] 03:04:52 DEBUG } 03:04:52 DEBUG ] 03:04:52 DEBUG } 03:04:52 DEBUG ] 03:04:52 DEBUG } 03:04:52 DEBUG } 03:04:52 DEBUG ] 03:04:52 DEBUG } 03:04:52 DEBUG ] 03:04:52 DEBUG } 03:04:52 DEBUG } 03:04:52 DEBUG ] 03:04:52 DEBUG } 03:04:52 DEBUG } 03:04:52 DEBUG ] 03:04:52 DEBUG 1713665092, 03:04:52 DEBUG } 03:04:52 DEBUG } 03:04:52 DEBUG } 03:04:52 DEBUG ] 03:04:52 DEBUG } 03:04:52 DEBUG ] 03:04:52 DEBUG } 03:04:52 DEBUG ] 03:04:52 DEBUG } 03:04:52 DEBUG } 03:04:52 DEBUG } 03:04:52 DEBUG } 03:04:52 DEBUG } 03:04:52 DEBUG ] 03:04:52 DEBUG } 03:04:52 DEBUG } 03:04:52 DEBUG } 03:04:52 DEBUG "85395239.86666666" 03:04:52 DEBUG } 03:04:52 DEBUG } 03:04:52 DEBUG } 03:04:52 DEBUG } 03:04:52 DEBUG } 03:04:52 DEBUG } 03:04:52 DEBUG } 03:04:52 DEBUG } 03:04:52 DEBUG } 03:04:52 DEBUG } 03:04:52 DEBUG } 03:04:52 DEBUG ] 03:04:52 DEBUG ] 03:04:52 DEBUG ] 03:04:52 DEBUG } 03:04:52 DEBUG } 03:04:52 DEBUG } 03:04:52 DEBUG } 03:04:52 DEBUG } 03:04:52 DEBUG } 03:04:52 DEBUG } 03:04:52 DEBUG ] 03:04:52 DEBUG } 03:04:52 DEBUG } 03:04:52 DEBUG } 03:05:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665152" 03:05:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713665152" 03:05:52 INFO [http_cmd]: http status code OK 03:05:52 ERROR [http_cmd]: ERROR 03:05:52 DEBUG --- status code --- 03:05:52 ERROR --- status code --- 03:05:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:05:52 ERROR http status code is 400 (expected [200, 401, 403, 404]) 03:05:52 DEBUG --- http response --- 03:05:52 ERROR --- http response --- 03:05:52 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 03:05:52 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 03:05:52 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 03:05:52 INFO 03:05:52 INFO [loop_until]: kubectl --namespace=xlou top node 03:05:52 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:05:52 INFO 03:05:52 INFO [loop_until]: kubectl --namespace=xlou top pods 03:05:52 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:05:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665152" 03:05:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665152" 03:05:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713665152" 03:05:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665152" 03:05:52 INFO [loop_until]: OK (rc = 0) 03:05:52 DEBUG --- stdout --- 03:05:52 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 22097m 8783Mi am-767f6b67b5-qcqz2 22221m 9120Mi am-767f6b67b5-vbltz 23615m 8708Mi ds-cts-0 20788m 11956Mi ds-cts-1 20656m 11766Mi ds-cts-2 21128m 11797Mi ds-idrepo-0 9486m 3930Mi ds-idrepo-1 9605m 3946Mi ds-idrepo-2 9784m 3915Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1179Mi idm-b85f9d978-wnhxb 3m 1239Mi lodemon-54c87f8cbb-jzrjs 2m 62Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-68c5d5b884-p97w5 12005m 770Mi 03:05:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713665152" 03:05:52 DEBUG --- stderr --- 03:05:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665152" 03:05:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665152" 03:05:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665152" 03:05:52 DEBUG 03:05:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665152" 03:05:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713665152" 03:05:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665152" 03:05:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665152" 03:05:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665152" 03:05:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665152" 03:05:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665152" 03:05:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665152" 03:05:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665152" 03:05:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665152" 03:05:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665152" 03:05:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665152" 03:05:52 INFO [loop_until]: OK (rc = 0) 03:05:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665152" 03:05:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665152" 03:05:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665152" 03:05:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713665152" 03:05:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665152" 03:05:52 INFO [http_cmd]: http status code OK 03:05:52 INFO [http_cmd]: http status code OK 03:05:52 INFO [http_cmd]: http status code OK 03:05:52 INFO [http_cmd]: http status code OK 03:05:52 INFO [http_cmd]: http status code OK 03:05:52 INFO [http_cmd]: http status code OK 03:05:52 INFO [http_cmd]: http status code OK 03:05:52 INFO [http_cmd]: http status code OK 03:05:52 INFO [http_cmd]: http status code OK 03:05:52 INFO [http_cmd]: http status code OK 03:05:52 INFO [http_cmd]: http status code OK 03:05:52 INFO [http_cmd]: http status code OK 03:05:52 INFO [http_cmd]: http status code OK 03:05:52 INFO [http_cmd]: http status code OK 03:05:52 DEBUG --- stdout --- 03:05:52 INFO [http_cmd]: http status code OK 03:05:52 INFO [http_cmd]: http status code OK 03:05:52 DEBUG --- status code --- 03:05:52 INFO [http_cmd]: http status code OK 03:05:52 INFO [http_cmd]: http status code OK 03:05:52 INFO [http_cmd]: http status code OK 03:05:52 DEBUG --- status code --- 03:05:52 DEBUG --- status code --- 03:05:52 DEBUG --- status code --- 03:05:52 DEBUG --- status code --- 03:05:52 INFO [http_cmd]: http status code OK 03:05:52 DEBUG --- status code --- 03:05:52 DEBUG --- status code --- 03:05:52 DEBUG --- status code --- 03:05:52 DEBUG --- status code --- 03:05:52 INFO [http_cmd]: http status code OK 03:05:52 INFO [http_cmd]: http status code OK 03:05:52 DEBUG --- status code --- 03:05:52 DEBUG --- status code --- 03:05:52 DEBUG --- status code --- 03:05:52 DEBUG --- status code --- 03:05:52 DEBUG --- status code --- 03:05:52 DEBUG --- status code --- 03:05:52 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 67m 0% 6288Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 9982m 33% 7778Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 21219m 71% 11588Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 84m 0% 3373Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 20376m 68% 15271Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 152m 0% 5261Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 21558m 72% 12048Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 19580m 65% 3202Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 24482m 82% 10958Mi 9% gke-xlou-cdm-ds-0f601751-j488 10304m 34% 7355Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 20345m 68% 16759Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 10112m 33% 8006Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20720m 69% 14530Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 11961m 75% 11138Mi 18% 03:05:52 DEBUG --- status code --- 03:05:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:05:52 INFO [http_cmd]: http status code OK 03:05:52 DEBUG --- status code --- 03:05:52 DEBUG --- status code --- 03:05:52 DEBUG --- status code --- 03:05:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:05:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:05:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:05:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:05:52 DEBUG --- status code --- 03:05:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:05:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:05:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:05:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:05:52 INFO [http_cmd]: http status code OK 03:05:52 DEBUG --- status code --- 03:05:52 DEBUG --- status code --- 03:05:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:05:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:05:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:05:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:05:52 DEBUG --- http response --- 03:05:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:05:52 DEBUG --- stderr --- 03:05:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:05:52 DEBUG --- http response --- 03:05:52 DEBUG --- status code --- 03:05:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:05:52 INFO [http_cmd]: http status code OK 03:05:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:05:52 DEBUG --- http response --- 03:05:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:05:52 DEBUG --- http response --- 03:05:52 DEBUG --- http response --- 03:05:52 DEBUG --- http response --- 03:05:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:05:52 DEBUG --- http response --- 03:05:52 DEBUG --- http response --- 03:05:52 DEBUG --- http response --- 03:05:52 DEBUG --- http response --- 03:05:52 DEBUG --- status code --- 03:05:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:05:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:05:52 DEBUG --- http response --- 03:05:52 DEBUG --- http response --- 03:05:52 DEBUG --- http response --- 03:05:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:05:52 DEBUG --- http response --- 03:05:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:05:52 DEBUG 03:05:52 DEBUG --- http response --- 03:05:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 03:05:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:05:52 DEBUG --- http response --- 03:05:52 DEBUG --- status code --- 03:05:52 DEBUG --- http response --- 03:05:52 DEBUG --- http response --- 03:05:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:05:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:05:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:05:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:05:52 DEBUG --- http response --- 03:05:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:05:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:05:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:05:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:05:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:05:52 DEBUG --- http response --- 03:05:52 DEBUG --- http response --- 03:05:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:05:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:05:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:05:52 DEBUG --- http response --- 03:05:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 03:05:52 DEBUG ----- output ----- 03:05:52 DEBUG ----- output ----- 03:05:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:05:52 DEBUG --- http response --- 03:05:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:05:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:05:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:05:52 DEBUG ----- output ----- 03:05:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:05:52 DEBUG ----- output ----- 03:05:52 DEBUG ----- output ----- 03:05:52 DEBUG ----- output ----- 03:05:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:05:52 DEBUG ----- output ----- 03:05:52 DEBUG ----- output ----- 03:05:52 DEBUG ----- output ----- 03:05:52 DEBUG ----- output ----- 03:05:52 DEBUG --- http response --- 03:05:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:05:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:05:52 DEBUG ----- output ----- 03:05:52 DEBUG ----- output ----- 03:05:52 DEBUG ----- output ----- 03:05:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:05:52 DEBUG ----- output ----- 03:05:52 DEBUG { 03:05:52 DEBUG "status": "success", 03:05:52 DEBUG ----- output ----- 03:05:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:05:52 DEBUG ----- output ----- 03:05:52 DEBUG --- http response --- 03:05:52 DEBUG ----- output ----- 03:05:52 DEBUG { 03:05:52 DEBUG ----- output ----- 03:05:52 DEBUG { 03:05:52 DEBUG { 03:05:52 DEBUG "status": "success", 03:05:52 DEBUG ----- output ----- 03:05:52 DEBUG { 03:05:52 DEBUG { 03:05:52 DEBUG { 03:05:52 DEBUG { 03:05:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:05:52 DEBUG ----- output ----- 03:05:52 DEBUG ----- output ----- 03:05:52 DEBUG { 03:05:52 DEBUG { 03:05:52 DEBUG "status": "success", 03:05:52 DEBUG ----- output ----- 03:05:52 DEBUG { 03:05:52 DEBUG "data": { 03:05:52 DEBUG { 03:05:52 DEBUG { 03:05:52 DEBUG ----- output ----- 03:05:52 DEBUG { 03:05:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:05:52 DEBUG { 03:05:52 DEBUG "status": "success", 03:05:52 DEBUG { 03:05:52 DEBUG "status": "success", 03:05:52 DEBUG "data": { 03:05:52 DEBUG "resultType": "vector", 03:05:52 DEBUG "result": [ 03:05:52 DEBUG "status": "success", 03:05:52 DEBUG "status": "success", 03:05:52 DEBUG "status": "success", 03:05:52 DEBUG "status": "success", 03:05:52 DEBUG ----- output ----- 03:05:52 DEBUG { 03:05:52 DEBUG { 03:05:52 DEBUG "status": "success", 03:05:52 DEBUG "data": { 03:05:52 DEBUG "resultType": "vector", 03:05:52 DEBUG "status": "success", 03:05:52 DEBUG "resultType": "vector", 03:05:52 DEBUG "status": "success", 03:05:52 DEBUG { 03:05:52 DEBUG "status": "success", 03:05:52 DEBUG { 03:05:52 DEBUG "status": "success", 03:05:52 DEBUG ----- output ----- 03:05:52 DEBUG "status": "success", 03:05:52 DEBUG "data": { 03:05:52 DEBUG "status": "success", 03:05:52 DEBUG "data": { 03:05:52 DEBUG { 03:05:52 DEBUG { 03:05:52 DEBUG { 03:05:52 DEBUG "data": { 03:05:52 DEBUG "data": { 03:05:52 DEBUG "data": { 03:05:52 DEBUG "data": { 03:05:52 DEBUG { 03:05:52 DEBUG "status": "success", 03:05:52 DEBUG "data": { 03:05:52 DEBUG "status": "success", 03:05:52 DEBUG "result": [ 03:05:52 DEBUG { 03:05:52 DEBUG "data": { 03:05:52 DEBUG "result": [ 03:05:52 DEBUG "data": { 03:05:52 DEBUG "status": "success", 03:05:52 DEBUG "data": { 03:05:52 DEBUG "status": "success", 03:05:52 DEBUG "data": { 03:05:52 DEBUG { 03:05:52 DEBUG "data": { 03:05:52 DEBUG "resultType": "vector", 03:05:52 DEBUG "data": { 03:05:52 DEBUG "resultType": "vector", 03:05:52 DEBUG "status": "success", 03:05:52 DEBUG "metric": { 03:05:52 DEBUG "status": "success", 03:05:52 DEBUG "resultType": "vector", 03:05:52 DEBUG "resultType": "vector", 03:05:52 DEBUG "result": [ 03:05:52 DEBUG "resultType": "vector", 03:05:52 DEBUG "status": "success", 03:05:52 DEBUG "data": { 03:05:52 DEBUG "resultType": "vector", 03:05:52 DEBUG "data": { 03:05:52 DEBUG { 03:05:52 DEBUG "status": "success", 03:05:52 DEBUG "resultType": "vector", 03:05:52 DEBUG "result": [ 03:05:52 DEBUG "resultType": "vector", 03:05:52 DEBUG "data": { 03:05:52 DEBUG "resultType": "vector", 03:05:52 DEBUG "data": { 03:05:52 DEBUG "resultType": "vector", 03:05:52 DEBUG "status": "success", 03:05:52 DEBUG "resultType": "vector", 03:05:52 DEBUG "result": [ 03:05:52 DEBUG "resultType": "vector", 03:05:52 DEBUG "result": [ 03:05:52 DEBUG "data": { 03:05:52 DEBUG "pod": "ds-idrepo-0" 03:05:52 DEBUG "data": { 03:05:52 DEBUG "result": [ 03:05:52 DEBUG "resultType": "vector", 03:05:52 DEBUG { 03:05:52 DEBUG "result": [ 03:05:52 DEBUG "data": { 03:05:52 DEBUG "resultType": "vector", 03:05:52 DEBUG "result": [ 03:05:52 DEBUG "resultType": "vector", 03:05:52 DEBUG "metric": { 03:05:52 DEBUG "data": { 03:05:52 DEBUG { 03:05:52 DEBUG "metric": { 03:05:52 DEBUG "result": [ 03:05:52 DEBUG "resultType": "vector", 03:05:52 DEBUG "result": [ 03:05:52 DEBUG "resultType": "vector", 03:05:52 DEBUG "result": [ 03:05:52 DEBUG "data": { 03:05:52 DEBUG "result": [ 03:05:52 DEBUG { 03:05:52 DEBUG "result": [ 03:05:52 DEBUG { 03:05:52 DEBUG "metric": { 03:05:52 DEBUG }, 03:05:52 DEBUG "resultType": "vector", 03:05:52 DEBUG { 03:05:52 DEBUG "result": [ 03:05:52 DEBUG { 03:05:52 DEBUG { 03:05:52 DEBUG "resultType": "vector", 03:05:52 DEBUG "result": [ 03:05:52 DEBUG { 03:05:52 DEBUG "result": [ 03:05:52 DEBUG "instance": "10.142.0.107:9100" 03:05:52 DEBUG "resultType": "vector", 03:05:52 DEBUG { 03:05:52 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:05:52 DEBUG { 03:05:52 DEBUG "result": [ 03:05:52 DEBUG { 03:05:52 DEBUG "result": [ 03:05:52 DEBUG { 03:05:52 DEBUG "resultType": "vector", 03:05:52 DEBUG { 03:05:52 DEBUG "metric": { 03:05:52 DEBUG { 03:05:52 DEBUG "instance": "10.142.0.107:9100" 03:05:52 DEBUG "resultType": "vector", 03:05:52 DEBUG "value": [ 03:05:52 DEBUG "result": [ 03:05:52 DEBUG "metric": { 03:05:52 DEBUG "metric": { 03:05:52 DEBUG "metric": { 03:05:52 DEBUG "metric": { 03:05:52 DEBUG "result": [ 03:05:52 DEBUG { 03:05:52 DEBUG "metric": { 03:05:52 DEBUG { 03:05:52 DEBUG }, 03:05:52 DEBUG "result": [ 03:05:52 DEBUG "metric": { 03:05:52 DEBUG }, 03:05:52 DEBUG "metric": { 03:05:52 DEBUG { 03:05:52 DEBUG "metric": { 03:05:52 DEBUG "metric": { 03:05:52 DEBUG "pod": "ds-idrepo-0" 03:05:52 DEBUG "result": [ 03:05:52 DEBUG "metric": { 03:05:52 DEBUG "pod": "ds-idrepo-0" 03:05:52 DEBUG "metric": { 03:05:52 DEBUG }, 03:05:52 DEBUG "result": [ 03:05:52 DEBUG 03:05:52 DEBUG { 03:05:52 DEBUG "pod": "ds-cts-0" 03:05:52 DEBUG "pod": "am-767f6b67b5-9zw22" 03:05:52 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:05:52 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:05:52 DEBUG { 03:05:52 DEBUG "metric": { 03:05:52 DEBUG "pod": "am-767f6b67b5-9zw22" 03:05:52 DEBUG "metric": { 03:05:52 DEBUG "value": [ 03:05:52 DEBUG { 03:05:52 DEBUG "pod": "ds-cts-0" 03:05:52 DEBUG "value": [ 03:05:52 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:05:52 DEBUG "metric": { 03:05:52 DEBUG "instance": "10.142.0.107:9100" 03:05:52 DEBUG }, 03:05:52 DEBUG { 03:05:52 DEBUG { 03:05:52 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:05:52 DEBUG }, 03:05:52 DEBUG "instance": "10.142.0.107:9100" 03:05:52 DEBUG "value": [ 03:05:52 DEBUG { 03:05:52 DEBUG 03:05:52 DEBUG "metric": { 03:05:52 DEBUG }, 03:05:52 DEBUG }, 03:05:52 DEBUG }, 03:05:52 DEBUG }, 03:05:52 DEBUG "metric": { 03:05:52 DEBUG "instance": "10.142.0.107:9100" 03:05:52 DEBUG }, 03:05:52 DEBUG "pod": "ds-cts-0" 03:05:52 DEBUG 03:05:52 DEBUG "metric": { 03:05:52 DEBUG }, 03:05:52 DEBUG 03:05:52 DEBUG }, 03:05:52 DEBUG "pod": "ds-idrepo-0" 03:05:52 DEBUG }, 03:05:52 DEBUG "value": [ 03:05:52 DEBUG "metric": { 03:05:52 DEBUG "metric": { 03:05:52 DEBUG }, 03:05:52 DEBUG "value": [ 03:05:52 DEBUG }, 03:05:52 DEBUG "value": [ 03:05:52 DEBUG "metric": { 03:05:52 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:05:52 DEBUG "pod": "am-767f6b67b5-9zw22" 03:05:52 DEBUG "value": [ 03:05:52 DEBUG "value": [ 03:05:52 DEBUG "value": [ 03:05:52 DEBUG "value": [ 03:05:52 DEBUG 03:05:52 DEBUG }, 03:05:52 DEBUG "value": [ 03:05:52 DEBUG }, 03:05:52 DEBUG 03:05:52 DEBUG "pod": "am-767f6b67b5-9zw22" 03:05:52 DEBUG "value": [ 03:05:52 DEBUG 03:05:52 DEBUG "value": [ 03:05:52 DEBUG 03:05:52 DEBUG "value": [ 03:05:52 DEBUG 03:05:52 DEBUG "pod": "am-767f6b67b5-9zw22" 03:05:52 DEBUG "pod": "am-767f6b67b5-9zw22" 03:05:52 DEBUG "value": [ 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG }, 03:05:52 DEBUG 03:05:52 DEBUG }, 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG "value": [ 03:05:52 DEBUG 03:05:52 DEBUG "value": [ 03:05:52 DEBUG 03:05:52 DEBUG }, 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG }, 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG }, 03:05:52 DEBUG "value": [ 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG "value": [ 03:05:52 DEBUG 03:05:52 DEBUG "value": [ 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG [...] 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG [...] 03:05:52 DEBUG "value": [ 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG "value": [ 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG }, 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG [...] 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG [...] 03:05:52 DEBUG "pod": "am-767f6b67b5-9zw22" 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG [...] 03:05:52 DEBUG [...] 03:05:52 DEBUG [...] 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG [...] 03:05:52 DEBUG "value": [ 03:05:52 DEBUG 03:05:52 DEBUG [...] 03:05:52 DEBUG [...] 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG [...] 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG }, 03:05:52 DEBUG 03:05:52 DEBUG [...] 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG [...] 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG [...] 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG [...] 03:05:52 DEBUG [...] 03:05:52 DEBUG [...] 03:05:52 DEBUG 03:05:52 DEBUG [...] 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG "value": [ 03:05:52 DEBUG [...] 03:05:52 DEBUG 03:05:52 DEBUG [...] 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG [...] 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG "instance": "10.142.0.43:9100" 03:05:52 DEBUG [...] 03:05:52 DEBUG 03:05:52 DEBUG "pod": "ds-cts-2" 03:05:52 DEBUG 03:05:52 DEBUG [...] 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG "pod": "am-767f6b67b5-vbltz" 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG }, 03:05:52 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG "pod": "am-767f6b67b5-vbltz" 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG "pod": "am-767f6b67b5-vbltz" 03:05:52 DEBUG }, 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG "pod": "ds-idrepo-2" 03:05:52 DEBUG [...] 03:05:52 DEBUG }, 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:05:52 DEBUG "pod": "ds-cts-2" 03:05:52 DEBUG 03:05:52 DEBUG "pod": "am-767f6b67b5-vbltz" 03:05:52 DEBUG }, 03:05:52 DEBUG }, 03:05:52 DEBUG 03:05:52 DEBUG 03:05:52 DEBUG }, 03:05:52 DEBUG 03:05:52 DEBUG }, 03:05:52 DEBUG 03:05:52 DEBUG "pod": "am-767f6b67b5-vbltz" 03:05:52 DEBUG }, 03:05:52 DEBUG "value": [ 03:05:52 DEBUG 03:05:52 DEBUG "instance": "10.142.0.43:9100" 03:05:52 DEBUG }, 03:05:52 DEBUG 03:05:52 DEBUG "value": [ 03:05:52 DEBUG "pod": "ds-idrepo-2" 03:05:52 DEBUG "instance": "10.142.0.43:9100" 03:05:52 DEBUG 03:05:52 DEBUG "instance": "10.142.0.43:9100" 03:05:52 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:05:52 DEBUG }, 03:05:52 DEBUG }, 03:05:52 DEBUG "pod": "am-767f6b67b5-vbltz" 03:05:52 DEBUG "value": [ 03:05:52 DEBUG "value": [ 03:05:52 DEBUG "value": [ 03:05:52 DEBUG [...] 03:05:52 DEBUG "instance": "10.142.0.43:9100" 03:05:52 DEBUG "value": [ 03:05:52 DEBUG "pod": "ds-cts-2" 03:05:52 DEBUG "value": [ 03:05:52 DEBUG 03:05:52 DEBUG "pod": "am-767f6b67b5-vbltz" 03:05:52 DEBUG "value": [ 03:05:52 DEBUG 1713665152, 03:05:52 DEBUG 03:05:52 DEBUG }, 03:05:52 DEBUG "value": [ 03:05:52 DEBUG 03:05:52 DEBUG 1713665152, 03:05:52 DEBUG }, 03:05:52 DEBUG }, 03:05:52 DEBUG "pod": "ds-cts-2" 03:05:52 DEBUG }, 03:05:52 DEBUG }, 03:05:52 DEBUG "value": [ 03:05:52 DEBUG "value": [ 03:05:52 DEBUG }, 03:05:52 DEBUG 1713665152, 03:05:52 DEBUG 1713665152, 03:05:52 DEBUG 1713665152, 03:05:52 DEBUG 03:05:52 DEBUG }, 03:05:52 DEBUG 1713665152, 03:05:52 DEBUG }, 03:05:52 DEBUG "value": [ 03:05:52 DEBUG }, 03:05:52 DEBUG }, 03:05:52 DEBUG 1713665152, 03:05:52 DEBUG "0" 03:05:52 DEBUG "pod": "ds-idrepo-2" 03:05:52 DEBUG 1713665152, 03:05:52 DEBUG "value": [ 03:05:52 DEBUG 03:05:52 DEBUG "3333.6" 03:05:52 DEBUG "value": [ 03:05:52 DEBUG "value": [ 03:05:52 DEBUG }, 03:05:52 DEBUG "value": [ 03:05:52 DEBUG "value": [ 03:05:52 DEBUG 1713665152, 03:05:52 DEBUG 1713665152, 03:05:52 DEBUG "value": [ 03:05:52 DEBUG "0" 03:05:52 DEBUG "3325.133333333334" 03:05:52 DEBUG "903938048" 03:05:52 DEBUG 03:05:52 DEBUG "value": [ 03:05:52 DEBUG "0" 03:05:52 DEBUG 1713665152, 03:05:52 DEBUG 1713665152, 03:05:52 DEBUG "value": [ 03:05:52 DEBUG "value": [ 03:05:52 DEBUG "0" 03:05:52 DEBUG ] 03:05:52 DEBUG }, 03:05:52 DEBUG "0.04933333333333774" 03:05:52 DEBUG 1713665152, 03:05:52 DEBUG "pod": "am-767f6b67b5-vbltz" 03:05:52 DEBUG ] 03:05:52 DEBUG 1713665152, 03:05:52 DEBUG 1713665152, 03:05:52 DEBUG "value": [ 03:05:52 DEBUG 1713665152, 03:05:52 DEBUG "100211779.03333333" 03:05:52 DEBUG "11.71746209348358" 03:05:52 DEBUG "0" 03:05:52 DEBUG ] 03:05:52 DEBUG ] 03:05:52 DEBUG ] 03:05:52 DEBUG ] 03:05:52 DEBUG 03:05:52 DEBUG 1713665152, 03:05:52 DEBUG ] 03:05:52 DEBUG "0.4628333333333333" 03:05:52 DEBUG "4326905328640" 03:05:52 DEBUG 1713665152, 03:05:52 DEBUG 1713665152, 03:05:52 DEBUG ] 03:05:52 DEBUG } 03:05:52 DEBUG "value": [ 03:05:52 DEBUG ] 03:05:52 DEBUG "0" 03:05:52 DEBUG ] 03:05:52 DEBUG } 03:05:52 DEBUG "0" 03:05:52 DEBUG ] 03:05:52 DEBUG "0.17583333333335344" 03:05:52 DEBUG ] 03:05:52 DEBUG 1713665152, 03:05:52 DEBUG ] 03:05:52 DEBUG } 03:05:52 DEBUG 1713665152, 03:05:52 DEBUG } 03:05:52 DEBUG } 03:05:52 DEBUG } 03:05:52 DEBUG "pod": "am-767f6b67b5-vbltz" 03:05:52 DEBUG "100215403.03333333" 03:05:52 DEBUG } 03:05:52 DEBUG ] 03:05:52 DEBUG ] 03:05:52 DEBUG "848.8804648804648" 03:05:52 DEBUG "3326.633333333334" 03:05:52 DEBUG } 03:05:52 DEBUG ] 03:05:52 DEBUG 1713665152, 03:05:52 DEBUG } 03:05:52 DEBUG } 03:05:52 DEBUG }, 03:05:52 DEBUG ] 03:05:52 DEBUG 1713665152, 03:05:52 DEBUG } 03:05:52 DEBUG ] 03:05:52 DEBUG } 03:05:52 DEBUG "0" 03:05:52 DEBUG } 03:05:52 DEBUG ] 03:05:52 DEBUG "3328.0000000000005" 03:05:52 DEBUG ] 03:05:52 DEBUG ] 03:05:52 DEBUG ] 03:05:52 DEBUG }, 03:05:52 DEBUG ] 03:05:52 DEBUG ] 03:05:52 DEBUG } 03:05:52 DEBUG } 03:05:52 DEBUG ] 03:05:52 DEBUG ] 03:05:52 DEBUG ] 03:05:52 DEBUG } 03:05:52 DEBUG "0" 03:05:52 DEBUG ] 03:05:52 DEBUG ] 03:05:52 DEBUG "value": [ 03:05:52 DEBUG } 03:05:52 DEBUG "0" 03:05:52 DEBUG ] 03:05:52 DEBUG } 03:05:52 DEBUG ] 03:05:52 DEBUG ] 03:05:52 DEBUG ] 03:05:52 DEBUG } 03:05:52 DEBUG ] 03:05:52 DEBUG } 03:05:52 DEBUG } 03:05:52 DEBUG } 03:05:52 DEBUG "value": [ 03:05:52 DEBUG } 03:05:52 DEBUG } 03:05:52 DEBUG } 03:05:52 DEBUG ] 03:05:52 DEBUG } 03:05:52 DEBUG } 03:05:52 DEBUG } 03:05:52 DEBUG } 03:05:52 DEBUG ] 03:05:52 DEBUG } 03:05:52 DEBUG } 03:05:52 DEBUG 1713665152, 03:05:52 DEBUG } 03:05:52 DEBUG ] 03:05:52 DEBUG } 03:05:52 DEBUG ] 03:05:52 DEBUG } 03:05:52 DEBUG } 03:05:52 DEBUG } 03:05:52 DEBUG } 03:05:52 DEBUG } 03:05:52 DEBUG } 03:05:52 DEBUG } 03:05:52 DEBUG } 03:05:52 DEBUG 1713665152, 03:05:52 DEBUG ] 03:05:52 DEBUG ] 03:05:52 DEBUG } 03:05:52 DEBUG ] 03:05:52 DEBUG ] 03:05:52 DEBUG } 03:05:52 DEBUG } 03:05:52 DEBUG } 03:05:52 DEBUG } 03:05:52 DEBUG "599006" 03:05:52 DEBUG } 03:05:52 DEBUG } 03:05:52 DEBUG } 03:05:52 DEBUG } 03:05:52 DEBUG ] 03:05:52 DEBUG } 03:05:52 DEBUG ] 03:05:52 DEBUG "149.888" 03:05:52 DEBUG } 03:05:52 DEBUG } 03:05:52 DEBUG } 03:05:52 DEBUG } 03:05:52 DEBUG } 03:05:52 DEBUG ] 03:05:52 DEBUG ] 03:05:52 DEBUG ] 03:05:52 DEBUG } 03:05:52 DEBUG } 03:05:52 DEBUG } 03:05:52 DEBUG ] 03:05:52 DEBUG } 03:05:52 DEBUG } 03:05:52 DEBUG } 03:05:52 DEBUG } 03:05:52 DEBUG } 03:05:52 DEBUG } 03:05:52 DEBUG } 03:05:52 DEBUG } 03:05:52 DEBUG } 03:05:52 DEBUG } 03:05:52 DEBUG } 03:05:52 DEBUG ] 03:05:52 DEBUG } 03:05:52 DEBUG ] 03:05:52 DEBUG } 03:05:52 DEBUG } 03:05:52 DEBUG } 03:05:52 DEBUG } 03:06:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713665212" 03:06:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665212" 03:06:52 ERROR [http_cmd]: ERROR 03:06:52 INFO [http_cmd]: http status code OK 03:06:52 ERROR --- status code --- 03:06:52 DEBUG --- status code --- 03:06:52 ERROR http status code is 400 (expected [200, 401, 403, 404]) 03:06:52 ERROR --- http response --- 03:06:52 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 03:06:52 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 03:06:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:06:52 DEBUG --- http response --- 03:06:52 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 03:06:52 INFO 03:06:52 INFO [loop_until]: kubectl --namespace=xlou top pods 03:06:52 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:06:52 INFO 03:06:52 INFO [loop_until]: kubectl --namespace=xlou top node 03:06:52 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:06:52 INFO [loop_until]: OK (rc = 0) 03:06:52 DEBUG --- stdout --- 03:06:52 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 21849m 8789Mi am-767f6b67b5-qcqz2 22706m 9122Mi am-767f6b67b5-vbltz 24715m 8786Mi ds-cts-0 20400m 12157Mi ds-cts-1 20427m 12093Mi ds-cts-2 20588m 12051Mi ds-idrepo-0 9574m 3930Mi ds-idrepo-1 9535m 3946Mi ds-idrepo-2 9813m 3916Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1179Mi idm-b85f9d978-wnhxb 3m 1239Mi lodemon-54c87f8cbb-jzrjs 18m 62Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-68c5d5b884-p97w5 11947m 776Mi 03:06:52 DEBUG --- stderr --- 03:06:52 DEBUG 03:06:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665212" 03:06:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665212" 03:06:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665212" 03:06:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713665212" 03:06:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665212" 03:06:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713665212" 03:06:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665212" 03:06:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665212" 03:06:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665212" 03:06:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713665212" 03:06:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665212" 03:06:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665212" 03:06:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665212" 03:06:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665212" 03:06:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665212" 03:06:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665212" 03:06:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665212" 03:06:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665212" 03:06:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665212" 03:06:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665212" 03:06:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665212" 03:06:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665212" 03:06:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665212" 03:06:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713665212" 03:06:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665212" 03:06:52 INFO [loop_until]: OK (rc = 0) 03:06:52 INFO [http_cmd]: http status code OK 03:06:52 INFO [http_cmd]: http status code OK 03:06:52 INFO [http_cmd]: http status code OK 03:06:52 INFO [http_cmd]: http status code OK 03:06:52 INFO [http_cmd]: http status code OK 03:06:52 INFO [http_cmd]: http status code OK 03:06:52 INFO [http_cmd]: http status code OK 03:06:52 INFO [http_cmd]: http status code OK 03:06:52 DEBUG --- status code --- 03:06:52 INFO [http_cmd]: http status code OK 03:06:52 INFO [http_cmd]: http status code OK 03:06:52 INFO [http_cmd]: http status code OK 03:06:52 INFO [http_cmd]: http status code OK 03:06:52 INFO [http_cmd]: http status code OK 03:06:52 INFO [http_cmd]: http status code OK 03:06:52 INFO [http_cmd]: http status code OK 03:06:52 INFO [http_cmd]: http status code OK 03:06:52 INFO [http_cmd]: http status code OK 03:06:52 INFO [http_cmd]: http status code OK 03:06:52 INFO [http_cmd]: http status code OK 03:06:52 DEBUG --- stdout --- 03:06:52 INFO [http_cmd]: http status code OK 03:06:52 DEBUG --- status code --- 03:06:52 DEBUG --- status code --- 03:06:52 DEBUG --- status code --- 03:06:52 DEBUG --- status code --- 03:06:52 INFO [http_cmd]: http status code OK 03:06:52 INFO [http_cmd]: http status code OK 03:06:52 DEBUG --- status code --- 03:06:52 INFO [http_cmd]: http status code OK 03:06:52 DEBUG --- status code --- 03:06:52 DEBUG --- status code --- 03:06:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:06:52 DEBUG --- status code --- 03:06:52 DEBUG --- status code --- 03:06:52 DEBUG --- status code --- 03:06:52 INFO [http_cmd]: http status code OK 03:06:52 DEBUG --- status code --- 03:06:52 DEBUG --- status code --- 03:06:52 DEBUG --- status code --- 03:06:52 DEBUG --- status code --- 03:06:52 DEBUG --- status code --- 03:06:52 DEBUG --- status code --- 03:06:52 DEBUG --- status code --- 03:06:52 DEBUG --- status code --- 03:06:52 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 6298Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 9954m 33% 7778Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 22055m 73% 11593Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 88m 0% 3362Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 20478m 68% 15476Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 154m 0% 5272Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 22447m 75% 12053Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 19751m 66% 3202Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 24704m 82% 11034Mi 9% gke-xlou-cdm-ds-0f601751-j488 10372m 34% 7356Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 20113m 67% 17200Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 9840m 32% 8006Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20706m 69% 14767Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 11619m 73% 11144Mi 19% 03:06:52 DEBUG --- status code --- 03:06:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:06:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:06:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:06:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:06:52 DEBUG --- status code --- 03:06:52 DEBUG --- status code --- 03:06:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:06:52 DEBUG --- status code --- 03:06:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:06:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:06:52 DEBUG --- http response --- 03:06:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:06:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:06:52 DEBUG --- http response --- 03:06:52 INFO [http_cmd]: http status code OK 03:06:52 DEBUG --- status code --- 03:06:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:06:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:06:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:06:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:06:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:06:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:06:52 DEBUG --- http response --- 03:06:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:06:52 DEBUG --- stderr --- 03:06:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:06:52 DEBUG --- http response --- 03:06:52 DEBUG --- http response --- 03:06:52 DEBUG --- http response --- 03:06:52 DEBUG --- http response --- 03:06:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:06:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:06:52 DEBUG --- http response --- 03:06:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:06:52 DEBUG --- http response --- 03:06:52 DEBUG --- http response --- 03:06:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:06:52 DEBUG --- http response --- 03:06:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:06:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:06:52 DEBUG ----- output ----- 03:06:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:06:52 DEBUG --- http response --- 03:06:52 DEBUG --- http response --- 03:06:52 DEBUG --- http response --- 03:06:52 DEBUG --- http response --- 03:06:52 DEBUG --- http response --- 03:06:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:06:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:06:52 DEBUG ----- output ----- 03:06:52 DEBUG 03:06:52 DEBUG --- http response --- 03:06:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:06:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:06:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:06:52 DEBUG ----- output ----- 03:06:52 DEBUG { 03:06:52 DEBUG --- http response --- 03:06:52 DEBUG --- http response --- 03:06:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:06:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:06:52 DEBUG ----- output ----- 03:06:52 DEBUG ----- output ----- 03:06:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:06:52 DEBUG --- http response --- 03:06:52 DEBUG --- status code --- 03:06:52 DEBUG { 03:06:52 DEBUG --- http response --- 03:06:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:06:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:06:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:06:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:06:52 DEBUG --- http response --- 03:06:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:06:52 DEBUG ----- output ----- 03:06:52 DEBUG { 03:06:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:06:52 DEBUG ----- output ----- 03:06:52 DEBUG ----- output ----- 03:06:52 DEBUG --- http response --- 03:06:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:06:52 DEBUG "status": "success", 03:06:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:06:52 DEBUG ----- output ----- 03:06:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:06:52 DEBUG { 03:06:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:06:52 DEBUG { 03:06:52 DEBUG ----- output ----- 03:06:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:06:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:06:52 DEBUG "status": "success", 03:06:52 DEBUG ----- output ----- 03:06:52 DEBUG { 03:06:52 DEBUG ----- output ----- 03:06:52 DEBUG ----- output ----- 03:06:52 DEBUG { 03:06:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:06:52 DEBUG { 03:06:52 DEBUG "status": "success", 03:06:52 DEBUG ----- output ----- 03:06:52 DEBUG --- http response --- 03:06:52 DEBUG { 03:06:52 DEBUG { 03:06:52 DEBUG "status": "success", 03:06:52 DEBUG ----- output ----- 03:06:52 DEBUG { 03:06:52 DEBUG ----- output ----- 03:06:52 DEBUG { 03:06:52 DEBUG ----- output ----- 03:06:52 DEBUG "status": "success", 03:06:52 DEBUG ----- output ----- 03:06:52 DEBUG "status": "success", 03:06:52 DEBUG { 03:06:52 DEBUG ----- output ----- 03:06:52 DEBUG --- http response --- 03:06:52 DEBUG "data": { 03:06:52 DEBUG "status": "success", 03:06:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:06:52 DEBUG { 03:06:52 DEBUG ----- output ----- 03:06:52 DEBUG "status": "success", 03:06:52 DEBUG ----- output ----- 03:06:52 DEBUG "status": "success", 03:06:52 DEBUG "data": { 03:06:52 DEBUG { 03:06:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:06:52 DEBUG "status": "success", 03:06:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:06:52 DEBUG "data": { 03:06:52 DEBUG "data": { 03:06:52 DEBUG "status": "success", 03:06:52 DEBUG { 03:06:52 DEBUG "status": "success", 03:06:52 DEBUG { 03:06:52 DEBUG "status": "success", 03:06:52 DEBUG { 03:06:52 DEBUG "data": { 03:06:52 DEBUG "status": "success", 03:06:52 DEBUG { 03:06:52 DEBUG "resultType": "vector", 03:06:52 DEBUG "data": { 03:06:52 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:06:52 DEBUG ----- output ----- 03:06:52 DEBUG "status": "success", 03:06:52 DEBUG { 03:06:52 DEBUG "data": { 03:06:52 DEBUG { 03:06:52 DEBUG "data": { 03:06:52 DEBUG "resultType": "vector", 03:06:52 DEBUG "status": "success", 03:06:52 DEBUG ----- output ----- 03:06:52 DEBUG "data": { 03:06:52 DEBUG ----- output ----- 03:06:52 DEBUG "resultType": "vector", 03:06:52 DEBUG "resultType": "vector", 03:06:52 DEBUG "data": { 03:06:52 DEBUG "status": "success", 03:06:52 DEBUG "data": { 03:06:52 DEBUG "data": { 03:06:52 DEBUG "status": "success", 03:06:52 DEBUG "data": { 03:06:52 DEBUG "resultType": "vector", 03:06:52 DEBUG "data": { 03:06:52 DEBUG "status": "success", 03:06:52 DEBUG "result": [ 03:06:52 DEBUG "resultType": "vector", 03:06:52 DEBUG ----- output ----- 03:06:52 DEBUG { 03:06:52 DEBUG "data": { 03:06:52 DEBUG "status": "success", 03:06:52 DEBUG "resultType": "vector", 03:06:52 DEBUG "status": "success", 03:06:52 DEBUG "resultType": "vector", 03:06:52 DEBUG "result": [ 03:06:52 DEBUG "data": { 03:06:52 DEBUG { 03:06:52 DEBUG "resultType": "vector", 03:06:52 DEBUG { 03:06:52 DEBUG "result": [ 03:06:52 DEBUG "result": [ 03:06:52 DEBUG "resultType": "vector", 03:06:52 DEBUG "data": { 03:06:52 DEBUG "resultType": "vector", 03:06:52 DEBUG "resultType": "vector", 03:06:52 DEBUG "data": { 03:06:52 DEBUG "resultType": "vector", 03:06:52 DEBUG "result": [ 03:06:52 DEBUG "resultType": "vector", 03:06:52 DEBUG "data": { 03:06:52 DEBUG { 03:06:52 DEBUG "result": [ 03:06:52 DEBUG { 03:06:52 DEBUG "status": "success", 03:06:52 DEBUG "resultType": "vector", 03:06:52 DEBUG "data": { 03:06:52 DEBUG "result": [ 03:06:52 DEBUG "data": { 03:06:52 DEBUG "resultType": "vector", 03:06:52 DEBUG { 03:06:52 DEBUG "resultType": "vector", 03:06:52 DEBUG "status": "success", 03:06:52 DEBUG "result": [ 03:06:52 DEBUG "status": "success", 03:06:52 DEBUG { 03:06:52 DEBUG { 03:06:52 DEBUG "result": [ 03:06:52 DEBUG "resultType": "vector", 03:06:52 DEBUG "result": [ 03:06:52 DEBUG "result": [ 03:06:52 DEBUG "resultType": "vector", 03:06:52 DEBUG "result": [ 03:06:52 DEBUG { 03:06:52 DEBUG "result": [ 03:06:52 DEBUG "resultType": "vector", 03:06:52 DEBUG "metric": { 03:06:52 DEBUG "instance": "10.142.0.107:9100" 03:06:52 DEBUG "status": "success", 03:06:52 DEBUG "data": { 03:06:52 DEBUG "resultType": "vector", 03:06:52 DEBUG "resultType": "vector", 03:06:52 DEBUG { 03:06:52 DEBUG "result": [ 03:06:52 DEBUG { 03:06:52 DEBUG "metric": { 03:06:52 DEBUG "result": [ 03:06:52 DEBUG "data": { 03:06:52 DEBUG { 03:06:52 DEBUG "data": { 03:06:52 DEBUG "metric": { 03:06:52 DEBUG "metric": { 03:06:52 DEBUG { 03:06:52 DEBUG "result": [ 03:06:52 DEBUG { 03:06:52 DEBUG { 03:06:52 DEBUG "result": [ 03:06:52 DEBUG { 03:06:52 DEBUG "metric": { 03:06:52 DEBUG { 03:06:52 DEBUG "result": [ 03:06:52 DEBUG }, 03:06:52 DEBUG "data": { 03:06:52 DEBUG { 03:06:52 DEBUG "result": [ 03:06:52 DEBUG "result": [ 03:06:52 DEBUG "result": [ 03:06:52 DEBUG "metric": { 03:06:52 DEBUG "metric": { 03:06:52 DEBUG "result": [ 03:06:52 DEBUG "pod": "ds-idrepo-0" 03:06:52 DEBUG { 03:06:52 DEBUG "resultType": "vector", 03:06:52 DEBUG "metric": { 03:06:52 DEBUG "resultType": "vector", 03:06:52 DEBUG "result": [ 03:06:52 DEBUG "pod": "ds-idrepo-0" 03:06:52 DEBUG "metric": { 03:06:52 DEBUG { 03:06:52 DEBUG "metric": { 03:06:52 DEBUG "metric": { 03:06:52 DEBUG { 03:06:52 DEBUG "metric": { 03:06:52 DEBUG "pod": "ds-idrepo-0" 03:06:52 DEBUG "metric": { 03:06:52 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:06:52 DEBUG "value": [ 03:06:52 DEBUG "resultType": "vector", 03:06:52 DEBUG "metric": { 03:06:52 DEBUG { 03:06:52 DEBUG "metric": { 03:06:52 DEBUG { 03:06:52 DEBUG "instance": "10.142.0.107:9100" 03:06:52 DEBUG "pod": "ds-cts-0" 03:06:52 DEBUG { 03:06:52 DEBUG }, 03:06:52 DEBUG "metric": { 03:06:52 DEBUG "result": [ 03:06:52 DEBUG "pod": "am-767f6b67b5-9zw22" 03:06:52 DEBUG { 03:06:52 DEBUG "pod": "ds-cts-0" 03:06:52 DEBUG }, 03:06:52 DEBUG "pod": "am-767f6b67b5-9zw22" 03:06:52 DEBUG "metric": { 03:06:52 DEBUG "pod": "am-767f6b67b5-9zw22" 03:06:52 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:06:52 DEBUG "metric": { 03:06:52 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:06:52 DEBUG }, 03:06:52 DEBUG }, 03:06:52 DEBUG { 03:06:52 DEBUG 03:06:52 DEBUG "result": [ 03:06:52 DEBUG "instance": "10.142.0.107:9100" 03:06:53 DEBUG }, 03:06:52 DEBUG "metric": { 03:06:53 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:06:52 DEBUG { 03:06:52 DEBUG }, 03:06:52 DEBUG "metric": { 03:06:52 DEBUG "value": [ 03:06:52 DEBUG "pod": "ds-idrepo-0" 03:06:52 DEBUG { 03:06:53 DEBUG "metric": { 03:06:52 DEBUG "metric": { 03:06:53 DEBUG "pod": "am-767f6b67b5-9zw22" 03:06:52 DEBUG "value": [ 03:06:52 DEBUG }, 03:06:52 DEBUG "pod": "am-767f6b67b5-9zw22" 03:06:52 DEBUG }, 03:06:53 DEBUG }, 03:06:53 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:06:53 DEBUG }, 03:06:53 DEBUG "value": [ 03:06:53 DEBUG "value": [ 03:06:53 DEBUG "metric": { 03:06:53 DEBUG 03:06:53 DEBUG { 03:06:53 DEBUG "metric": { 03:06:53 DEBUG "value": [ 03:06:53 DEBUG }, 03:06:52 DEBUG }, 03:06:53 DEBUG "metric": { 03:06:53 DEBUG "value": [ 03:06:53 DEBUG "pod": "am-767f6b67b5-9zw22" 03:06:53 DEBUG 03:06:53 DEBUG }, 03:06:52 DEBUG }, 03:06:53 DEBUG "pod": "ds-cts-0" 03:06:53 DEBUG }, 03:06:52 DEBUG }, 03:06:53 DEBUG 03:06:53 DEBUG "value": [ 03:06:53 DEBUG }, 03:06:53 DEBUG "value": [ 03:06:53 DEBUG "value": [ 03:06:53 DEBUG }, 03:06:53 DEBUG "value": [ 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:06:53 DEBUG 03:06:53 DEBUG "pod": "am-767f6b67b5-9zw22" 03:06:52 DEBUG "instance": "10.142.0.107:9100" 03:06:53 DEBUG 03:06:53 DEBUG "value": [ 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG }, 03:06:53 DEBUG 03:06:53 DEBUG "value": [ 03:06:53 DEBUG "value": [ 03:06:53 DEBUG }, 03:06:53 DEBUG "value": [ 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG "value": [ 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG "value": [ 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG }, 03:06:53 DEBUG [...] 03:06:53 DEBUG }, 03:06:53 DEBUG }, 03:06:53 DEBUG 03:06:53 DEBUG [...] 03:06:53 DEBUG "value": [ 03:06:53 DEBUG 03:06:53 DEBUG "instance": "10.142.0.107:9100" 03:06:53 DEBUG "value": [ 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG "value": [ 03:06:53 DEBUG 03:06:53 DEBUG "value": [ 03:06:53 DEBUG 03:06:53 DEBUG [...] 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG "value": [ 03:06:53 DEBUG 03:06:53 DEBUG "value": [ 03:06:53 DEBUG "value": [ 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG }, 03:06:53 DEBUG 03:06:53 DEBUG [...] 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG [...] 03:06:53 DEBUG [...] 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG [...] 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG "value": [ 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG [...] 03:06:53 DEBUG 03:06:53 DEBUG [...] 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG [...] 03:06:53 DEBUG [...] 03:06:53 DEBUG 03:06:53 DEBUG [...] 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG [...] 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG [...] 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG [...] 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG [...] 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG [...] 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:06:53 DEBUG [...] 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG [...] 03:06:53 DEBUG "pod": "ds-idrepo-2" 03:06:53 DEBUG [...] 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG "pod": "ds-cts-2" 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG "instance": "10.142.0.43:9100" 03:06:53 DEBUG 03:06:53 DEBUG [...] 03:06:53 DEBUG [...] 03:06:53 DEBUG 03:06:53 DEBUG "instance": "10.142.0.43:9100" 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG }, 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG [...] 03:06:53 DEBUG 03:06:53 DEBUG [...] 03:06:53 DEBUG 03:06:53 DEBUG }, 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG }, 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG }, 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG }, 03:06:53 DEBUG 03:06:53 DEBUG }, 03:06:53 DEBUG 03:06:53 DEBUG [...] 03:06:53 DEBUG 03:06:53 DEBUG "value": [ 03:06:53 DEBUG 03:06:53 DEBUG "pod": "ds-idrepo-2" 03:06:53 DEBUG 03:06:53 DEBUG "pod": "am-767f6b67b5-vbltz" 03:06:53 DEBUG 03:06:53 DEBUG "pod": "ds-cts-2" 03:06:53 DEBUG "value": [ 03:06:53 DEBUG 03:06:53 DEBUG "pod": "am-767f6b67b5-vbltz" 03:06:53 DEBUG "pod": "am-767f6b67b5-vbltz" 03:06:53 DEBUG 03:06:53 DEBUG "value": [ 03:06:53 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:06:53 DEBUG "pod": "ds-idrepo-2" 03:06:53 DEBUG "value": [ 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG "value": [ 03:06:53 DEBUG 03:06:53 DEBUG "instance": "10.142.0.43:9100" 03:06:53 DEBUG }, 03:06:53 DEBUG 03:06:53 DEBUG 03:06:53 DEBUG 1713665212, 03:06:53 DEBUG 03:06:53 DEBUG }, 03:06:53 DEBUG 03:06:53 DEBUG }, 03:06:53 DEBUG 03:06:53 DEBUG }, 03:06:53 DEBUG 1713665212, 03:06:53 DEBUG "0" 03:06:53 DEBUG }, 03:06:53 DEBUG }, 03:06:53 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:06:53 DEBUG 1713665212, 03:06:53 DEBUG }, 03:06:53 DEBUG }, 03:06:53 DEBUG 1713665212, 03:06:53 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:06:53 DEBUG "pod": "am-767f6b67b5-vbltz" 03:06:53 DEBUG 03:06:53 DEBUG 1713665212, 03:06:53 DEBUG "value": [ 03:06:53 DEBUG "value": [ 03:06:53 DEBUG "pod": "ds-cts-2" 03:06:53 DEBUG 03:06:53 DEBUG "pod": "am-767f6b67b5-vbltz" 03:06:53 DEBUG "0" 03:06:53 DEBUG "pod": "am-767f6b67b5-vbltz" 03:06:53 DEBUG "value": [ 03:06:53 DEBUG 03:06:53 DEBUG "value": [ 03:06:53 DEBUG 03:06:53 DEBUG "value": [ 03:06:53 DEBUG ] 03:06:53 DEBUG "value": [ 03:06:53 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:06:53 DEBUG "value": [ 03:06:53 DEBUG }, 03:06:53 DEBUG "909172736" 03:06:53 DEBUG "value": [ 03:06:53 DEBUG "value": [ 03:06:53 DEBUG "4331037692928" 03:06:53 DEBUG }, 03:06:53 DEBUG }, 03:06:53 DEBUG 03:06:53 DEBUG "0.1410666666665918" 03:06:53 DEBUG 1713665212, 03:06:53 DEBUG 1713665212, 03:06:53 DEBUG }, 03:06:53 DEBUG 03:06:53 DEBUG }, 03:06:53 DEBUG ] 03:06:53 DEBUG }, 03:06:53 DEBUG 1713665212, 03:06:53 DEBUG "pod": "ds-cts-2" 03:06:53 DEBUG 1713665212, 03:06:53 DEBUG "pod": "am-767f6b67b5-vbltz" 03:06:53 DEBUG 1713665212, 03:06:53 DEBUG } 03:06:53 DEBUG 1713665212, 03:06:53 DEBUG }, 03:06:53 DEBUG 1713665212, 03:06:53 DEBUG "0" 03:06:53 DEBUG ] 03:06:53 DEBUG 1713665212, 03:06:53 DEBUG 1713665212, 03:06:53 DEBUG ] 03:06:53 DEBUG "value": [ 03:06:53 DEBUG "value": [ 03:06:53 DEBUG "instance": "10.142.0.43:9100" 03:06:53 DEBUG ] 03:06:53 DEBUG "11.737303086988991" 03:06:53 DEBUG "0.0433333333333356" 03:06:53 DEBUG "value": [ 03:06:53 DEBUG "instance": "10.142.0.43:9100" 03:06:53 DEBUG "value": [ 03:06:53 DEBUG } 03:06:53 DEBUG ] 03:06:53 DEBUG "0" 03:06:53 DEBUG }, 03:06:53 DEBUG "149.888" 03:06:53 DEBUG }, 03:06:53 DEBUG "0" 03:06:53 DEBUG ] 03:06:53 DEBUG ] 03:06:53 DEBUG "value": [ 03:06:53 DEBUG ] 03:06:53 DEBUG } 03:06:53 DEBUG "value": [ 03:06:53 DEBUG "11.766516581135189" 03:06:53 DEBUG "0" 03:06:53 DEBUG } 03:06:53 DEBUG 1713665212, 03:06:53 DEBUG 1713665212, 03:06:53 DEBUG }, 03:06:53 DEBUG } 03:06:53 DEBUG ] 03:06:53 DEBUG ] 03:06:53 DEBUG 1713665212, 03:06:53 DEBUG }, 03:06:53 DEBUG 1713665212, 03:06:53 DEBUG "value": [ 03:06:53 DEBUG } 03:06:53 DEBUG ] 03:06:53 DEBUG "value": [ 03:06:53 DEBUG ] 03:06:53 DEBUG "value": [ 03:06:53 DEBUG "3321.6666666666674" 03:06:53 DEBUG } 03:06:53 DEBUG } 03:06:53 DEBUG 1713665212, 03:06:53 DEBUG } 03:06:53 DEBUG ] 03:06:53 DEBUG 1713665212, 03:06:53 DEBUG ] 03:06:53 DEBUG ] 03:06:53 DEBUG ] 03:06:53 DEBUG "77862.48522182813" 03:06:53 DEBUG "599219.5" 03:06:53 DEBUG "value": [ 03:06:53 DEBUG ] 03:06:53 DEBUG } 03:06:53 DEBUG } 03:06:53 DEBUG "0" 03:06:53 DEBUG "value": [ 03:06:53 DEBUG "3319.633333333334" 03:06:53 DEBUG 1713665212, 03:06:53 DEBUG } 03:06:53 DEBUG } 03:06:53 DEBUG 1713665212, 03:06:53 DEBUG } 03:06:53 DEBUG 1713665212, 03:06:53 DEBUG "3329.6" 03:06:53 DEBUG ] 03:06:53 DEBUG } 03:06:53 DEBUG "0" 03:06:53 DEBUG ] 03:06:53 DEBUG } 03:06:53 DEBUG } 03:06:53 DEBUG } 03:06:53 DEBUG } 03:06:53 DEBUG } 03:06:53 DEBUG ] 03:06:53 DEBUG ] 03:06:53 DEBUG 1713665212, 03:06:53 DEBUG } 03:06:53 DEBUG ] 03:06:53 DEBUG ] 03:06:53 DEBUG ] 03:06:53 DEBUG 1713665212, 03:06:53 DEBUG ] 03:06:53 DEBUG "3318.433333333334" 03:06:53 DEBUG ] 03:06:53 DEBUG "0.43856666666666666" 03:06:53 DEBUG ] 03:06:53 DEBUG ] 03:06:53 DEBUG ] 03:06:53 DEBUG } 03:06:53 DEBUG ] 03:06:53 DEBUG ] 03:06:53 DEBUG } 03:06:53 DEBUG } 03:06:53 DEBUG ] 03:06:53 DEBUG ] 03:06:53 DEBUG } 03:06:53 DEBUG } 03:06:53 DEBUG } 03:06:53 DEBUG "100493915.33333333" 03:06:53 DEBUG } 03:06:53 DEBUG } 03:06:53 DEBUG } 03:06:53 DEBUG } 03:06:53 DEBUG "100490818.56666666" 03:06:53 DEBUG } 03:06:53 DEBUG ] 03:06:53 DEBUG } 03:06:53 DEBUG ] 03:06:53 DEBUG } 03:06:53 DEBUG } 03:06:53 DEBUG } 03:06:53 DEBUG } 03:06:53 DEBUG } 03:06:53 DEBUG "0" 03:06:53 DEBUG ] 03:06:53 DEBUG } 03:06:53 DEBUG ] 03:06:53 DEBUG ] 03:06:53 DEBUG ] 03:06:53 DEBUG } 03:06:53 DEBUG } 03:06:53 DEBUG ] 03:06:53 DEBUG ] 03:06:53 DEBUG ] 03:06:53 DEBUG } 03:06:53 DEBUG } 03:06:53 DEBUG } 03:06:53 DEBUG ] 03:06:53 DEBUG ] 03:06:53 DEBUG } 03:06:53 DEBUG } 03:06:53 DEBUG ] 03:06:53 DEBUG } 03:06:53 DEBUG } 03:06:53 DEBUG } 03:06:53 DEBUG } 03:06:53 DEBUG } 03:06:53 DEBUG } 03:06:53 DEBUG } 03:06:53 DEBUG } 03:06:53 DEBUG } 03:06:53 DEBUG ] 03:06:53 DEBUG } 03:06:53 DEBUG } 03:06:53 DEBUG } 03:06:53 DEBUG } 03:06:53 DEBUG ] 03:06:53 DEBUG } 03:06:53 DEBUG } 03:06:53 DEBUG } 03:06:53 DEBUG ] 03:06:53 DEBUG } 03:06:53 DEBUG ] 03:06:53 DEBUG } 03:06:53 DEBUG } 03:06:53 DEBUG } 03:06:53 DEBUG } 03:06:53 DEBUG } 03:06:53 DEBUG } 03:06:53 DEBUG } 03:06:53 DEBUG } 03:06:53 DEBUG } 03:06:53 DEBUG } 03:06:53 DEBUG } 03:06:53 DEBUG } 03:07:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713665272" 03:07:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665272" 03:07:52 INFO [http_cmd]: http status code OK 03:07:52 DEBUG --- status code --- 03:07:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:07:52 DEBUG --- http response --- 03:07:52 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 03:07:52 ERROR [http_cmd]: ERROR 03:07:52 ERROR --- status code --- 03:07:52 ERROR http status code is 400 (expected [200, 401, 403, 404]) 03:07:52 ERROR --- http response --- 03:07:52 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 03:07:52 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 03:07:52 INFO 03:07:52 INFO [loop_until]: kubectl --namespace=xlou top pods 03:07:52 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:07:52 INFO [loop_until]: OK (rc = 0) 03:07:52 DEBUG --- stdout --- 03:07:52 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 22147m 8790Mi am-767f6b67b5-qcqz2 22263m 9122Mi am-767f6b67b5-vbltz 24407m 8829Mi ds-cts-0 20723m 12308Mi ds-cts-1 20416m 12209Mi ds-cts-2 20459m 12252Mi ds-idrepo-0 9508m 3930Mi ds-idrepo-1 9712m 3946Mi ds-idrepo-2 9843m 3916Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1179Mi idm-b85f9d978-wnhxb 3m 1239Mi lodemon-54c87f8cbb-jzrjs 2m 62Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-68c5d5b884-p97w5 11510m 780Mi 03:07:52 DEBUG --- stderr --- 03:07:52 DEBUG 03:07:53 INFO 03:07:53 INFO [loop_until]: kubectl --namespace=xlou top node 03:07:53 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:07:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665273" 03:07:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665273" 03:07:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665273" 03:07:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713665273" 03:07:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665273" 03:07:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713665273" 03:07:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665273" 03:07:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665273" 03:07:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665273" 03:07:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665273" 03:07:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665273" 03:07:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665273" 03:07:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713665273" 03:07:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713665273" 03:07:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665273" 03:07:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665273" 03:07:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665273" 03:07:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665273" 03:07:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665273" 03:07:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665273" 03:07:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665273" 03:07:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665273" 03:07:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665273" 03:07:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665273" 03:07:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665273" 03:07:53 INFO [http_cmd]: http status code OK 03:07:53 INFO [loop_until]: OK (rc = 0) 03:07:53 INFO [http_cmd]: http status code OK 03:07:53 INFO [http_cmd]: http status code OK 03:07:53 INFO [http_cmd]: http status code OK 03:07:53 INFO [http_cmd]: http status code OK 03:07:53 DEBUG --- status code --- 03:07:53 INFO [http_cmd]: http status code OK 03:07:53 INFO [http_cmd]: http status code OK 03:07:53 INFO [http_cmd]: http status code OK 03:07:53 INFO [http_cmd]: http status code OK 03:07:53 INFO [http_cmd]: http status code OK 03:07:53 INFO [http_cmd]: http status code OK 03:07:53 INFO [http_cmd]: http status code OK 03:07:53 INFO [http_cmd]: http status code OK 03:07:53 INFO [http_cmd]: http status code OK 03:07:53 INFO [http_cmd]: http status code OK 03:07:53 INFO [http_cmd]: http status code OK 03:07:53 DEBUG --- status code --- 03:07:53 DEBUG --- status code --- 03:07:53 INFO [http_cmd]: http status code OK 03:07:53 INFO [http_cmd]: http status code OK 03:07:53 DEBUG --- stdout --- 03:07:53 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 65m 0% 6292Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 9928m 33% 7772Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 21914m 73% 11595Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 84m 0% 3369Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 20438m 68% 15635Mi 14% gke-xlou-cdm-default-pool-1b0769c8-wpwl 149m 0% 5272Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 22350m 74% 12053Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 19714m 66% 3194Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 24736m 82% 11089Mi 9% gke-xlou-cdm-ds-0f601751-j488 10392m 34% 7361Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 20322m 68% 17321Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 10134m 33% 8015Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20519m 68% 14963Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 12026m 75% 11142Mi 18% 03:07:53 DEBUG --- status code --- 03:07:53 INFO [http_cmd]: http status code OK 03:07:53 INFO [http_cmd]: http status code OK 03:07:53 DEBUG --- status code --- 03:07:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:07:53 DEBUG --- http response --- 03:07:53 INFO [http_cmd]: http status code OK 03:07:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:07:53 DEBUG --- status code --- 03:07:53 DEBUG --- status code --- 03:07:53 DEBUG --- status code --- 03:07:53 DEBUG --- status code --- 03:07:53 DEBUG --- status code --- 03:07:53 DEBUG --- status code --- 03:07:53 DEBUG --- status code --- 03:07:53 DEBUG --- status code --- 03:07:53 DEBUG --- status code --- 03:07:53 DEBUG --- status code --- 03:07:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:07:53 INFO [http_cmd]: http status code OK 03:07:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:07:53 DEBUG --- status code --- 03:07:53 DEBUG --- status code --- 03:07:53 INFO [http_cmd]: http status code OK 03:07:53 DEBUG --- stderr --- 03:07:53 DEBUG 03:07:53 DEBUG --- status code --- 03:07:53 DEBUG --- status code --- 03:07:53 DEBUG --- status code --- 03:07:53 INFO [http_cmd]: http status code OK 03:07:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:07:53 INFO [http_cmd]: http status code OK 03:07:53 DEBUG --- status code --- 03:07:53 DEBUG --- http response --- 03:07:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:07:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:07:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:07:53 DEBUG --- http response --- 03:07:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:07:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:07:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:07:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:07:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:07:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:07:53 DEBUG --- http response --- 03:07:53 DEBUG --- status code --- 03:07:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:07:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:07:53 DEBUG --- http response --- 03:07:53 DEBUG --- http response --- 03:07:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:07:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:07:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:07:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:07:53 DEBUG --- status code --- 03:07:53 DEBUG ----- output ----- 03:07:53 DEBUG --- status code --- 03:07:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:07:53 DEBUG ----- output ----- 03:07:53 DEBUG { 03:07:53 DEBUG --- http response --- 03:07:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:07:53 DEBUG --- http response --- 03:07:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:07:53 DEBUG --- http response --- 03:07:53 DEBUG --- http response --- 03:07:53 DEBUG --- http response --- 03:07:53 DEBUG --- http response --- 03:07:53 DEBUG --- http response --- 03:07:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:07:53 DEBUG --- http response --- 03:07:53 DEBUG --- status code --- 03:07:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:07:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:07:53 DEBUG ----- output ----- 03:07:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:07:53 DEBUG --- http response --- 03:07:53 DEBUG --- http response --- 03:07:53 DEBUG --- http response --- 03:07:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:07:53 DEBUG { 03:07:53 DEBUG "status": "success", 03:07:53 DEBUG --- http response --- 03:07:53 DEBUG "status": "success", 03:07:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:07:53 DEBUG --- http response --- 03:07:53 DEBUG --- http response --- 03:07:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:07:53 DEBUG ----- output ----- 03:07:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:07:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:07:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:07:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:07:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:07:53 DEBUG ----- output ----- 03:07:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:07:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:07:53 DEBUG --- http response --- 03:07:53 DEBUG --- http response --- 03:07:53 DEBUG { 03:07:53 DEBUG ----- output ----- 03:07:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:07:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:07:53 DEBUG --- http response --- 03:07:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:07:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:07:53 DEBUG "data": { 03:07:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:07:53 DEBUG "data": { 03:07:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:07:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:07:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:07:53 DEBUG ----- output ----- 03:07:53 DEBUG { 03:07:53 DEBUG ----- output ----- 03:07:53 DEBUG ----- output ----- 03:07:53 DEBUG ----- output ----- 03:07:53 DEBUG ----- output ----- 03:07:53 DEBUG ----- output ----- 03:07:53 DEBUG { 03:07:53 DEBUG ----- output ----- 03:07:53 DEBUG --- http response --- 03:07:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:07:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:07:53 DEBUG "status": "success", 03:07:53 DEBUG { 03:07:53 DEBUG ----- output ----- 03:07:53 DEBUG ----- output ----- 03:07:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:07:53 DEBUG ----- output ----- 03:07:53 DEBUG --- http response --- 03:07:53 DEBUG "resultType": "vector", 03:07:53 DEBUG ----- output ----- 03:07:53 DEBUG "resultType": "vector", 03:07:53 DEBUG ----- output ----- 03:07:53 DEBUG ----- output ----- 03:07:53 DEBUG ----- output ----- 03:07:53 DEBUG { 03:07:53 DEBUG "status": "success", 03:07:53 DEBUG { 03:07:53 DEBUG { 03:07:53 DEBUG { 03:07:53 DEBUG { 03:07:53 DEBUG { 03:07:53 DEBUG "status": "success", 03:07:53 DEBUG { 03:07:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:07:53 DEBUG ----- output ----- 03:07:53 DEBUG ----- output ----- 03:07:53 DEBUG "data": { 03:07:53 DEBUG "status": "success", 03:07:53 DEBUG { 03:07:53 DEBUG { 03:07:53 DEBUG ----- output ----- 03:07:53 DEBUG { 03:07:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:07:53 DEBUG "result": [ 03:07:53 DEBUG { 03:07:53 DEBUG "result": [ 03:07:53 DEBUG { 03:07:53 DEBUG { 03:07:53 DEBUG { 03:07:53 DEBUG "status": "success", 03:07:53 DEBUG "data": { 03:07:53 DEBUG "status": "success", 03:07:53 DEBUG "status": "success", 03:07:53 DEBUG "status": "success", 03:07:53 DEBUG "status": "success", 03:07:53 DEBUG "status": "success", 03:07:53 DEBUG "data": { 03:07:53 DEBUG "status": "success", 03:07:53 DEBUG ----- output ----- 03:07:53 DEBUG { 03:07:53 DEBUG { 03:07:53 DEBUG "resultType": "vector", 03:07:53 DEBUG "data": { 03:07:53 DEBUG "status": "success", 03:07:53 DEBUG "status": "success", 03:07:53 DEBUG { 03:07:53 DEBUG "status": "success", 03:07:53 DEBUG ----- output ----- 03:07:53 DEBUG { 03:07:53 DEBUG "status": "success", 03:07:53 DEBUG { 03:07:53 DEBUG "status": "success", 03:07:53 DEBUG "status": "success", 03:07:53 DEBUG "status": "success", 03:07:53 DEBUG "data": { 03:07:53 DEBUG "resultType": "vector", 03:07:53 DEBUG "data": { 03:07:53 DEBUG "data": { 03:07:53 DEBUG "data": { 03:07:53 DEBUG "data": { 03:07:53 DEBUG "data": { 03:07:53 DEBUG "resultType": "vector", 03:07:53 DEBUG "data": { 03:07:53 DEBUG { 03:07:53 DEBUG "status": "success", 03:07:53 DEBUG "status": "success", 03:07:53 DEBUG "result": [ 03:07:53 DEBUG "resultType": "vector", 03:07:53 DEBUG "data": { 03:07:53 DEBUG "data": { 03:07:53 DEBUG "status": "success", 03:07:53 DEBUG "data": { 03:07:53 DEBUG { 03:07:53 DEBUG "metric": { 03:07:53 DEBUG "data": { 03:07:53 DEBUG "metric": { 03:07:53 DEBUG "data": { 03:07:53 DEBUG "data": { 03:07:53 DEBUG "data": { 03:07:53 DEBUG "resultType": "vector", 03:07:53 DEBUG "result": [ 03:07:53 DEBUG "resultType": "vector", 03:07:53 DEBUG "resultType": "vector", 03:07:53 DEBUG "resultType": "vector", 03:07:53 DEBUG "resultType": "vector", 03:07:53 DEBUG "result": [ 03:07:53 DEBUG "resultType": "vector", 03:07:53 DEBUG "status": "success", 03:07:53 DEBUG "data": { 03:07:53 DEBUG "data": { 03:07:53 DEBUG "resultType": "vector", 03:07:53 DEBUG "result": [ 03:07:53 DEBUG "result": [ 03:07:53 DEBUG "resultType": "vector", 03:07:53 DEBUG "resultType": "vector", 03:07:53 DEBUG "data": { 03:07:53 DEBUG "resultType": "vector", 03:07:53 DEBUG "status": "success", 03:07:53 DEBUG "instance": "10.142.0.107:9100" 03:07:53 DEBUG "resultType": "vector", 03:07:53 DEBUG "instance": "10.142.0.107:9100" 03:07:53 DEBUG "resultType": "vector", 03:07:53 DEBUG "resultType": "vector", 03:07:53 DEBUG "resultType": "vector", 03:07:53 DEBUG "result": [ 03:07:53 DEBUG { 03:07:53 DEBUG "result": [ 03:07:53 DEBUG "result": [ 03:07:53 DEBUG "result": [ 03:07:53 DEBUG "result": [ 03:07:53 DEBUG { 03:07:53 DEBUG "result": [ 03:07:53 DEBUG "data": { 03:07:53 DEBUG "resultType": "vector", 03:07:53 DEBUG "resultType": "vector", 03:07:53 DEBUG { 03:07:53 DEBUG { 03:07:53 DEBUG { 03:07:53 DEBUG "result": [ 03:07:53 DEBUG "result": [ 03:07:53 DEBUG "resultType": "vector", 03:07:53 DEBUG "result": [ 03:07:53 DEBUG "data": { 03:07:53 DEBUG }, 03:07:53 DEBUG "result": [ 03:07:53 DEBUG }, 03:07:53 DEBUG "result": [ 03:07:53 DEBUG "result": [ 03:07:53 DEBUG "result": [ 03:07:53 DEBUG { 03:07:53 DEBUG "metric": { 03:07:53 DEBUG { 03:07:53 DEBUG { 03:07:53 DEBUG { 03:07:53 DEBUG { 03:07:53 DEBUG "metric": { 03:07:53 DEBUG "pod": "ds-idrepo-0" 03:07:53 DEBUG "resultType": "vector", 03:07:53 DEBUG "result": [ 03:07:53 DEBUG "result": [ 03:07:53 DEBUG "metric": { 03:07:53 DEBUG "metric": { 03:07:53 DEBUG "metric": { 03:07:53 DEBUG { 03:07:53 DEBUG { 03:07:53 DEBUG "result": [ 03:07:53 DEBUG { 03:07:53 DEBUG "resultType": "vector", 03:07:53 DEBUG "value": [ 03:07:53 DEBUG { 03:07:53 DEBUG "value": [ 03:07:53 DEBUG { 03:07:53 DEBUG { 03:07:53 DEBUG { 03:07:53 DEBUG "metric": { 03:07:53 DEBUG "pod": "ds-cts-0" 03:07:53 DEBUG "metric": { 03:07:53 DEBUG "metric": { 03:07:53 DEBUG "metric": { 03:07:53 DEBUG "metric": { 03:07:53 DEBUG }, 03:07:53 DEBUG { 03:07:53 DEBUG "result": [ 03:07:53 DEBUG { 03:07:53 DEBUG { 03:07:53 DEBUG "pod": "am-767f6b67b5-9zw22" 03:07:53 DEBUG "pod": "ds-idrepo-0" 03:07:53 DEBUG "pod": "ds-cts-0" 03:07:53 DEBUG "metric": { 03:07:53 DEBUG "metric": { 03:07:53 DEBUG { 03:07:53 DEBUG "metric": { 03:07:53 DEBUG "result": [ 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG "metric": { 03:07:53 DEBUG "metric": { 03:07:53 DEBUG "metric": { 03:07:53 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:07:53 DEBUG }, 03:07:53 DEBUG "value": [ 03:07:53 DEBUG "pod": "ds-cts-0" 03:07:53 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:07:53 DEBUG "pod": "am-767f6b67b5-9zw22" 03:07:53 DEBUG "value": [ 03:07:53 DEBUG "metric": { 03:07:53 DEBUG { 03:07:53 DEBUG "metric": { 03:07:53 DEBUG "metric": { 03:07:53 DEBUG }, 03:07:53 DEBUG }, 03:07:53 DEBUG }, 03:07:53 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:07:53 DEBUG "pod": "ds-idrepo-0" 03:07:53 DEBUG "metric": { 03:07:53 DEBUG "pod": "am-767f6b67b5-9zw22" 03:07:53 DEBUG { 03:07:53 DEBUG 03:07:53 DEBUG "metric": { 03:07:53 DEBUG 03:07:53 DEBUG "pod": "ds-idrepo-0" 03:07:53 DEBUG "instance": "10.142.0.107:9100" 03:07:53 DEBUG "pod": "am-767f6b67b5-9zw22" 03:07:53 DEBUG }, 03:07:53 DEBUG "instance": "10.142.0.107:9100" 03:07:53 DEBUG 03:07:53 DEBUG }, 03:07:53 DEBUG }, 03:07:53 DEBUG }, 03:07:53 DEBUG 03:07:53 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:07:53 DEBUG "metric": { 03:07:53 DEBUG "pod": "am-767f6b67b5-9zw22" 03:07:53 DEBUG "pod": "am-767f6b67b5-9zw22" 03:07:53 DEBUG "value": [ 03:07:53 DEBUG "value": [ 03:07:53 DEBUG "value": [ 03:07:53 DEBUG }, 03:07:53 DEBUG }, 03:07:53 DEBUG "instance": "10.142.0.107:9100" 03:07:53 DEBUG }, 03:07:53 DEBUG "metric": { 03:07:53 DEBUG [...] 03:07:53 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:07:53 DEBUG 03:07:53 DEBUG }, 03:07:53 DEBUG }, 03:07:53 DEBUG }, 03:07:53 DEBUG "value": [ 03:07:53 DEBUG }, 03:07:53 DEBUG 03:07:53 DEBUG "value": [ 03:07:53 DEBUG "value": [ 03:07:53 DEBUG "value": [ 03:07:53 DEBUG 03:07:53 DEBUG }, 03:07:53 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:07:53 DEBUG }, 03:07:53 DEBUG }, 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG "value": [ 03:07:53 DEBUG 03:07:53 DEBUG "value": [ 03:07:53 DEBUG "value": [ 03:07:53 DEBUG "pod": "am-767f6b67b5-9zw22" 03:07:53 DEBUG 03:07:53 DEBUG }, 03:07:53 DEBUG [...] 03:07:53 DEBUG "value": [ 03:07:53 DEBUG "value": [ 03:07:53 DEBUG "value": [ 03:07:53 DEBUG 03:07:53 DEBUG "value": [ 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG "value": [ 03:07:53 DEBUG }, 03:07:53 DEBUG "value": [ 03:07:53 DEBUG "value": [ 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG }, 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG }, 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG [...] 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG [...] 03:07:53 DEBUG 03:07:53 DEBUG "value": [ 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG "value": [ 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG "value": [ 03:07:53 DEBUG "instance": "10.142.0.43:9100" 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG "value": [ 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG [...] 03:07:53 DEBUG [...] 03:07:53 DEBUG [...] 03:07:53 DEBUG 03:07:53 DEBUG [...] 03:07:53 DEBUG 03:07:53 DEBUG [...] 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG [...] 03:07:53 DEBUG [...] 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG [...] 03:07:53 DEBUG [...] 03:07:53 DEBUG [...] 03:07:53 DEBUG [...] 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG [...] 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG [...] 03:07:53 DEBUG "instance": "10.142.0.43:9100" 03:07:53 DEBUG }, 03:07:53 DEBUG }, 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG [...] 03:07:53 DEBUG [...] 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG "value": [ 03:07:53 DEBUG [...] 03:07:53 DEBUG "value": [ 03:07:53 DEBUG 03:07:53 DEBUG [...] 03:07:53 DEBUG 03:07:53 DEBUG [...] 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG "pod": "ds-cts-2" 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG "pod": "am-767f6b67b5-vbltz" 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG [...] 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG [...] 03:07:53 DEBUG 1713665273, 03:07:53 DEBUG 03:07:53 DEBUG 1713665273, 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG }, 03:07:53 DEBUG "pod": "ds-idrepo-2" 03:07:53 DEBUG [...] 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG }, 03:07:53 DEBUG 03:07:53 DEBUG "pod": "ds-cts-2" 03:07:53 DEBUG 03:07:53 DEBUG "pod": "idm-b85f9d978-txpwv" 03:07:53 DEBUG "pod": "ds-idrepo-2" 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG "0.17093333333327979" 03:07:53 DEBUG 03:07:53 DEBUG "100422382.06666666" 03:07:53 DEBUG "instance": "10.142.0.43:9100" 03:07:53 DEBUG 03:07:53 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG "pod": "ds-cts-2" 03:07:53 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:07:53 DEBUG "pod": "am-767f6b67b5-vbltz" 03:07:53 DEBUG "value": [ 03:07:53 DEBUG }, 03:07:53 DEBUG 03:07:53 DEBUG "pod": "idm-b85f9d978-txpwv" 03:07:53 DEBUG 03:07:53 DEBUG 03:07:53 DEBUG "value": [ 03:07:53 DEBUG "pod": "ds-cts-2" 03:07:53 DEBUG }, 03:07:53 DEBUG 03:07:53 DEBUG }, 03:07:53 DEBUG }, 03:07:53 DEBUG "pod": "am-767f6b67b5-vbltz" 03:07:53 DEBUG 03:07:53 DEBUG ] 03:07:53 DEBUG 03:07:53 DEBUG ] 03:07:53 DEBUG }, 03:07:53 DEBUG 03:07:53 DEBUG }, 03:07:53 DEBUG 03:07:53 DEBUG "instance": "10.142.0.43:9100" 03:07:53 DEBUG }, 03:07:53 DEBUG }, 03:07:53 DEBUG }, 03:07:53 DEBUG 1713665273, 03:07:53 DEBUG "value": [ 03:07:53 DEBUG 03:07:53 DEBUG }, 03:07:53 DEBUG "pod": "am-767f6b67b5-vbltz" 03:07:53 DEBUG "pod": "am-767f6b67b5-vbltz" 03:07:53 DEBUG 1713665273, 03:07:53 DEBUG }, 03:07:53 DEBUG "value": [ 03:07:53 DEBUG 03:07:53 DEBUG "value": [ 03:07:53 DEBUG "value": [ 03:07:53 DEBUG }, 03:07:53 DEBUG 03:07:53 DEBUG } 03:07:53 DEBUG "pod": "ds-idrepo-2" 03:07:53 DEBUG } 03:07:53 DEBUG "value": [ 03:07:53 DEBUG "pod": "am-767f6b67b5-vbltz" 03:07:53 DEBUG "value": [ 03:07:53 DEBUG "pod": "idm-b85f9d978-txpwv" 03:07:53 DEBUG }, 03:07:53 DEBUG "value": [ 03:07:53 DEBUG "value": [ 03:07:53 DEBUG "value": [ 03:07:53 DEBUG "0" 03:07:53 DEBUG 1713665273, 03:07:53 DEBUG 03:07:53 DEBUG "pod": "idm-b85f9d978-txpwv" 03:07:53 DEBUG }, 03:07:53 DEBUG }, 03:07:53 DEBUG "599570" 03:07:53 DEBUG "value": [ 03:07:53 DEBUG 1713665273, 03:07:53 DEBUG "instance": "10.142.0.43:9100" 03:07:53 DEBUG 1713665273, 03:07:53 DEBUG 1713665273, 03:07:53 DEBUG "value": [ 03:07:53 DEBUG "pod": "am-767f6b67b5-vbltz" 03:07:53 DEBUG ] 03:07:53 DEBUG }, 03:07:53 DEBUG ] 03:07:53 DEBUG 1713665273, 03:07:53 DEBUG }, 03:07:53 DEBUG 1713665273, 03:07:53 DEBUG }, 03:07:53 DEBUG "value": [ 03:07:53 DEBUG 1713665273, 03:07:53 DEBUG 1713665273, 03:07:53 DEBUG 1713665273, 03:07:53 DEBUG ] 03:07:53 DEBUG "0" 03:07:53 DEBUG "value": [ 03:07:53 DEBUG }, 03:07:53 DEBUG "value": [ 03:07:53 DEBUG "value": [ 03:07:53 DEBUG ] 03:07:53 DEBUG 1713665273, 03:07:53 DEBUG "0.45146666666666657" 03:07:53 DEBUG }, 03:07:53 DEBUG "8783.809140865835" 03:07:53 DEBUG "0" 03:07:53 DEBUG 1713665273, 03:07:53 DEBUG }, 03:07:53 DEBUG } 03:07:53 DEBUG "value": [ 03:07:53 DEBUG } 03:07:53 DEBUG "4335284835328" 03:07:53 DEBUG "value": [ 03:07:53 DEBUG "912961536" 03:07:53 DEBUG "value": [ 03:07:53 DEBUG 1713665273, 03:07:53 DEBUG "0" 03:07:53 DEBUG "11.789703151515155" 03:07:53 DEBUG "149.889" 03:07:53 DEBUG } 03:07:53 DEBUG ] 03:07:53 DEBUG } 03:07:53 DEBUG "value": [ 03:07:53 DEBUG 1713665273, 03:07:53 DEBUG 1713665273, 03:07:53 DEBUG } 03:07:53 DEBUG "0" 03:07:53 DEBUG ] 03:07:53 DEBUG "value": [ 03:07:53 DEBUG ] 03:07:53 DEBUG } 03:07:53 DEBUG ] 03:07:53 DEBUG "value": [ 03:07:53 DEBUG } 03:07:53 DEBUG 1713665273, 03:07:53 DEBUG } 03:07:53 DEBUG ] 03:07:53 DEBUG 1713665273, 03:07:53 DEBUG ] 03:07:53 DEBUG 1713665273, 03:07:53 DEBUG "0.04833333333333486" 03:07:53 DEBUG ] 03:07:53 DEBUG } 03:07:53 DEBUG ] 03:07:53 DEBUG ] 03:07:53 DEBUG ] 03:07:53 DEBUG 1713665273, 03:07:53 DEBUG 1713665273, 03:07:53 DEBUG "3314.4666666666667" 03:07:53 DEBUG "0" 03:07:53 DEBUG ] 03:07:53 DEBUG ] 03:07:53 DEBUG } 03:07:53 DEBUG 1713665273, 03:07:53 DEBUG "3331.366666666667" 03:07:53 DEBUG ] 03:07:53 DEBUG } 03:07:53 DEBUG 1713665273, 03:07:53 DEBUG "0" 03:07:53 DEBUG } 03:07:53 DEBUG "3330.0333333333333" 03:07:53 DEBUG } 03:07:53 DEBUG "0.003259482643076172" 03:07:53 DEBUG ] 03:07:53 DEBUG ] 03:07:53 DEBUG } 03:07:53 DEBUG } 03:07:53 DEBUG } 03:07:53 DEBUG ] 03:07:53 DEBUG "411.741053478086" 03:07:53 DEBUG "0" 03:07:53 DEBUG ] 03:07:53 DEBUG ] 03:07:53 DEBUG } 03:07:53 DEBUG } 03:07:53 DEBUG ] 03:07:53 DEBUG } 03:07:53 DEBUG ] 03:07:53 DEBUG } 03:07:53 DEBUG ] 03:07:53 DEBUG "3329.4" 03:07:53 DEBUG ] 03:07:53 DEBUG ] 03:07:53 DEBUG ] 03:07:53 DEBUG ] 03:07:53 DEBUG ] 03:07:53 DEBUG } 03:07:53 DEBUG } 03:07:53 DEBUG } 03:07:53 DEBUG } 03:07:53 DEBUG } 03:07:53 DEBUG } 03:07:53 DEBUG ] 03:07:53 DEBUG ] 03:07:53 DEBUG } 03:07:53 DEBUG } 03:07:53 DEBUG } 03:07:53 DEBUG ] 03:07:53 DEBUG } 03:07:53 DEBUG "100426163.53333333" 03:07:53 DEBUG } 03:07:53 DEBUG ] 03:07:53 DEBUG } 03:07:53 DEBUG ] 03:07:53 DEBUG } 03:07:53 DEBUG } 03:07:53 DEBUG } 03:07:53 DEBUG } 03:07:53 DEBUG } 03:07:53 DEBUG ] 03:07:53 DEBUG ] 03:07:53 DEBUG ] 03:07:53 DEBUG } 03:07:53 DEBUG } 03:07:53 DEBUG ] 03:07:53 DEBUG ] 03:07:53 DEBUG } 03:07:53 DEBUG ] 03:07:53 DEBUG } 03:07:53 DEBUG } 03:07:53 DEBUG } 03:07:53 DEBUG } 03:07:53 DEBUG ] 03:07:53 DEBUG } 03:07:53 DEBUG ] 03:07:53 DEBUG } 03:07:53 DEBUG ] 03:07:53 DEBUG } 03:07:53 DEBUG } 03:07:53 DEBUG } 03:07:53 DEBUG ] 03:07:53 DEBUG } 03:07:53 DEBUG ] 03:07:53 DEBUG } 03:07:53 DEBUG } 03:07:53 DEBUG } 03:07:53 DEBUG } 03:07:53 DEBUG ] 03:07:53 DEBUG } 03:07:53 DEBUG } 03:07:53 DEBUG } 03:07:53 DEBUG } 03:07:53 DEBUG } 03:07:53 DEBUG } 03:07:53 DEBUG } 03:07:53 DEBUG } 03:07:53 DEBUG } 03:07:53 DEBUG } 03:07:53 DEBUG ] 03:07:53 DEBUG } 03:07:53 DEBUG } 03:07:53 DEBUG } 03:07:53 DEBUG } 03:07:53 DEBUG } 03:07:53 DEBUG } 03:07:53 DEBUG } 03:07:53 DEBUG } 03:07:53 DEBUG } 03:08:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713665332" 03:08:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665332" 03:08:52 ERROR [http_cmd]: ERROR 03:08:52 INFO [http_cmd]: http status code OK 03:08:52 ERROR --- status code --- 03:08:52 DEBUG --- status code --- 03:08:52 ERROR http status code is 400 (expected [200, 401, 403, 404]) 03:08:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:08:52 ERROR --- http response --- 03:08:52 DEBUG --- http response --- 03:08:52 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 03:08:52 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 03:08:52 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 03:08:52 INFO 03:08:52 INFO [loop_until]: kubectl --namespace=xlou top pods 03:08:52 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:08:53 INFO [loop_until]: OK (rc = 0) 03:08:53 DEBUG --- stdout --- 03:08:53 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 21341m 8789Mi am-767f6b67b5-qcqz2 22576m 9120Mi am-767f6b67b5-vbltz 24039m 8814Mi ds-cts-0 19729m 12320Mi ds-cts-1 19967m 12284Mi ds-cts-2 20391m 12251Mi ds-idrepo-0 9570m 3930Mi ds-idrepo-1 9508m 3946Mi ds-idrepo-2 9573m 3916Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 3m 1179Mi idm-b85f9d978-wnhxb 3m 1239Mi lodemon-54c87f8cbb-jzrjs 15m 62Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-68c5d5b884-p97w5 11539m 783Mi 03:08:53 DEBUG --- stderr --- 03:08:53 DEBUG 03:08:53 INFO 03:08:53 INFO [loop_until]: kubectl --namespace=xlou top node 03:08:53 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:08:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665333" 03:08:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665333" 03:08:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665333" 03:08:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665333" 03:08:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665333" 03:08:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665333" 03:08:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665333" 03:08:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665333" 03:08:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665333" 03:08:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665333" 03:08:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713665333" 03:08:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665333" 03:08:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665333" 03:08:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713665333" 03:08:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665333" 03:08:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713665333" 03:08:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665333" 03:08:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665333" 03:08:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665333" 03:08:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665333" 03:08:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665333" 03:08:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665333" 03:08:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665333" 03:08:53 INFO [http_cmd]: http status code OK 03:08:53 INFO [loop_until]: OK (rc = 0) 03:08:53 INFO [http_cmd]: http status code OK 03:08:53 INFO [http_cmd]: http status code OK 03:08:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713665333" 03:08:53 INFO [http_cmd]: http status code OK 03:08:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665333" 03:08:53 INFO [http_cmd]: http status code OK 03:08:53 INFO [http_cmd]: http status code OK 03:08:53 INFO [http_cmd]: http status code OK 03:08:53 INFO [http_cmd]: http status code OK 03:08:53 INFO [http_cmd]: http status code OK 03:08:53 INFO [http_cmd]: http status code OK 03:08:53 INFO [http_cmd]: http status code OK 03:08:53 INFO [http_cmd]: http status code OK 03:08:53 INFO [http_cmd]: http status code OK 03:08:53 INFO [http_cmd]: http status code OK 03:08:53 INFO [http_cmd]: http status code OK 03:08:53 DEBUG --- status code --- 03:08:53 DEBUG --- stdout --- 03:08:53 INFO [http_cmd]: http status code OK 03:08:53 DEBUG --- status code --- 03:08:53 INFO [http_cmd]: http status code OK 03:08:53 INFO [http_cmd]: http status code OK 03:08:53 DEBUG --- status code --- 03:08:53 INFO [http_cmd]: http status code OK 03:08:53 DEBUG --- status code --- 03:08:53 INFO [http_cmd]: http status code OK 03:08:53 INFO [http_cmd]: http status code OK 03:08:53 DEBUG --- status code --- 03:08:53 INFO [http_cmd]: http status code OK 03:08:53 DEBUG --- status code --- 03:08:53 DEBUG --- status code --- 03:08:53 DEBUG --- status code --- 03:08:53 DEBUG --- status code --- 03:08:53 DEBUG --- status code --- 03:08:53 DEBUG --- status code --- 03:08:53 DEBUG --- status code --- 03:08:53 INFO [http_cmd]: http status code OK 03:08:53 DEBUG --- status code --- 03:08:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:08:53 DEBUG --- http response --- 03:08:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:08:53 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 62m 0% 6292Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 9949m 33% 7770Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 21910m 73% 11594Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 82m 0% 3370Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 19815m 66% 15635Mi 14% gke-xlou-cdm-default-pool-1b0769c8-wpwl 156m 0% 5265Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 22404m 75% 12055Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 19594m 65% 3182Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 24635m 82% 11089Mi 9% gke-xlou-cdm-ds-0f601751-j488 10355m 34% 7374Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 19963m 66% 17413Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 10146m 33% 8007Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20220m 67% 14963Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 11989m 75% 11144Mi 19% 03:08:53 DEBUG --- status code --- 03:08:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:08:53 DEBUG --- status code --- 03:08:53 DEBUG --- status code --- 03:08:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:08:53 DEBUG --- status code --- 03:08:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:08:53 DEBUG --- status code --- 03:08:53 DEBUG --- status code --- 03:08:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:08:53 DEBUG --- status code --- 03:08:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:08:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:08:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:08:53 INFO [http_cmd]: http status code OK 03:08:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:08:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:08:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:08:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:08:53 DEBUG --- status code --- 03:08:53 DEBUG --- status code --- 03:08:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:08:53 DEBUG ----- output ----- 03:08:53 DEBUG --- http response --- 03:08:53 DEBUG --- stderr --- 03:08:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:08:53 DEBUG --- http response --- 03:08:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:08:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:08:53 DEBUG --- http response --- 03:08:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:08:53 DEBUG --- http response --- 03:08:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:08:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:08:53 DEBUG --- http response --- 03:08:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:08:53 DEBUG --- http response --- 03:08:53 DEBUG --- http response --- 03:08:53 DEBUG --- http response --- 03:08:53 DEBUG --- status code --- 03:08:53 DEBUG --- http response --- 03:08:53 DEBUG --- http response --- 03:08:53 DEBUG --- http response --- 03:08:53 DEBUG --- http response --- 03:08:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:08:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:08:53 INFO [http_cmd]: http status code OK 03:08:53 DEBUG --- status code --- 03:08:53 DEBUG { 03:08:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:08:53 DEBUG --- http response --- 03:08:53 DEBUG 03:08:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:08:53 DEBUG --- http response --- 03:08:53 DEBUG --- http response --- 03:08:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:08:53 DEBUG --- http response --- 03:08:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:08:53 DEBUG --- http response --- 03:08:53 DEBUG --- http response --- 03:08:53 DEBUG --- http response --- 03:08:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:08:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:08:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:08:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:08:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:08:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:08:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:08:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:08:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 03:08:53 DEBUG --- http response --- 03:08:53 DEBUG --- http response --- 03:08:53 DEBUG --- status code --- 03:08:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:08:53 DEBUG "status": "success", 03:08:53 DEBUG ----- output ----- 03:08:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:08:53 DEBUG ----- output ----- 03:08:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:08:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 03:08:53 DEBUG ----- output ----- 03:08:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:08:53 DEBUG ----- output ----- 03:08:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:08:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:08:53 DEBUG ----- output ----- 03:08:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:08:53 DEBUG ----- output ----- 03:08:53 DEBUG ----- output ----- 03:08:53 DEBUG ----- output ----- 03:08:53 DEBUG --- http response --- 03:08:53 DEBUG ----- output ----- 03:08:53 DEBUG ----- output ----- 03:08:53 DEBUG ----- output ----- 03:08:53 DEBUG ----- output ----- 03:08:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:08:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:08:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:08:53 DEBUG --- http response --- 03:08:53 DEBUG "data": { 03:08:53 DEBUG { 03:08:53 DEBUG ----- output ----- 03:08:53 DEBUG { 03:08:53 DEBUG "status": "success", 03:08:53 DEBUG ----- output ----- 03:08:53 DEBUG { 03:08:53 DEBUG ----- output ----- 03:08:53 DEBUG { 03:08:53 DEBUG ----- output ----- 03:08:53 DEBUG ----- output ----- 03:08:53 DEBUG { 03:08:53 DEBUG "status": "success", 03:08:53 DEBUG "data": { 03:08:53 DEBUG { 03:08:53 DEBUG { 03:08:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:08:53 DEBUG { 03:08:53 DEBUG { 03:08:53 DEBUG { 03:08:53 DEBUG { 03:08:53 DEBUG ----- output ----- 03:08:53 DEBUG ----- output ----- 03:08:53 DEBUG --- http response --- 03:08:53 DEBUG "resultType": "vector", 03:08:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:08:53 DEBUG "status": "success", 03:08:53 DEBUG { 03:08:53 DEBUG "data": { 03:08:53 DEBUG ----- output ----- 03:08:53 DEBUG { 03:08:53 DEBUG "status": "success", 03:08:53 DEBUG { 03:08:53 DEBUG "status": "success", 03:08:53 DEBUG { 03:08:53 DEBUG { 03:08:53 DEBUG { 03:08:53 DEBUG "resultType": "vector", 03:08:53 DEBUG "status": "success", 03:08:53 DEBUG ----- output ----- 03:08:53 DEBUG "status": "success", 03:08:53 DEBUG ----- output ----- 03:08:53 DEBUG "status": "success", 03:08:53 DEBUG "status": "success", 03:08:53 DEBUG "status": "success", 03:08:53 DEBUG "status": "success", 03:08:53 DEBUG { 03:08:53 DEBUG { 03:08:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:08:53 DEBUG "result": [ 03:08:53 DEBUG ----- output ----- 03:08:53 DEBUG "data": { 03:08:53 DEBUG "status": "success", 03:08:53 DEBUG "resultType": "vector", 03:08:53 DEBUG { 03:08:53 DEBUG "status": "success", 03:08:53 DEBUG "data": { 03:08:53 DEBUG "status": "success", 03:08:53 DEBUG "data": { 03:08:53 DEBUG "status": "success", 03:08:53 DEBUG "status": "success", 03:08:53 DEBUG "status": "success", 03:08:53 DEBUG "result": [ 03:08:53 DEBUG "data": { 03:08:53 DEBUG { 03:08:53 DEBUG "data": { 03:08:53 DEBUG { 03:08:53 DEBUG "data": { 03:08:53 DEBUG "data": { 03:08:53 DEBUG "data": { 03:08:53 DEBUG "data": { 03:08:53 DEBUG "status": "success", 03:08:53 DEBUG "status": "success", 03:08:53 DEBUG ----- output ----- 03:08:53 DEBUG { 03:08:53 DEBUG { 03:08:53 DEBUG "resultType": "vector", 03:08:53 DEBUG "data": { 03:08:53 DEBUG "resultType": "vector", 03:08:53 DEBUG "status": "success", 03:08:53 DEBUG "data": { 03:08:53 DEBUG "resultType": "vector", 03:08:53 DEBUG "data": { 03:08:53 DEBUG "resultType": "vector", 03:08:53 DEBUG "data": { 03:08:53 DEBUG "data": { 03:08:53 DEBUG "data": { 03:08:53 DEBUG { 03:08:53 DEBUG "resultType": "vector", 03:08:53 DEBUG "status": "success", 03:08:53 DEBUG "resultType": "vector", 03:08:53 DEBUG "status": "success", 03:08:53 DEBUG "data": { 03:08:53 DEBUG "resultType": "vector", 03:08:53 DEBUG "resultType": "vector", 03:08:53 DEBUG "resultType": "vector", 03:08:53 DEBUG "data": { 03:08:53 DEBUG "data": { 03:08:53 DEBUG { 03:08:53 DEBUG "metric": { 03:08:53 DEBUG "status": "success", 03:08:53 DEBUG "result": [ 03:08:53 DEBUG "result": [ 03:08:53 DEBUG "data": { 03:08:53 DEBUG "result": [ 03:08:53 DEBUG { 03:08:53 DEBUG "result": [ 03:08:53 DEBUG "resultType": "vector", 03:08:53 DEBUG "result": [ 03:08:53 DEBUG "resultType": "vector", 03:08:53 DEBUG "resultType": "vector", 03:08:53 DEBUG "resultType": "vector", 03:08:53 DEBUG "metric": { 03:08:53 DEBUG "result": [ 03:08:53 DEBUG "data": { 03:08:53 DEBUG "result": [ 03:08:53 DEBUG { 03:08:53 DEBUG "resultType": "vector", 03:08:53 DEBUG "result": [ 03:08:53 DEBUG "result": [ 03:08:53 DEBUG "result": [ 03:08:53 DEBUG "resultType": "vector", 03:08:53 DEBUG "resultType": "vector", 03:08:53 DEBUG "status": "success", 03:08:53 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:08:53 DEBUG "data": { 03:08:53 DEBUG { 03:08:53 DEBUG { 03:08:53 DEBUG "resultType": "vector", 03:08:53 DEBUG "metric": { 03:08:53 DEBUG "resultType": "vector", 03:08:53 DEBUG { 03:08:53 DEBUG "result": [ 03:08:53 DEBUG { 03:08:53 DEBUG "result": [ 03:08:53 DEBUG "result": [ 03:08:53 DEBUG "result": [ 03:08:53 DEBUG "pod": "am-767f6b67b5-9zw22" 03:08:53 DEBUG { 03:08:53 DEBUG "resultType": "vector", 03:08:53 DEBUG "resultType": "vector", 03:08:53 DEBUG "metric": { 03:08:53 DEBUG "result": [ 03:08:53 DEBUG { 03:08:53 DEBUG { 03:08:53 DEBUG { 03:08:53 DEBUG "result": [ 03:08:53 DEBUG "result": [ 03:08:53 DEBUG "data": { 03:08:53 DEBUG }, 03:08:53 DEBUG "resultType": "vector", 03:08:53 DEBUG "metric": { 03:08:53 DEBUG "metric": { 03:08:53 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:08:53 DEBUG }, 03:08:53 DEBUG "result": [ 03:08:53 DEBUG "metric": { 03:08:53 DEBUG { 03:08:53 DEBUG "result": [ 03:08:53 DEBUG { 03:08:53 DEBUG { 03:08:53 DEBUG { 03:08:53 DEBUG }, 03:08:53 DEBUG "metric": { 03:08:53 DEBUG "result": [ 03:08:53 DEBUG "result": [ 03:08:53 DEBUG "pod": "ds-idrepo-0" 03:08:53 DEBUG { 03:08:53 DEBUG "metric": { 03:08:53 DEBUG "pod": "ds-idrepo-0" 03:08:53 DEBUG "metric": { 03:08:53 DEBUG { 03:08:53 DEBUG { 03:08:53 DEBUG "resultType": "vector", 03:08:53 DEBUG "value": [ 03:08:53 DEBUG "result": [ 03:08:53 DEBUG "pod": "ds-cts-0" 03:08:53 DEBUG "value": [ 03:08:53 DEBUG { 03:08:53 DEBUG "instance": "10.142.0.107:9100" 03:08:53 DEBUG "instance": "10.142.0.107:9100" 03:08:53 DEBUG "metric": { 03:08:53 DEBUG "metric": { 03:08:53 DEBUG { 03:08:53 DEBUG "metric": { 03:08:53 DEBUG "metric": { 03:08:53 DEBUG "metric": { 03:08:53 DEBUG "value": [ 03:08:53 DEBUG "pod": "ds-cts-0" 03:08:53 DEBUG { 03:08:53 DEBUG { 03:08:53 DEBUG }, 03:08:53 DEBUG "metric": { 03:08:53 DEBUG }, 03:08:53 DEBUG "metric": { 03:08:53 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:08:53 DEBUG "metric": { 03:08:53 DEBUG "metric": { 03:08:53 DEBUG "result": [ 03:08:53 DEBUG 03:08:53 DEBUG { 03:08:53 DEBUG }, 03:08:53 DEBUG 03:08:53 DEBUG "metric": { 03:08:53 DEBUG }, 03:08:53 DEBUG }, 03:08:53 DEBUG "instance": "10.142.0.107:9100" 03:08:53 DEBUG "pod": "am-767f6b67b5-9zw22" 03:08:53 DEBUG "metric": { 03:08:53 DEBUG "instance": "10.142.0.107:9100" 03:08:53 DEBUG "pod": "am-767f6b67b5-9zw22" 03:08:53 DEBUG }, 03:08:53 DEBUG 03:08:53 DEBUG }, 03:08:53 DEBUG "metric": { 03:08:53 DEBUG "metric": { 03:08:53 DEBUG "value": [ 03:08:53 DEBUG "pod": "am-767f6b67b5-9zw22" 03:08:53 DEBUG "value": [ 03:08:53 DEBUG "pod": "ds-idrepo-0" 03:08:53 DEBUG }, 03:08:53 DEBUG "value": [ 03:08:53 DEBUG "pod": "am-767f6b67b5-9zw22" 03:08:53 DEBUG { 03:08:53 DEBUG 03:08:53 DEBUG "metric": { 03:08:53 DEBUG "value": [ 03:08:53 DEBUG 03:08:53 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:08:53 DEBUG "value": [ 03:08:53 DEBUG 03:08:53 DEBUG }, 03:08:53 DEBUG }, 03:08:53 DEBUG "value": [ 03:08:53 DEBUG }, 03:08:53 DEBUG "value": [ 03:08:53 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:08:53 DEBUG }, 03:08:53 DEBUG "value": [ 03:08:53 DEBUG 03:08:53 DEBUG 03:08:53 DEBUG 03:08:53 DEBUG "instance": "10.142.0.107:9100" 03:08:53 DEBUG 03:08:53 DEBUG }, 03:08:53 DEBUG 03:08:53 DEBUG }, 03:08:53 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:08:53 DEBUG "metric": { 03:08:53 DEBUG 03:08:53 DEBUG "pod": "ds-cts-0" 03:08:53 DEBUG 03:08:53 DEBUG 03:08:53 DEBUG }, 03:08:53 DEBUG 03:08:53 DEBUG "value": [ 03:08:53 DEBUG "value": [ 03:08:53 DEBUG 03:08:53 DEBUG "value": [ 03:08:53 DEBUG 03:08:53 DEBUG "pod": "am-767f6b67b5-9zw22" 03:08:53 DEBUG "value": [ 03:08:53 DEBUG 03:08:53 DEBUG 03:08:53 DEBUG "pod": "ds-idrepo-0" 03:08:53 DEBUG 03:08:53 DEBUG }, 03:08:53 DEBUG }, 03:08:53 DEBUG 03:08:53 DEBUG "value": [ 03:08:53 DEBUG 03:08:53 DEBUG "value": [ 03:08:53 DEBUG 03:08:53 DEBUG "pod": "am-767f6b67b5-9zw22" 03:08:53 DEBUG [...] 03:08:53 DEBUG }, 03:08:53 DEBUG 03:08:53 DEBUG [...] 03:08:53 DEBUG "value": [ 03:08:53 DEBUG 03:08:53 DEBUG 03:08:53 DEBUG 03:08:53 DEBUG 03:08:53 DEBUG 03:08:53 DEBUG 03:08:53 DEBUG }, 03:08:53 DEBUG 03:08:53 DEBUG 03:08:53 DEBUG [...] 03:08:53 DEBUG }, 03:08:53 DEBUG "value": [ 03:08:53 DEBUG 03:08:53 DEBUG "value": [ 03:08:53 DEBUG 03:08:53 DEBUG 03:08:53 DEBUG 03:08:53 DEBUG }, 03:08:53 DEBUG "value": [ 03:08:53 DEBUG }, 03:08:53 DEBUG 03:08:53 DEBUG "value": [ 03:08:53 DEBUG 03:08:53 DEBUG 03:08:53 DEBUG 03:08:53 DEBUG [...] 03:08:53 DEBUG 03:08:53 DEBUG 03:08:53 DEBUG 03:08:53 DEBUG 03:08:53 DEBUG 03:08:53 DEBUG "value": [ 03:08:53 DEBUG 03:08:53 DEBUG [...] 03:08:53 DEBUG 03:08:53 DEBUG 03:08:53 DEBUG "value": [ 03:08:53 DEBUG 03:08:53 DEBUG 03:08:53 DEBUG [...] 03:08:53 DEBUG 03:08:53 DEBUG [...] 03:08:53 DEBUG 03:08:53 DEBUG "value": [ 03:08:53 DEBUG 03:08:53 DEBUG 03:08:53 DEBUG 03:08:53 DEBUG [...] 03:08:53 DEBUG 03:08:53 DEBUG 03:08:53 DEBUG 03:08:53 DEBUG 03:08:53 DEBUG 03:08:53 DEBUG [...] 03:08:53 DEBUG 03:08:53 DEBUG [...] 03:08:53 DEBUG 03:08:53 DEBUG 03:08:53 DEBUG 03:08:53 DEBUG 03:08:53 DEBUG 03:08:53 DEBUG 03:08:53 DEBUG 03:08:53 DEBUG [...] 03:08:53 DEBUG 03:08:53 DEBUG 03:08:53 DEBUG 03:08:53 DEBUG 03:08:53 DEBUG 03:08:53 DEBUG 03:08:53 DEBUG 03:08:53 DEBUG 03:08:53 DEBUG 03:08:53 DEBUG 03:08:53 DEBUG 03:08:53 DEBUG 03:08:53 DEBUG [...] 03:08:53 DEBUG [...] 03:08:53 DEBUG 03:08:53 DEBUG [...] 03:08:53 DEBUG 03:08:53 DEBUG 03:08:53 DEBUG [...] 03:08:53 DEBUG 03:08:53 DEBUG 03:08:53 DEBUG [...] 03:08:53 DEBUG 03:08:53 DEBUG 03:08:53 DEBUG 03:08:53 DEBUG 03:08:53 DEBUG [...] 03:08:53 DEBUG 03:08:53 DEBUG 03:08:53 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:08:53 DEBUG 03:08:53 DEBUG [...] 03:08:53 DEBUG 03:08:53 DEBUG 03:08:53 DEBUG 03:08:53 DEBUG [...] 03:08:53 DEBUG 03:08:53 DEBUG 03:08:53 DEBUG 03:08:53 DEBUG 03:08:53 DEBUG 03:08:53 DEBUG 03:08:53 DEBUG 03:08:53 DEBUG 03:08:53 DEBUG 03:08:53 DEBUG "pod": "am-767f6b67b5-vbltz" 03:08:53 DEBUG 03:08:53 DEBUG 03:08:53 DEBUG [...] 03:08:53 DEBUG 03:08:53 DEBUG 03:08:53 DEBUG 03:08:53 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:08:53 DEBUG 03:08:53 DEBUG }, 03:08:53 DEBUG 03:08:53 DEBUG 03:08:53 DEBUG [...] 03:08:53 DEBUG "pod": "ds-cts-2" 03:08:53 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:08:53 DEBUG 03:08:53 DEBUG "instance": "10.142.0.43:9100" 03:08:53 DEBUG 03:08:53 DEBUG 03:08:53 DEBUG 03:08:53 DEBUG 03:08:53 DEBUG 03:08:53 DEBUG [...] 03:08:53 DEBUG 03:08:53 DEBUG "pod": "ds-cts-2" 03:08:53 DEBUG }, 03:08:53 DEBUG 03:08:53 DEBUG [...] 03:08:53 DEBUG 03:08:53 DEBUG 03:08:53 DEBUG "pod": "ds-idrepo-2" 03:08:53 DEBUG 03:08:53 DEBUG }, 03:08:53 DEBUG [...] 03:08:53 DEBUG "value": [ 03:08:53 DEBUG [...] 03:08:53 DEBUG 03:08:53 DEBUG 03:08:53 DEBUG 03:08:53 DEBUG }, 03:08:53 DEBUG 03:08:53 DEBUG 03:08:53 DEBUG "instance": "10.142.0.43:9100" 03:08:53 DEBUG 03:08:53 DEBUG "instance": "10.142.0.43:9100" 03:08:53 DEBUG 03:08:53 DEBUG "pod": "am-767f6b67b5-vbltz" 03:08:53 DEBUG 03:08:53 DEBUG 03:08:53 DEBUG }, 03:08:53 DEBUG "value": [ 03:08:53 DEBUG 03:08:53 DEBUG 03:08:53 DEBUG 03:08:53 DEBUG "pod": "ds-idrepo-2" 03:08:53 DEBUG }, 03:08:53 DEBUG 03:08:53 DEBUG "value": [ 03:08:53 DEBUG 03:08:53 DEBUG 1713665333, 03:08:53 DEBUG "916729856" 03:08:53 DEBUG ] 03:08:53 DEBUG 03:08:53 DEBUG }, 03:08:53 DEBUG "value": [ 03:08:53 DEBUG }, 03:08:53 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:08:53 DEBUG }, 03:08:53 DEBUG 03:08:53 DEBUG }, 03:08:53 DEBUG "instance": "10.142.0.43:9100" 03:08:53 DEBUG }, 03:08:53 DEBUG "value": [ 03:08:53 DEBUG "pod": "am-767f6b67b5-vbltz" 03:08:53 DEBUG "value": [ 03:08:53 DEBUG 1713665333, 03:08:53 DEBUG "pod": "ds-idrepo-2" 03:08:53 DEBUG 03:08:53 DEBUG 03:08:53 DEBUG }, 03:08:53 DEBUG "value": [ 03:08:53 DEBUG "pod": "ds-cts-2" 03:08:53 DEBUG 1713665333, 03:08:53 DEBUG "0" 03:08:53 DEBUG 03:08:53 DEBUG 03:08:53 DEBUG } 03:08:53 DEBUG "pod": "am-767f6b67b5-vbltz" 03:08:53 DEBUG "value": [ 03:08:53 DEBUG 1713665333, 03:08:53 DEBUG "value": [ 03:08:53 DEBUG }, 03:08:53 DEBUG "value": [ 03:08:53 DEBUG "pod": "am-767f6b67b5-vbltz" 03:08:53 DEBUG "value": [ 03:08:53 DEBUG }, 03:08:53 DEBUG 1713665333, 03:08:53 DEBUG 03:08:53 DEBUG }, 03:08:53 DEBUG 1713665333, 03:08:53 DEBUG "0.5624666666666664" 03:08:53 DEBUG }, 03:08:53 DEBUG 03:08:53 DEBUG "pod": "am-767f6b67b5-vbltz" 03:08:53 DEBUG "value": [ 03:08:53 DEBUG 1713665333, 03:08:53 DEBUG "0" 03:08:53 DEBUG ] 03:08:53 DEBUG 03:08:53 DEBUG 03:08:53 DEBUG 03:08:53 DEBUG ] 03:08:53 DEBUG }, 03:08:53 DEBUG 1713665333, 03:08:53 DEBUG "11.746690705667977" 03:08:53 DEBUG 1713665333, 03:08:53 DEBUG "value": [ 03:08:53 DEBUG 1713665333, 03:08:53 DEBUG }, 03:08:53 DEBUG "value": [ 03:08:53 DEBUG "value": [ 03:08:53 DEBUG "3334.366666666667" 03:08:53 DEBUG 03:08:53 DEBUG "value": [ 03:08:53 DEBUG 1713665333, 03:08:53 DEBUG "0" 03:08:53 DEBUG ] 03:08:53 DEBUG ] 03:08:53 DEBUG }, 03:08:53 DEBUG 1713665333, 03:08:53 DEBUG }, 03:08:53 DEBUG "value": [ 03:08:53 DEBUG } 03:08:53 DEBUG "pod": "ds-cts-2" 03:08:53 DEBUG 03:08:53 DEBUG "pod": "am-767f6b67b5-vbltz" 03:08:53 DEBUG } 03:08:53 DEBUG "value": [ 03:08:53 DEBUG 1713665333, 03:08:53 DEBUG "3332.4333333333334" 03:08:53 DEBUG "100014642.16666666" 03:08:53 DEBUG 1713665333, 03:08:53 DEBUG "0.4387666666665874" 03:08:53 DEBUG 1713665333, 03:08:53 DEBUG 1713665333, 03:08:53 DEBUG 1713665333, 03:08:53 DEBUG ] 03:08:53 DEBUG "pod": "am-767f6b67b5-vbltz" 03:08:53 DEBUG "instance": "10.142.0.43:9100" 03:08:53 DEBUG } 03:08:53 DEBUG "600051" 03:08:53 DEBUG } 03:08:53 DEBUG "value": [ 03:08:53 DEBUG "value": [ 03:08:53 DEBUG "0" 03:08:53 DEBUG ] 03:08:53 DEBUG 1713665333, 03:08:53 DEBUG ] 03:08:53 DEBUG }, 03:08:53 DEBUG "pod": "am-767f6b67b5-vbltz" 03:08:53 DEBUG }, 03:08:53 DEBUG } 03:08:53 DEBUG ] 03:08:53 DEBUG ] 03:08:53 DEBUG "0" 03:08:53 DEBUG ] 03:08:53 DEBUG "0" 03:08:53 DEBUG ] 03:08:53 DEBUG "0.10166666666665568" 03:08:53 DEBUG "3314.4333333333334" 03:08:53 DEBUG "100018936.23333333" 03:08:53 DEBUG } 03:08:53 DEBUG }, 03:08:53 DEBUG "value": [ 03:08:53 DEBUG ] 03:08:53 DEBUG ] 03:08:53 DEBUG ] 03:08:53 DEBUG 1713665333, 03:08:53 DEBUG 1713665333, 03:08:53 DEBUG "0" 03:08:53 DEBUG } 03:08:53 DEBUG "0" 03:08:53 DEBUG } 03:08:53 DEBUG "value": [ 03:08:53 DEBUG }, 03:08:53 DEBUG "value": [ 03:08:53 DEBUG "value": [ 03:08:53 DEBUG } 03:08:53 DEBUG ] 03:08:53 DEBUG } 03:08:53 DEBUG ] 03:08:53 DEBUG } 03:08:53 DEBUG ] 03:08:53 DEBUG ] 03:08:53 DEBUG ] 03:08:53 DEBUG ] 03:08:53 DEBUG 1713665333, 03:08:53 DEBUG "3337.0666666666666" 03:08:53 DEBUG } 03:08:53 DEBUG } 03:08:53 DEBUG ] 03:08:53 DEBUG "149.889" 03:08:53 DEBUG ] 03:08:53 DEBUG ] 03:08:53 DEBUG } 03:08:53 DEBUG ] 03:08:53 DEBUG } 03:08:53 DEBUG 1713665333, 03:08:53 DEBUG } 03:08:53 DEBUG 1713665333, 03:08:53 DEBUG 1713665333, 03:08:53 DEBUG ] 03:08:53 DEBUG } 03:08:53 DEBUG ] 03:08:53 DEBUG } 03:08:53 DEBUG ] 03:08:53 DEBUG } 03:08:53 DEBUG } 03:08:53 DEBUG } 03:08:53 DEBUG } 03:08:53 DEBUG ] 03:08:53 DEBUG }, 03:08:53 DEBUG } 03:08:53 DEBUG } 03:08:53 DEBUG ] 03:08:53 DEBUG } 03:08:53 DEBUG ] 03:08:53 DEBUG } 03:08:53 DEBUG } 03:08:53 DEBUG } 03:08:53 DEBUG "0" 03:08:53 DEBUG ] 03:08:53 DEBUG "0" 03:08:53 DEBUG "0" 03:08:53 DEBUG } 03:08:53 DEBUG ] 03:08:53 DEBUG } 03:08:53 DEBUG } 03:08:53 DEBUG } 03:08:53 DEBUG ] 03:08:53 DEBUG ] 03:08:53 DEBUG ] 03:08:53 DEBUG } 03:08:53 DEBUG } 03:08:53 DEBUG "value": [ 03:08:53 DEBUG } 03:08:53 DEBUG } 03:08:53 DEBUG ] 03:08:53 DEBUG } 03:08:53 DEBUG } 03:08:53 DEBUG ] 03:08:53 DEBUG ] 03:08:53 DEBUG } 03:08:53 DEBUG ] 03:08:53 DEBUG ] 03:08:53 DEBUG } 03:08:53 DEBUG } 03:08:53 DEBUG } 03:08:53 DEBUG } 03:08:53 DEBUG } 03:08:53 DEBUG } 03:08:53 DEBUG } 03:08:53 DEBUG ] 03:08:53 DEBUG 1713665333, 03:08:53 DEBUG ] 03:08:53 DEBUG } 03:08:53 DEBUG ] 03:08:53 DEBUG } 03:08:53 DEBUG } 03:08:53 DEBUG } 03:08:53 DEBUG } 03:08:53 DEBUG } 03:08:53 DEBUG ] 03:08:53 DEBUG } 03:08:53 DEBUG } 03:08:53 DEBUG } 03:08:53 DEBUG } 03:08:53 DEBUG "4340084202496" 03:08:53 DEBUG ] 03:08:53 DEBUG } 03:08:53 DEBUG } 03:08:53 DEBUG } 03:08:53 DEBUG ] 03:08:53 DEBUG } 03:08:53 DEBUG ] 03:08:53 DEBUG } 03:08:53 DEBUG } 03:08:53 DEBUG } 03:08:53 DEBUG } 03:08:53 DEBUG } 03:08:53 DEBUG } 03:08:53 DEBUG } 03:08:53 DEBUG ] 03:08:53 DEBUG } 03:08:53 DEBUG } 03:08:53 DEBUG ] 03:08:53 DEBUG } 03:08:53 DEBUG } 03:08:53 DEBUG } 03:08:53 DEBUG } 03:08:53 DEBUG } 03:09:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713665392" 03:09:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665392" 03:09:52 ERROR [http_cmd]: ERROR 03:09:52 INFO [http_cmd]: http status code OK 03:09:52 ERROR --- status code --- 03:09:52 ERROR http status code is 400 (expected [200, 401, 403, 404]) 03:09:52 ERROR --- http response --- 03:09:52 DEBUG --- status code --- 03:09:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:09:52 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 03:09:52 DEBUG --- http response --- 03:09:52 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 03:09:52 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 03:09:53 INFO 03:09:53 INFO [loop_until]: kubectl --namespace=xlou top pods 03:09:53 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:09:53 INFO [loop_until]: OK (rc = 0) 03:09:53 DEBUG --- stdout --- 03:09:53 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 22792m 8794Mi am-767f6b67b5-qcqz2 22053m 9122Mi am-767f6b67b5-vbltz 23921m 8819Mi ds-cts-0 20161m 12317Mi ds-cts-1 20082m 12287Mi ds-cts-2 20545m 12371Mi ds-idrepo-0 9768m 3930Mi ds-idrepo-1 9552m 3946Mi ds-idrepo-2 9786m 3916Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1179Mi idm-b85f9d978-wnhxb 3m 1239Mi lodemon-54c87f8cbb-jzrjs 17m 62Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-68c5d5b884-p97w5 11447m 787Mi 03:09:53 DEBUG --- stderr --- 03:09:53 DEBUG 03:09:53 INFO 03:09:53 INFO [loop_until]: kubectl --namespace=xlou top node 03:09:53 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:09:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713665393" 03:09:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665393" 03:09:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665393" 03:09:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665393" 03:09:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665393" 03:09:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665393" 03:09:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665393" 03:09:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665393" 03:09:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665393" 03:09:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665393" 03:09:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713665393" 03:09:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665393" 03:09:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665393" 03:09:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665393" 03:09:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665393" 03:09:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665393" 03:09:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665393" 03:09:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665393" 03:09:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713665393" 03:09:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713665393" 03:09:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665393" 03:09:53 INFO [http_cmd]: http status code OK 03:09:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665393" 03:09:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665393" 03:09:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665393" 03:09:53 INFO [loop_until]: OK (rc = 0) 03:09:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665393" 03:09:53 INFO [http_cmd]: http status code OK 03:09:53 INFO [http_cmd]: http status code OK 03:09:53 INFO [http_cmd]: http status code OK 03:09:53 INFO [http_cmd]: http status code OK 03:09:53 INFO [http_cmd]: http status code OK 03:09:53 INFO [http_cmd]: http status code OK 03:09:53 INFO [http_cmd]: http status code OK 03:09:53 INFO [http_cmd]: http status code OK 03:09:53 INFO [http_cmd]: http status code OK 03:09:53 INFO [http_cmd]: http status code OK 03:09:53 INFO [http_cmd]: http status code OK 03:09:53 INFO [http_cmd]: http status code OK 03:09:53 DEBUG --- status code --- 03:09:53 INFO [http_cmd]: http status code OK 03:09:53 INFO [http_cmd]: http status code OK 03:09:53 INFO [http_cmd]: http status code OK 03:09:53 INFO [http_cmd]: http status code OK 03:09:53 INFO [http_cmd]: http status code OK 03:09:53 INFO [http_cmd]: http status code OK 03:09:53 DEBUG --- stdout --- 03:09:53 DEBUG --- status code --- 03:09:53 DEBUG --- status code --- 03:09:53 INFO [http_cmd]: http status code OK 03:09:53 DEBUG --- status code --- 03:09:53 DEBUG --- status code --- 03:09:53 DEBUG --- status code --- 03:09:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:09:53 INFO [http_cmd]: http status code OK 03:09:53 INFO [http_cmd]: http status code OK 03:09:53 DEBUG --- status code --- 03:09:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:09:53 DEBUG --- status code --- 03:09:53 DEBUG --- status code --- 03:09:53 DEBUG --- status code --- 03:09:53 DEBUG --- status code --- 03:09:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:09:53 DEBUG --- status code --- 03:09:53 DEBUG --- status code --- 03:09:53 DEBUG --- status code --- 03:09:53 DEBUG --- status code --- 03:09:53 DEBUG --- status code --- 03:09:53 DEBUG --- status code --- 03:09:53 INFO [http_cmd]: http status code OK 03:09:53 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 63m 0% 6280Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 10023m 33% 7769Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 22099m 74% 11605Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 84m 0% 3372Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 19837m 66% 15606Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 144m 0% 5282Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 22349m 74% 12055Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 19845m 66% 3201Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 24695m 82% 11084Mi 9% gke-xlou-cdm-ds-0f601751-j488 10404m 34% 7370Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 19864m 66% 17410Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 10221m 34% 8016Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20366m 68% 15089Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 12069m 75% 11222Mi 19% 03:09:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:09:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:09:53 DEBUG --- status code --- 03:09:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:09:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:09:53 DEBUG --- http response --- 03:09:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:09:53 DEBUG --- status code --- 03:09:53 DEBUG --- status code --- 03:09:53 DEBUG --- http response --- 03:09:53 DEBUG --- status code --- 03:09:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:09:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:09:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:09:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:09:53 DEBUG --- http response --- 03:09:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:09:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:09:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:09:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:09:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:09:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:09:53 DEBUG --- http response --- 03:09:53 DEBUG --- stderr --- 03:09:53 DEBUG 03:09:53 DEBUG --- http response --- 03:09:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:09:53 DEBUG --- http response --- 03:09:53 DEBUG --- status code --- 03:09:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 03:09:53 DEBUG --- http response --- 03:09:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:09:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:09:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:09:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:09:53 DEBUG --- http response --- 03:09:53 DEBUG --- http response --- 03:09:53 DEBUG --- http response --- 03:09:53 DEBUG --- http response --- 03:09:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:09:53 DEBUG --- http response --- 03:09:53 DEBUG --- http response --- 03:09:53 DEBUG --- http response --- 03:09:53 DEBUG --- http response --- 03:09:53 DEBUG --- http response --- 03:09:53 DEBUG --- status code --- 03:09:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:09:53 DEBUG --- http response --- 03:09:53 INFO [http_cmd]: http status code OK 03:09:53 INFO [http_cmd]: http status code OK 03:09:53 DEBUG ----- output ----- 03:09:53 DEBUG --- http response --- 03:09:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:09:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:09:53 DEBUG ----- output ----- 03:09:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:09:53 DEBUG --- http response --- 03:09:53 DEBUG --- http response --- 03:09:53 DEBUG ----- output ----- 03:09:53 DEBUG --- http response --- 03:09:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:09:53 DEBUG ----- output ----- 03:09:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:09:53 DEBUG ----- output ----- 03:09:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:09:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:09:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:09:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:09:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:09:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:09:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:09:53 DEBUG ----- output ----- 03:09:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:09:53 DEBUG --- status code --- 03:09:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:09:53 DEBUG { 03:09:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:09:53 DEBUG --- http response --- 03:09:53 DEBUG ----- output ----- 03:09:53 DEBUG { 03:09:53 DEBUG ----- output ----- 03:09:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:09:53 DEBUG ----- output ----- 03:09:53 DEBUG { 03:09:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:09:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:09:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:09:53 DEBUG { 03:09:53 DEBUG ----- output ----- 03:09:53 DEBUG ----- output ----- 03:09:53 DEBUG ----- output ----- 03:09:53 DEBUG ----- output ----- 03:09:53 DEBUG ----- output ----- 03:09:53 DEBUG ----- output ----- 03:09:53 DEBUG ----- output ----- 03:09:53 DEBUG --- http response --- 03:09:53 DEBUG { 03:09:53 DEBUG ----- output ----- 03:09:53 DEBUG --- status code --- 03:09:53 DEBUG --- http response --- 03:09:53 DEBUG "status": "success", 03:09:53 DEBUG ----- output ----- 03:09:53 DEBUG { 03:09:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:09:53 DEBUG "status": "success", 03:09:53 DEBUG { 03:09:53 DEBUG { 03:09:53 DEBUG "status": "success", 03:09:53 DEBUG { 03:09:53 DEBUG ----- output ----- 03:09:53 DEBUG ----- output ----- 03:09:53 DEBUG ----- output ----- 03:09:53 DEBUG "status": "success", 03:09:53 DEBUG { 03:09:53 DEBUG { 03:09:53 DEBUG { 03:09:53 DEBUG { 03:09:53 DEBUG { 03:09:53 DEBUG { 03:09:53 DEBUG { 03:09:53 DEBUG "status": "success", 03:09:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:09:53 DEBUG { 03:09:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:09:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:09:53 DEBUG ----- output ----- 03:09:53 DEBUG { 03:09:53 DEBUG "status": "success", 03:09:53 DEBUG ----- output ----- 03:09:53 DEBUG "data": { 03:09:53 DEBUG "resultType": "vector", 03:09:53 DEBUG "status": "success", 03:09:53 DEBUG "data": { 03:09:53 DEBUG "status": "success", 03:09:53 DEBUG { 03:09:53 DEBUG { 03:09:53 DEBUG { 03:09:53 DEBUG "data": { 03:09:53 DEBUG "status": "success", 03:09:53 DEBUG "status": "success", 03:09:53 DEBUG "status": "success", 03:09:53 DEBUG "status": "success", 03:09:53 DEBUG "status": "success", 03:09:53 DEBUG "status": "success", 03:09:53 DEBUG "status": "success", 03:09:53 DEBUG "data": { 03:09:53 DEBUG "resultType": "vector", 03:09:53 DEBUG "status": "success", 03:09:53 DEBUG --- http response --- 03:09:53 DEBUG { 03:09:53 DEBUG "status": "success", 03:09:53 DEBUG "data": { 03:09:53 DEBUG "data": { 03:09:53 DEBUG { 03:09:53 DEBUG "result": [ 03:09:53 DEBUG "data": { 03:09:53 DEBUG "resultType": "vector", 03:09:53 DEBUG "status": "success", 03:09:53 DEBUG "data": { 03:09:53 DEBUG "resultType": "vector", 03:09:53 DEBUG "result": [ 03:09:53 DEBUG "status": "success", 03:09:53 DEBUG "resultType": "vector", 03:09:53 DEBUG "data": { 03:09:53 DEBUG "resultType": "vector", 03:09:53 DEBUG "result": [ 03:09:53 DEBUG "data": { 03:09:53 DEBUG "data": { 03:09:53 DEBUG "data": { 03:09:53 DEBUG "data": { 03:09:53 DEBUG "result": [ 03:09:53 DEBUG { 03:09:53 DEBUG ----- output ----- 03:09:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 03:09:53 DEBUG "status": "success", 03:09:53 DEBUG "data": { 03:09:53 DEBUG "resultType": "vector", 03:09:53 DEBUG "resultType": "vector", 03:09:53 DEBUG "status": "success", 03:09:53 DEBUG { 03:09:53 DEBUG "resultType": "vector", 03:09:53 DEBUG "result": [ 03:09:53 DEBUG "data": { 03:09:53 DEBUG { 03:09:53 DEBUG "status": "success", 03:09:53 DEBUG "data": { 03:09:53 DEBUG "status": "success", 03:09:53 DEBUG "result": [ 03:09:53 DEBUG "data": { 03:09:53 DEBUG "data": { 03:09:53 DEBUG { 03:09:53 DEBUG "resultType": "vector", 03:09:53 DEBUG "resultType": "vector", 03:09:53 DEBUG "resultType": "vector", 03:09:53 DEBUG "resultType": "vector", 03:09:53 DEBUG "metric": { 03:09:53 DEBUG { 03:09:53 DEBUG ----- output ----- 03:09:53 DEBUG "data": { 03:09:53 DEBUG "resultType": "vector", 03:09:53 DEBUG "data": { 03:09:53 DEBUG "result": [ 03:09:53 DEBUG "result": [ 03:09:53 DEBUG "data": { 03:09:53 DEBUG "metric": { 03:09:53 DEBUG "result": [ 03:09:53 DEBUG { 03:09:53 DEBUG "resultType": "vector", 03:09:53 DEBUG "metric": { 03:09:53 DEBUG "resultType": "vector", 03:09:53 DEBUG "data": { 03:09:53 DEBUG { 03:09:53 DEBUG "resultType": "vector", 03:09:53 DEBUG "resultType": "vector", 03:09:53 DEBUG "metric": { 03:09:53 DEBUG "result": [ 03:09:53 DEBUG "result": [ 03:09:53 DEBUG "result": [ 03:09:53 DEBUG "result": [ 03:09:53 DEBUG "data": { 03:09:53 DEBUG "pod": "ds-idrepo-0" 03:09:53 DEBUG "status": "success", 03:09:53 DEBUG { 03:09:53 DEBUG "resultType": "vector", 03:09:53 DEBUG "result": [ 03:09:53 DEBUG "resultType": "vector", 03:09:53 DEBUG "result": [ 03:09:53 DEBUG { 03:09:53 DEBUG "resultType": "vector", 03:09:53 DEBUG "pod": "ds-idrepo-1" 03:09:53 DEBUG { 03:09:53 DEBUG "metric": { 03:09:53 DEBUG "pod": "ds-idrepo-0" 03:09:53 DEBUG "pod": "am-767f6b67b5-9zw22" 03:09:53 DEBUG "result": [ 03:09:53 DEBUG "resultType": "vector", 03:09:53 DEBUG "metric": { 03:09:53 DEBUG "result": [ 03:09:53 DEBUG "result": [ 03:09:53 DEBUG "instance": "10.142.0.107:9100" 03:09:53 DEBUG { 03:09:53 DEBUG { 03:09:53 DEBUG { 03:09:53 DEBUG { 03:09:53 DEBUG "resultType": "vector", 03:09:53 DEBUG }, 03:09:53 DEBUG "data": { 03:09:53 DEBUG "status": "success", 03:09:53 DEBUG "result": [ 03:09:53 DEBUG { 03:09:53 DEBUG { 03:09:53 DEBUG { 03:09:53 DEBUG "metric": { 03:09:53 DEBUG "result": [ 03:09:53 DEBUG }, 03:09:53 DEBUG "metric": { 03:09:53 DEBUG "result": [ 03:09:53 DEBUG { 03:09:53 DEBUG }, 03:09:53 DEBUG { 03:09:53 DEBUG "result": [ 03:09:53 DEBUG "instance": "10.142.0.107:9100" 03:09:53 DEBUG { 03:09:53 DEBUG { 03:09:53 DEBUG }, 03:09:53 DEBUG "metric": { 03:09:53 DEBUG "metric": { 03:09:53 DEBUG "metric": { 03:09:53 DEBUG "metric": { 03:09:53 DEBUG "result": [ 03:09:53 DEBUG "value": [ 03:09:53 DEBUG "resultType": "vector", 03:09:53 DEBUG "data": { 03:09:53 DEBUG { 03:09:53 DEBUG "metric": { 03:09:53 DEBUG "metric": { 03:09:53 DEBUG "metric": { 03:09:53 DEBUG "pod": "ds-cts-0" 03:09:53 DEBUG { 03:09:53 DEBUG "value": [ 03:09:53 DEBUG "pod": "ds-idrepo-1" 03:09:53 DEBUG "metric": { 03:09:53 DEBUG "value": [ 03:09:53 DEBUG }, 03:09:53 DEBUG "metric": { 03:09:53 DEBUG { 03:09:53 DEBUG }, 03:09:53 DEBUG "value": [ 03:09:53 DEBUG "metric": { 03:09:53 DEBUG "pod": "am-767f6b67b5-9zw22" 03:09:53 DEBUG "value": [ 03:09:53 DEBUG "pod": "ds-idrepo-0" 03:09:53 DEBUG "pod": "ds-idrepo-0" 03:09:53 DEBUG "pod": "ds-cts-0" 03:09:53 DEBUG { 03:09:53 DEBUG 03:09:53 DEBUG "result": [ 03:09:53 DEBUG { 03:09:53 DEBUG "metric": { 03:09:53 DEBUG "pod": "am-767f6b67b5-9zw22" 03:09:53 DEBUG "instance": "10.142.0.107:9100" 03:09:53 DEBUG }, 03:09:53 DEBUG "instance": "10.142.0.107:9100" 03:09:53 DEBUG }, 03:09:53 DEBUG 03:09:53 DEBUG }, 03:09:53 DEBUG "pod": "am-767f6b67b5-9zw22" 03:09:53 DEBUG }, 03:09:53 DEBUG "value": [ 03:09:53 DEBUG "pod": "ds-cts-0" 03:09:53 DEBUG "metric": { 03:09:53 DEBUG 03:09:53 DEBUG "metric": { 03:09:53 DEBUG "instance": "10.142.0.107:9100" 03:09:53 DEBUG }, 03:09:53 DEBUG 03:09:53 DEBUG }, 03:09:53 DEBUG }, 03:09:53 DEBUG }, 03:09:53 DEBUG "metric": { 03:09:53 DEBUG 03:09:53 DEBUG "resultType": "vector", 03:09:53 DEBUG "metric": { 03:09:53 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:09:53 DEBUG }, 03:09:53 DEBUG }, 03:09:53 DEBUG "value": [ 03:09:53 DEBUG "metric": { 03:09:53 DEBUG "value": [ 03:09:53 DEBUG 03:09:53 DEBUG "value": [ 03:09:53 DEBUG "value": [ 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG }, 03:09:53 DEBUG "pod": "am-767f6b67b5-9zw22" 03:09:53 DEBUG 03:09:53 DEBUG "pod": "ds-idrepo-1" 03:09:53 DEBUG }, 03:09:53 DEBUG "value": [ 03:09:53 DEBUG 03:09:53 DEBUG "value": [ 03:09:53 DEBUG "value": [ 03:09:53 DEBUG "value": [ 03:09:53 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:09:53 DEBUG 03:09:53 DEBUG "result": [ 03:09:53 DEBUG "pod": "am-767f6b67b5-9zw22" 03:09:53 DEBUG }, 03:09:53 DEBUG "value": [ 03:09:53 DEBUG "value": [ 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG "value": [ 03:09:53 DEBUG }, 03:09:53 DEBUG 03:09:53 DEBUG }, 03:09:53 DEBUG "value": [ 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG }, 03:09:53 DEBUG [...] 03:09:53 DEBUG { 03:09:53 DEBUG }, 03:09:53 DEBUG "value": [ 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG "pod": "am-767f6b67b5-9zw22" 03:09:53 DEBUG 03:09:53 DEBUG [...] 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG "value": [ 03:09:53 DEBUG [...] 03:09:53 DEBUG "value": [ 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG [...] 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG "value": [ 03:09:53 DEBUG 03:09:53 DEBUG "metric": { 03:09:53 DEBUG "value": [ 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG [...] 03:09:53 DEBUG }, 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG [...] 03:09:53 DEBUG [...] 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG "pod": "ds-idrepo-1" 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG "value": [ 03:09:53 DEBUG [...] 03:09:53 DEBUG 03:09:53 DEBUG [...] 03:09:53 DEBUG [...] 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG "instance": "10.142.0.43:9100" 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG [...] 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG [...] 03:09:53 DEBUG [...] 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG }, 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG [...] 03:09:53 DEBUG [...] 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG [...] 03:09:53 DEBUG 03:09:53 DEBUG }, 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG "instance": "10.142.0.43:9100" 03:09:53 DEBUG [...] 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG "pod": "ds-idrepo-2" 03:09:53 DEBUG "value": [ 03:09:53 DEBUG 03:09:53 DEBUG [...] 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG "pod": "ds-cts-0" 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG [...] 03:09:53 DEBUG "value": [ 03:09:53 DEBUG 03:09:53 DEBUG [...] 03:09:53 DEBUG 03:09:53 DEBUG }, 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG [...] 03:09:53 DEBUG }, 03:09:53 DEBUG 03:09:53 DEBUG [...] 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG }, 03:09:53 DEBUG 03:09:53 DEBUG "pod": "ds-cts-0" 03:09:53 DEBUG "pod": "am-767f6b67b5-vbltz" 03:09:53 DEBUG "pod": "ds-idrepo-2" 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG 1713665393, 03:09:53 DEBUG [...] 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG "value": [ 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG "value": [ 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG "instance": "10.142.0.43:9100" 03:09:53 DEBUG "pod": "ds-cts-2" 03:09:53 DEBUG [...] 03:09:53 DEBUG "instance": "10.142.0.43:9100" 03:09:53 DEBUG "value": [ 03:09:53 DEBUG }, 03:09:53 DEBUG 03:09:53 DEBUG }, 03:09:53 DEBUG "value": [ 03:09:53 DEBUG 1713665393, 03:09:53 DEBUG 03:09:53 DEBUG "0.41963333333335034" 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG 1713665393, 03:09:53 DEBUG 03:09:53 DEBUG "pod": "ds-cts-2" 03:09:53 DEBUG "pod": "ds-cts-2" 03:09:53 DEBUG 03:09:53 DEBUG 1713665393, 03:09:53 DEBUG "0" 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG "pod": "am-767f6b67b5-vbltz" 03:09:53 DEBUG }, 03:09:53 DEBUG }, 03:09:53 DEBUG "value": [ 03:09:53 DEBUG }, 03:09:53 DEBUG 1713665393, 03:09:53 DEBUG "value": [ 03:09:53 DEBUG "pod": "am-767f6b67b5-vbltz" 03:09:53 DEBUG }, 03:09:53 DEBUG "3312.9333333333334" 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG ] 03:09:53 DEBUG 03:09:53 DEBUG "pod": "am-767f6b67b5-vbltz" 03:09:53 DEBUG "instance": "10.142.0.43:9100" 03:09:53 DEBUG "0.08399999999999368" 03:09:53 DEBUG "pod": "ds-idrepo-2" 03:09:53 DEBUG }, 03:09:53 DEBUG }, 03:09:53 DEBUG 03:09:53 DEBUG ] 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:09:53 DEBUG }, 03:09:53 DEBUG "value": [ 03:09:53 DEBUG 03:09:53 DEBUG 1713665393, 03:09:53 DEBUG "value": [ 03:09:53 DEBUG "415328627.8678562" 03:09:53 DEBUG 1713665393, 03:09:53 DEBUG }, 03:09:53 DEBUG "value": [ 03:09:53 DEBUG ] 03:09:53 DEBUG "pod": "am-767f6b67b5-vbltz" 03:09:53 DEBUG "pod": "ds-cts-2" 03:09:53 DEBUG } 03:09:53 DEBUG 03:09:53 DEBUG }, 03:09:53 DEBUG }, 03:09:53 DEBUG ] 03:09:53 DEBUG }, 03:09:53 DEBUG "value": [ 03:09:53 DEBUG "value": [ 03:09:53 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:09:53 DEBUG } 03:09:53 DEBUG [...] 03:09:53 DEBUG "pod": "am-767f6b67b5-vbltz" 03:09:53 DEBUG }, 03:09:53 DEBUG "value": [ 03:09:53 DEBUG 1713665393, 03:09:53 DEBUG 03:09:53 DEBUG 03:09:53 DEBUG 1713665393, 03:09:53 DEBUG ] 03:09:53 DEBUG "52068743.633630715" 03:09:53 DEBUG "value": [ 03:09:53 DEBUG 1713665393, 03:09:53 DEBUG } 03:09:53 DEBUG }, 03:09:53 DEBUG }, 03:09:53 DEBUG ] 03:09:53 DEBUG "pod": "ds-cts-0" 03:09:53 DEBUG "value": [ 03:09:53 DEBUG "value": [ 03:09:53 DEBUG } 03:09:53 DEBUG "value": [ 03:09:53 DEBUG 1713665393, 03:09:53 DEBUG 1713665393, 03:09:53 DEBUG }, 03:09:53 DEBUG "value": [ 03:09:53 DEBUG 03:09:53 DEBUG }, 03:09:53 DEBUG "value": [ 03:09:53 DEBUG 1713665393, 03:09:53 DEBUG "4345468898304" 03:09:53 DEBUG "pod": "am-767f6b67b5-vbltz" 03:09:53 DEBUG "0.5437666666666667" 03:09:53 DEBUG ] 03:09:53 DEBUG } 03:09:53 DEBUG ] 03:09:53 DEBUG 1713665393, 03:09:53 DEBUG "0" 03:09:53 DEBUG ] 03:09:53 DEBUG "value": [ 03:09:53 DEBUG "value": [ 03:09:53 DEBUG } 03:09:53 DEBUG }, 03:09:53 DEBUG 1713665393, 03:09:53 DEBUG 1713665393, 03:09:53 DEBUG ] 03:09:53 DEBUG 1713665393, 03:09:53 DEBUG "0" 03:09:53 DEBUG "0" 03:09:53 DEBUG 1713665393, 03:09:53 DEBUG ] 03:09:53 DEBUG 03:09:53 DEBUG "value": [ 03:09:53 DEBUG 1713665393, 03:09:53 DEBUG "149.889" 03:09:53 DEBUG ] 03:09:53 DEBUG }, 03:09:53 DEBUG ] 03:09:53 DEBUG } 03:09:53 DEBUG "100588035.63333333" 03:09:53 DEBUG } 03:09:53 DEBUG "3337.366666666667" 03:09:53 DEBUG ] 03:09:53 DEBUG } 03:09:53 DEBUG 1713665393, 03:09:53 DEBUG 1713665393, 03:09:53 DEBUG } 03:09:53 DEBUG "value": [ 03:09:53 DEBUG "3339.8333333333335" 03:09:53 DEBUG "100591288.9" 03:09:53 DEBUG } 03:09:53 DEBUG "0" 03:09:53 DEBUG ] 03:09:53 DEBUG ] 03:09:53 DEBUG "11.875007108388363" 03:09:53 DEBUG } 03:09:53 DEBUG 03:09:53 DEBUG "0" 03:09:53 DEBUG 1713665393, 03:09:53 DEBUG ] 03:09:53 DEBUG } 03:09:53 DEBUG "value": [ 03:09:53 DEBUG } 03:09:53 DEBUG ] 03:09:53 DEBUG ] 03:09:53 DEBUG ] 03:09:53 DEBUG ] 03:09:53 DEBUG } 03:09:53 DEBUG } 03:09:53 DEBUG "3338.9333333333334" 03:09:53 DEBUG "0" 03:09:53 DEBUG 1713665393, 03:09:53 DEBUG ] 03:09:53 DEBUG ] 03:09:53 DEBUG } 03:09:53 DEBUG ] 03:09:53 DEBUG } 03:09:53 DEBUG ] 03:09:53 DEBUG ] 03:09:53 DEBUG } 03:09:53 DEBUG "pod": "ds-cts-0" 03:09:53 DEBUG ] 03:09:53 DEBUG "1003614208" 03:09:53 DEBUG } 03:09:53 DEBUG ] 03:09:53 DEBUG 1713665393, 03:09:53 DEBUG } 03:09:53 DEBUG } 03:09:53 DEBUG } 03:09:53 DEBUG } 03:09:53 DEBUG } 03:09:53 DEBUG ] 03:09:53 DEBUG ] 03:09:53 DEBUG ] 03:09:53 DEBUG "46878293.16262053" 03:09:53 DEBUG } 03:09:53 DEBUG } 03:09:53 DEBUG } 03:09:53 DEBUG } 03:09:53 DEBUG } 03:09:53 DEBUG } 03:09:53 DEBUG }, 03:09:53 DEBUG } 03:09:53 DEBUG ] 03:09:53 DEBUG ] 03:09:53 DEBUG } 03:09:53 DEBUG "600529" 03:09:53 DEBUG } 03:09:53 DEBUG } 03:09:53 DEBUG ] 03:09:53 DEBUG } 03:09:53 DEBUG } 03:09:53 DEBUG } 03:09:53 DEBUG ] 03:09:53 DEBUG ] 03:09:53 DEBUG ] 03:09:53 DEBUG ] 03:09:53 DEBUG ] 03:09:53 DEBUG } 03:09:53 DEBUG ] 03:09:53 DEBUG "value": [ 03:09:53 DEBUG ] 03:09:53 DEBUG } 03:09:53 DEBUG } 03:09:53 DEBUG } 03:09:53 DEBUG ] 03:09:53 DEBUG ] 03:09:53 DEBUG } 03:09:53 DEBUG } 03:09:53 DEBUG ] 03:09:53 DEBUG ] 03:09:53 DEBUG } 03:09:53 DEBUG } 03:09:53 DEBUG } 03:09:53 DEBUG } 03:09:53 DEBUG } 03:09:53 DEBUG } 03:09:53 DEBUG 1713665393, 03:09:53 DEBUG } 03:09:53 DEBUG ] 03:09:53 DEBUG } 03:09:53 DEBUG } 03:09:53 DEBUG } 03:09:53 DEBUG } 03:09:53 DEBUG } 03:09:53 DEBUG } 03:09:53 DEBUG ] 03:09:53 DEBUG } 03:09:53 DEBUG } 03:09:53 DEBUG } 03:09:53 DEBUG } 03:09:53 DEBUG } 03:09:53 DEBUG "11.734965319840628" 03:09:53 DEBUG } 03:09:53 DEBUG } 03:09:53 DEBUG ] 03:09:53 DEBUG } 03:09:53 DEBUG } 03:09:53 DEBUG } 03:09:53 DEBUG } 03:09:53 DEBUG ] 03:09:53 DEBUG } 03:09:53 DEBUG } 03:09:53 DEBUG } 03:09:53 DEBUG } 03:09:53 DEBUG } 03:09:53 DEBUG ] 03:09:53 DEBUG } 03:09:53 DEBUG } 03:10:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713665452" 03:10:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665452" 03:10:52 INFO [http_cmd]: http status code OK 03:10:52 DEBUG --- status code --- 03:10:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:10:52 DEBUG --- http response --- 03:10:52 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 03:10:52 ERROR [http_cmd]: ERROR 03:10:52 ERROR --- status code --- 03:10:52 ERROR http status code is 400 (expected [200, 401, 403, 404]) 03:10:52 ERROR --- http response --- 03:10:52 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 03:10:52 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 03:10:53 INFO 03:10:53 INFO [loop_until]: kubectl --namespace=xlou top pods 03:10:53 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:10:53 INFO [loop_until]: OK (rc = 0) 03:10:53 DEBUG --- stdout --- 03:10:53 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 22022m 8794Mi am-767f6b67b5-qcqz2 21990m 9122Mi am-767f6b67b5-vbltz 23970m 8814Mi ds-cts-0 19796m 12324Mi ds-cts-1 19514m 12291Mi ds-cts-2 20148m 12370Mi ds-idrepo-0 9716m 3930Mi ds-idrepo-1 9463m 3946Mi ds-idrepo-2 9656m 3915Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1179Mi idm-b85f9d978-wnhxb 4m 1239Mi lodemon-54c87f8cbb-jzrjs 15m 62Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-68c5d5b884-p97w5 11522m 867Mi 03:10:53 DEBUG --- stderr --- 03:10:53 DEBUG 03:10:53 INFO 03:10:53 INFO [loop_until]: kubectl --namespace=xlou top node 03:10:53 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:10:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665453" 03:10:53 INFO [http_cmd]: http status code OK 03:10:53 DEBUG --- status code --- 03:10:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:10:53 DEBUG --- http response --- 03:10:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:10:53 DEBUG ----- output ----- 03:10:53 DEBUG { 03:10:53 DEBUG "status": "success", 03:10:53 DEBUG "data": { 03:10:53 DEBUG "resultType": "vector", 03:10:53 DEBUG "result": [ 03:10:53 DEBUG { 03:10:53 DEBUG "metric": { 03:10:53 DEBUG "instance": "10.142.0.107:9100" 03:10:53 DEBUG }, 03:10:53 DEBUG "value": [ 03:10:53 DEBUG 03:10:53 DEBUG 03:10:53 DEBUG 03:10:53 DEBUG [...] 03:10:53 DEBUG 03:10:53 DEBUG 03:10:53 DEBUG 03:10:53 DEBUG "instance": "10.142.0.43:9100" 03:10:53 DEBUG }, 03:10:53 DEBUG "value": [ 03:10:53 DEBUG 1713665453, 03:10:53 DEBUG "0.42123333333342616" 03:10:53 DEBUG ] 03:10:53 DEBUG } 03:10:53 DEBUG ] 03:10:53 DEBUG } 03:10:53 DEBUG } 03:10:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665453" 03:10:53 INFO [http_cmd]: http status code OK 03:10:53 DEBUG --- status code --- 03:10:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:10:53 DEBUG --- http response --- 03:10:53 INFO [loop_until]: OK (rc = 0) 03:10:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:10:53 DEBUG --- stdout --- 03:10:53 DEBUG ----- output ----- 03:10:53 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 66m 0% 6295Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 9960m 33% 7783Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 22081m 73% 11605Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 85m 0% 3360Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 19193m 64% 15640Mi 14% gke-xlou-cdm-default-pool-1b0769c8-wpwl 157m 0% 5283Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 22532m 75% 12054Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 19861m 66% 3199Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 24795m 83% 11107Mi 9% gke-xlou-cdm-ds-0f601751-j488 10359m 34% 7371Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 19668m 65% 17423Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 10186m 34% 8023Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20321m 68% 15087Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 11684m 73% 11223Mi 19% 03:10:53 DEBUG { 03:10:53 DEBUG --- stderr --- 03:10:53 DEBUG "status": "success", 03:10:53 DEBUG 03:10:53 DEBUG "data": { 03:10:53 DEBUG "resultType": "vector", 03:10:53 DEBUG "result": [ 03:10:53 DEBUG { 03:10:53 DEBUG "metric": { 03:10:53 DEBUG "pod": "ds-cts-0" 03:10:53 DEBUG }, 03:10:53 DEBUG "value": [ 03:10:53 DEBUG 03:10:53 DEBUG 03:10:53 DEBUG 03:10:53 DEBUG [...] 03:10:53 DEBUG 03:10:53 DEBUG 03:10:53 DEBUG 03:10:53 DEBUG "pod": "ds-cts-2" 03:10:53 DEBUG }, 03:10:53 DEBUG "value": [ 03:10:53 DEBUG 1713665453, 03:10:53 DEBUG "0.5533333333333337" 03:10:53 DEBUG ] 03:10:53 DEBUG } 03:10:53 DEBUG ] 03:10:53 DEBUG } 03:10:53 DEBUG } 03:10:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665453" 03:10:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665453" 03:10:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665453" 03:10:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665453" 03:10:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665453" 03:10:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665453" 03:10:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665453" 03:10:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665453" 03:10:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665453" 03:10:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713665453" 03:10:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665453" 03:10:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665453" 03:10:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665453" 03:10:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665453" 03:10:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713665453" 03:10:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713665453" 03:10:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665453" 03:10:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665453" 03:10:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665453" 03:10:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713665453" 03:10:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665453" 03:10:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665453" 03:10:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665453" 03:10:53 INFO [http_cmd]: http status code OK 03:10:53 INFO [http_cmd]: http status code OK 03:10:53 INFO [http_cmd]: http status code OK 03:10:53 INFO [http_cmd]: http status code OK 03:10:53 INFO [http_cmd]: http status code OK 03:10:53 INFO [http_cmd]: http status code OK 03:10:53 INFO [http_cmd]: http status code OK 03:10:53 INFO [http_cmd]: http status code OK 03:10:53 INFO [http_cmd]: http status code OK 03:10:53 INFO [http_cmd]: http status code OK 03:10:53 INFO [http_cmd]: http status code OK 03:10:53 INFO [http_cmd]: http status code OK 03:10:53 INFO [http_cmd]: http status code OK 03:10:53 INFO [http_cmd]: http status code OK 03:10:53 INFO [http_cmd]: http status code OK 03:10:53 DEBUG --- status code --- 03:10:53 INFO [http_cmd]: http status code OK 03:10:53 DEBUG --- status code --- 03:10:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:10:53 DEBUG --- status code --- 03:10:53 DEBUG --- status code --- 03:10:53 DEBUG --- status code --- 03:10:53 INFO [http_cmd]: http status code OK 03:10:53 DEBUG --- status code --- 03:10:53 DEBUG --- status code --- 03:10:53 INFO [http_cmd]: http status code OK 03:10:53 DEBUG --- status code --- 03:10:53 DEBUG --- status code --- 03:10:53 DEBUG --- status code --- 03:10:53 DEBUG --- status code --- 03:10:53 DEBUG --- status code --- 03:10:53 DEBUG --- status code --- 03:10:53 DEBUG --- status code --- 03:10:53 INFO [http_cmd]: http status code OK 03:10:53 DEBUG --- status code --- 03:10:53 DEBUG --- status code --- 03:10:53 DEBUG --- status code --- 03:10:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:10:53 INFO [http_cmd]: http status code OK 03:10:53 INFO [http_cmd]: http status code OK 03:10:53 DEBUG --- http response --- 03:10:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:10:53 INFO [http_cmd]: http status code OK 03:10:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:10:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:10:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:10:53 INFO [http_cmd]: http status code OK 03:10:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:10:53 DEBUG --- http response --- 03:10:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:10:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:10:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:10:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:10:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:10:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:10:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:10:53 DEBUG --- http response --- 03:10:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:10:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:10:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:10:53 DEBUG --- http response --- 03:10:53 DEBUG --- status code --- 03:10:53 DEBUG --- status code --- 03:10:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:10:53 DEBUG --- http response --- 03:10:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:10:53 DEBUG --- http response --- 03:10:53 DEBUG --- http response --- 03:10:53 DEBUG --- http response --- 03:10:53 DEBUG --- status code --- 03:10:53 DEBUG --- status code --- 03:10:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:10:53 DEBUG --- http response --- 03:10:53 DEBUG --- http response --- 03:10:53 DEBUG --- http response --- 03:10:53 DEBUG --- http response --- 03:10:53 DEBUG --- http response --- 03:10:53 DEBUG --- http response --- 03:10:53 DEBUG --- status code --- 03:10:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:10:53 DEBUG --- http response --- 03:10:53 DEBUG --- http response --- 03:10:53 DEBUG --- http response --- 03:10:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:10:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:10:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:10:53 DEBUG ----- output ----- 03:10:53 DEBUG ----- output ----- 03:10:53 DEBUG --- status code --- 03:10:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 03:10:53 DEBUG ----- output ----- 03:10:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:10:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:10:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:10:53 DEBUG ----- output ----- 03:10:53 DEBUG { 03:10:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:10:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:10:53 DEBUG ----- output ----- 03:10:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:10:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:10:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:10:53 DEBUG ----- output ----- 03:10:53 DEBUG { 03:10:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:10:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:10:53 DEBUG --- http response --- 03:10:53 DEBUG ----- output ----- 03:10:53 DEBUG --- http response --- 03:10:53 DEBUG { 03:10:53 DEBUG { 03:10:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:10:53 DEBUG { 03:10:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:10:53 DEBUG ----- output ----- 03:10:53 DEBUG --- http response --- 03:10:53 DEBUG --- http response --- 03:10:53 DEBUG "status": "success", 03:10:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:10:53 DEBUG ----- output ----- 03:10:53 DEBUG { 03:10:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:10:53 DEBUG ----- output ----- 03:10:53 DEBUG ----- output ----- 03:10:53 DEBUG --- http response --- 03:10:53 DEBUG "status": "success", 03:10:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:10:53 DEBUG ----- output ----- 03:10:53 DEBUG ----- output ----- 03:10:53 DEBUG { 03:10:53 DEBUG { 03:10:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:10:53 DEBUG "status": "success", 03:10:53 DEBUG "status": "success", 03:10:53 DEBUG --- http response --- 03:10:53 DEBUG "status": "success", 03:10:53 DEBUG ----- output ----- 03:10:53 DEBUG { 03:10:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:10:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:10:53 DEBUG "data": { 03:10:53 DEBUG ----- output ----- 03:10:53 DEBUG { 03:10:53 DEBUG "status": "success", 03:10:53 DEBUG ----- output ----- 03:10:53 DEBUG { 03:10:53 DEBUG { 03:10:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 03:10:53 DEBUG "data": { 03:10:53 DEBUG ----- output ----- 03:10:53 DEBUG { 03:10:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:10:53 DEBUG "status": "success", 03:10:53 DEBUG "status": "success", 03:10:53 DEBUG ----- output ----- 03:10:53 DEBUG "data": { 03:10:53 DEBUG "data": { 03:10:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:10:53 DEBUG ----- output ----- 03:10:53 DEBUG { 03:10:53 DEBUG "status": "success", 03:10:53 DEBUG "data": { 03:10:53 DEBUG "resultType": "vector", 03:10:53 DEBUG "resultType": "vector", 03:10:53 DEBUG "status": "success", 03:10:53 DEBUG "status": "success", 03:10:53 DEBUG "data": { 03:10:53 DEBUG { 03:10:53 DEBUG "status": "success", 03:10:53 DEBUG "status": "success", 03:10:53 DEBUG ----- output ----- 03:10:53 DEBUG "resultType": "vector", 03:10:53 DEBUG { 03:10:53 DEBUG "status": "success", 03:10:53 DEBUG ----- output ----- 03:10:53 DEBUG "data": { 03:10:53 DEBUG "data": { 03:10:53 DEBUG { 03:10:53 DEBUG "resultType": "vector", 03:10:53 DEBUG "resultType": "vector", 03:10:53 DEBUG { 03:10:53 DEBUG "data": { 03:10:53 DEBUG "resultType": "vector", 03:10:53 DEBUG "result": [ 03:10:53 DEBUG ----- output ----- 03:10:53 DEBUG "result": [ 03:10:53 DEBUG { 03:10:53 DEBUG "data": { 03:10:53 DEBUG "data": { 03:10:53 DEBUG "resultType": "vector", 03:10:53 DEBUG "status": "success", 03:10:53 DEBUG "data": { 03:10:53 DEBUG "data": { 03:10:53 DEBUG { 03:10:53 DEBUG "result": [ 03:10:53 DEBUG "status": "success", 03:10:53 DEBUG "data": { 03:10:53 DEBUG { 03:10:53 DEBUG "resultType": "vector", 03:10:53 DEBUG "resultType": "vector", 03:10:53 DEBUG "status": "success", 03:10:53 DEBUG "result": [ 03:10:53 DEBUG "result": [ 03:10:53 DEBUG "status": "success", 03:10:53 DEBUG "result": [ 03:10:53 DEBUG ----- output ----- 03:10:53 DEBUG { 03:10:53 DEBUG { 03:10:53 DEBUG { 03:10:53 DEBUG "status": "success", 03:10:53 DEBUG "resultType": "vector", 03:10:53 DEBUG "resultType": "vector", 03:10:53 DEBUG "result": [ 03:10:53 DEBUG "data": { 03:10:53 DEBUG "resultType": "vector", 03:10:53 DEBUG "resultType": "vector", 03:10:53 DEBUG "status": "success", 03:10:53 DEBUG { 03:10:53 DEBUG "data": { 03:10:53 DEBUG "resultType": "vector", 03:10:53 DEBUG "status": "success", 03:10:53 DEBUG "result": [ 03:10:53 DEBUG "result": [ 03:10:53 DEBUG "data": { 03:10:53 DEBUG { 03:10:53 DEBUG { 03:10:53 DEBUG "data": { 03:10:53 DEBUG { 03:10:53 DEBUG { 03:10:53 DEBUG "metric": { 03:10:53 DEBUG "status": "success", 03:10:53 DEBUG "data": { 03:10:53 DEBUG "resultType": "vector", 03:10:53 DEBUG "result": [ 03:10:53 DEBUG "result": [ 03:10:53 DEBUG { 03:10:53 DEBUG "resultType": "vector", 03:10:53 DEBUG "result": [ 03:10:53 DEBUG "result": [ 03:10:53 DEBUG "data": { 03:10:53 DEBUG "metric": { 03:10:53 DEBUG "resultType": "vector", 03:10:53 DEBUG "result": [ 03:10:53 DEBUG "data": { 03:10:53 DEBUG { 03:10:53 DEBUG { 03:10:53 DEBUG "resultType": "vector", 03:10:53 DEBUG "metric": { 03:10:53 DEBUG "metric": { 03:10:53 DEBUG "resultType": "vector", 03:10:53 DEBUG "metric": { 03:10:53 DEBUG "status": "success", 03:10:53 DEBUG "pod": "am-767f6b67b5-9zw22" 03:10:53 DEBUG "result": [ 03:10:53 DEBUG "data": { 03:10:53 DEBUG "metric": { 03:10:53 DEBUG { 03:10:53 DEBUG { 03:10:53 DEBUG "metric": { 03:10:53 DEBUG { 03:10:53 DEBUG "result": [ 03:10:53 DEBUG { 03:10:53 DEBUG "metric": { 03:10:53 DEBUG "pod": "am-767f6b67b5-9zw22" 03:10:53 DEBUG }, 03:10:53 DEBUG { 03:10:53 DEBUG "resultType": "vector", 03:10:53 DEBUG "metric": { 03:10:53 DEBUG "metric": { 03:10:53 DEBUG "result": [ 03:10:53 DEBUG "pod": "ds-cts-0" 03:10:53 DEBUG "pod": "ds-idrepo-0" 03:10:53 DEBUG "result": [ 03:10:53 DEBUG { 03:10:53 DEBUG "data": { 03:10:53 DEBUG }, 03:10:53 DEBUG { 03:10:53 DEBUG "resultType": "vector", 03:10:53 DEBUG "instance": "10.142.0.107:9100" 03:10:53 DEBUG "metric": { 03:10:53 DEBUG "metric": { 03:10:53 DEBUG "pod": "am-767f6b67b5-9zw22" 03:10:53 DEBUG "metric": { 03:10:53 DEBUG { 03:10:53 DEBUG "value": [ 03:10:53 DEBUG "result": [ 03:10:53 DEBUG "resultType": "vector", 03:10:53 DEBUG "pod": "am-767f6b67b5-9zw22" 03:10:53 DEBUG "metric": { 03:10:53 DEBUG "result": [ 03:10:53 DEBUG "pod": "ds-cts-0" 03:10:53 DEBUG "pod": "ds-idrepo-0" 03:10:53 DEBUG { 03:10:53 DEBUG }, 03:10:53 DEBUG }, 03:10:53 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:10:53 DEBUG "metric": { 03:10:53 DEBUG "resultType": "vector", 03:10:53 DEBUG "value": [ 03:10:53 DEBUG "metric": { 03:10:53 DEBUG "result": [ 03:10:54 DEBUG { 03:10:53 DEBUG "instance": "10.142.0.107:9100" 03:10:53 DEBUG "pod": "ds-idrepo-0" 03:10:53 DEBUG }, 03:10:53 DEBUG "instance": "10.142.0.107:9100" 03:10:53 DEBUG "metric": { 03:10:53 DEBUG 03:10:53 DEBUG { 03:10:53 DEBUG "result": [ 03:10:53 DEBUG }, 03:10:53 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:10:53 DEBUG { 03:10:54 DEBUG }, 03:10:54 DEBUG }, 03:10:54 DEBUG "metric": { 03:10:54 DEBUG "value": [ 03:10:54 DEBUG "value": [ 03:10:54 DEBUG }, 03:10:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:10:54 DEBUG "result": [ 03:10:54 DEBUG 03:10:54 DEBUG "pod": "am-767f6b67b5-9zw22" 03:10:54 DEBUG "metric": { 03:10:53 DEBUG }, 03:10:54 DEBUG }, 03:10:54 DEBUG }, 03:10:54 DEBUG "value": [ 03:10:54 DEBUG }, 03:10:54 DEBUG "pod": "ds-idrepo-0" 03:10:54 DEBUG }, 03:10:54 DEBUG "metric": { 03:10:54 DEBUG { 03:10:54 DEBUG "value": [ 03:10:54 DEBUG }, 03:10:54 DEBUG "metric": { 03:10:54 DEBUG "value": [ 03:10:54 DEBUG "value": [ 03:10:54 DEBUG "pod": "am-767f6b67b5-9zw22" 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG "value": [ 03:10:54 DEBUG }, 03:10:54 DEBUG { 03:10:54 DEBUG 03:10:54 DEBUG }, 03:10:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:10:54 DEBUG "value": [ 03:10:54 DEBUG "value": [ 03:10:54 DEBUG "value": [ 03:10:54 DEBUG 03:10:54 DEBUG "value": [ 03:10:54 DEBUG "value": [ 03:10:54 DEBUG 03:10:54 DEBUG "instance": "10.142.0.107:9100" 03:10:54 DEBUG "metric": { 03:10:54 DEBUG 03:10:54 DEBUG "value": [ 03:10:54 DEBUG "pod": "am-767f6b67b5-9zw22" 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG }, 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG "value": [ 03:10:54 DEBUG "metric": { 03:10:54 DEBUG 03:10:54 DEBUG "value": [ 03:10:54 DEBUG }, 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG }, 03:10:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG }, 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG "value": [ 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:10:54 DEBUG [...] 03:10:54 DEBUG 03:10:54 DEBUG "value": [ 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG [...] 03:10:54 DEBUG 03:10:54 DEBUG "value": [ 03:10:54 DEBUG }, 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG "value": [ 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG [...] 03:10:54 DEBUG [...] 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG }, 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG [...] 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG "value": [ 03:10:54 DEBUG [...] 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG [...] 03:10:54 DEBUG [...] 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG [...] 03:10:54 DEBUG 03:10:54 DEBUG "value": [ 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG [...] 03:10:54 DEBUG [...] 03:10:54 DEBUG [...] 03:10:54 DEBUG 03:10:54 DEBUG [...] 03:10:54 DEBUG "pod": "ds-cts-2" 03:10:54 DEBUG }, 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG [...] 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG [...] 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG [...] 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG "value": [ 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG [...] 03:10:54 DEBUG 03:10:54 DEBUG "pod": "ds-idrepo-2" 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG "pod": "am-767f6b67b5-vbltz" 03:10:54 DEBUG }, 03:10:54 DEBUG [...] 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG 1713665453, 03:10:54 DEBUG 03:10:54 DEBUG [...] 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG "pod": "am-767f6b67b5-vbltz" 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG "pod": "ds-cts-2" 03:10:54 DEBUG }, 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG "value": [ 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG "instance": "10.142.0.43:9100" 03:10:54 DEBUG 03:10:54 DEBUG "pod": "ds-idrepo-2" 03:10:54 DEBUG 03:10:54 DEBUG "0" 03:10:54 DEBUG [...] 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG [...] 03:10:54 DEBUG }, 03:10:54 DEBUG [...] 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG }, 03:10:54 DEBUG "value": [ 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG [...] 03:10:54 DEBUG 1713665453, 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG }, 03:10:54 DEBUG "instance": "10.142.0.43:9100" 03:10:54 DEBUG 03:10:54 DEBUG }, 03:10:54 DEBUG "instance": "10.142.0.43:9100" 03:10:54 DEBUG ] 03:10:54 DEBUG 03:10:54 DEBUG "instance": "10.142.0.43:9100" 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG "pod": "ds-idrepo-2" 03:10:54 DEBUG "pod": "ds-cts-2" 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG "value": [ 03:10:54 DEBUG 1713665453, 03:10:54 DEBUG "pod": "ds-cts-0" 03:10:54 DEBUG "pod": "am-767f6b67b5-vbltz" 03:10:54 DEBUG 03:10:54 DEBUG "3331.4666666666667" 03:10:54 DEBUG "pod": "am-767f6b67b5-vbltz" 03:10:54 DEBUG 03:10:54 DEBUG "value": [ 03:10:54 DEBUG }, 03:10:54 DEBUG "pod": "am-767f6b67b5-vbltz" 03:10:54 DEBUG "value": [ 03:10:54 DEBUG }, 03:10:54 DEBUG } 03:10:54 DEBUG 03:10:54 DEBUG }, 03:10:54 DEBUG "value": [ 03:10:54 DEBUG 03:10:54 DEBUG 03:10:54 DEBUG }, 03:10:54 DEBUG }, 03:10:54 DEBUG "pod": "am-767f6b67b5-vbltz" 03:10:54 DEBUG "pod": "am-767f6b67b5-vbltz" 03:10:54 DEBUG 1713665453, 03:10:54 DEBUG "0" 03:10:54 DEBUG }, 03:10:54 DEBUG }, 03:10:54 DEBUG 03:10:54 DEBUG ] 03:10:54 DEBUG }, 03:10:54 DEBUG 03:10:54 DEBUG 1713665453, 03:10:54 DEBUG "value": [ 03:10:54 DEBUG }, 03:10:54 DEBUG 1713665453, 03:10:54 DEBUG "value": [ 03:10:54 DEBUG ] 03:10:54 DEBUG 03:10:54 DEBUG "value": [ 03:10:54 DEBUG 1713665453, 03:10:54 DEBUG "pod": "ds-cts-0" 03:10:54 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:10:54 DEBUG "value": [ 03:10:54 DEBUG "value": [ 03:10:54 DEBUG }, 03:10:54 DEBUG }, 03:10:54 DEBUG "0" 03:10:54 DEBUG ] 03:10:54 DEBUG "value": [ 03:10:54 DEBUG "value": [ 03:10:54 DEBUG 03:10:54 DEBUG } 03:10:54 DEBUG "value": [ 03:10:54 DEBUG "pod": "am-767f6b67b5-vbltz" 03:10:54 DEBUG "100603595.93333334" 03:10:54 DEBUG 1713665453, 03:10:54 DEBUG "value": [ 03:10:54 DEBUG "0" 03:10:54 DEBUG 1713665453, 03:10:54 DEBUG } 03:10:54 DEBUG "pod": "am-767f6b67b5-vbltz" 03:10:54 DEBUG 1713665453, 03:10:54 DEBUG "3335.2" 03:10:54 DEBUG }, 03:10:54 DEBUG }, 03:10:54 DEBUG 1713665453, 03:10:54 DEBUG 1713665453, 03:10:54 DEBUG "value": [ 03:10:54 DEBUG "value": [ 03:10:54 DEBUG ] 03:10:54 DEBUG } 03:10:54 DEBUG 1713665453, 03:10:54 DEBUG 1713665453, 03:10:54 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:10:54 DEBUG ] 03:10:54 DEBUG 1713665453, 03:10:54 DEBUG }, 03:10:54 DEBUG "value": [ 03:10:54 DEBUG "0.08933333333333164" 03:10:54 DEBUG 1713665453, 03:10:54 DEBUG ] 03:10:54 DEBUG "4350645756928" 03:10:54 DEBUG ] 03:10:54 DEBUG }, 03:10:54 DEBUG "100606663.06666666" 03:10:54 DEBUG ] 03:10:54 DEBUG "value": [ 03:10:54 DEBUG "value": [ 03:10:54 DEBUG "0" 03:10:54 DEBUG "0" 03:10:54 DEBUG 1713665453, 03:10:54 DEBUG 1713665453, 03:10:54 DEBUG } 03:10:54 DEBUG ] 03:10:54 DEBUG "0" 03:10:54 DEBUG "0" 03:10:54 DEBUG }, 03:10:54 DEBUG } 03:10:54 DEBUG "887.4740111170881" 03:10:54 DEBUG 1713665453, 03:10:54 DEBUG ] 03:10:54 DEBUG } 03:10:54 DEBUG "3334.2666666666664" 03:10:54 DEBUG } 03:10:54 DEBUG } 03:10:54 DEBUG "value": [ 03:10:54 DEBUG ] 03:10:54 DEBUG } 03:10:54 DEBUG } 03:10:54 DEBUG 1713665453, 03:10:54 DEBUG 1713665453, 03:10:54 DEBUG ] 03:10:54 DEBUG ] 03:10:54 DEBUG "0" 03:10:54 DEBUG "149.889" 03:10:54 DEBUG ] 03:10:54 DEBUG } 03:10:54 DEBUG ] 03:10:54 DEBUG ] 03:10:54 DEBUG "value": [ 03:10:54 DEBUG } 03:10:54 DEBUG ] 03:10:54 DEBUG "600674" 03:10:54 DEBUG ] 03:10:54 DEBUG ] 03:10:54 DEBUG ] 03:10:54 DEBUG ] 03:10:54 DEBUG ] 03:10:54 DEBUG 1713665453, 03:10:54 DEBUG } 03:10:54 DEBUG ] 03:10:54 DEBUG "0" 03:10:54 DEBUG "11.806110512642716" 03:10:54 DEBUG } 03:10:54 DEBUG } 03:10:54 DEBUG ] 03:10:54 DEBUG ] 03:10:54 DEBUG } 03:10:54 DEBUG } 03:10:54 DEBUG } 03:10:54 DEBUG } 03:10:54 DEBUG 1713665453, 03:10:54 DEBUG } 03:10:54 DEBUG ] 03:10:54 DEBUG } 03:10:54 DEBUG } 03:10:54 DEBUG } 03:10:54 DEBUG } 03:10:54 DEBUG } 03:10:54 DEBUG "3335.8333333333335" 03:10:54 DEBUG ] 03:10:54 DEBUG } 03:10:54 DEBUG ] 03:10:54 DEBUG ] 03:10:54 DEBUG ] 03:10:54 DEBUG ] 03:10:54 DEBUG } 03:10:54 DEBUG } 03:10:54 DEBUG } 03:10:54 DEBUG ] 03:10:54 DEBUG ] 03:10:54 DEBUG "1008218112" 03:10:54 DEBUG ] 03:10:54 DEBUG } 03:10:54 DEBUG ] 03:10:54 DEBUG } 03:10:54 DEBUG } 03:10:54 DEBUG ] 03:10:54 DEBUG } 03:10:54 DEBUG ] 03:10:54 DEBUG } 03:10:54 DEBUG } 03:10:54 DEBUG } 03:10:54 DEBUG } 03:10:54 DEBUG } 03:10:54 DEBUG } 03:10:54 DEBUG ] 03:10:54 DEBUG ] 03:10:54 DEBUG } 03:10:54 DEBUG } 03:10:54 DEBUG ] 03:10:54 DEBUG } 03:10:54 DEBUG ] 03:10:54 DEBUG } 03:10:54 DEBUG } 03:10:54 DEBUG } 03:10:54 DEBUG } 03:10:54 DEBUG ] 03:10:54 DEBUG } 03:10:54 DEBUG } 03:10:54 DEBUG } 03:10:54 DEBUG } 03:10:54 DEBUG } 03:10:54 DEBUG } 03:10:54 DEBUG } 03:10:54 DEBUG } 03:10:54 DEBUG } 03:10:54 DEBUG } 03:10:54 DEBUG } 03:10:54 DEBUG } 03:10:54 DEBUG ] 03:10:54 DEBUG } 03:10:54 DEBUG ] 03:10:54 DEBUG } 03:10:54 DEBUG } 03:10:54 DEBUG ] 03:10:54 DEBUG } 03:10:54 DEBUG } 03:10:54 DEBUG } 03:10:54 DEBUG } 03:10:54 DEBUG } 03:10:54 DEBUG } 03:10:54 DEBUG } 03:11:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665512" 03:11:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713665512" 03:11:52 ERROR [http_cmd]: ERROR 03:11:52 INFO [http_cmd]: http status code OK 03:11:52 ERROR --- status code --- 03:11:52 DEBUG --- status code --- 03:11:52 ERROR http status code is 400 (expected [200, 401, 403, 404]) 03:11:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:11:52 ERROR --- http response --- 03:11:52 DEBUG --- http response --- 03:11:52 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 03:11:52 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 03:11:52 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 03:11:53 INFO 03:11:53 INFO [loop_until]: kubectl --namespace=xlou top pods 03:11:53 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:11:53 INFO [loop_until]: OK (rc = 0) 03:11:53 DEBUG --- stdout --- 03:11:53 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 21508m 8794Mi am-767f6b67b5-qcqz2 22669m 9126Mi am-767f6b67b5-vbltz 23763m 8820Mi ds-cts-0 19865m 12325Mi ds-cts-1 19914m 12290Mi ds-cts-2 19585m 12373Mi ds-idrepo-0 9671m 3918Mi ds-idrepo-1 9312m 3946Mi ds-idrepo-2 9644m 3914Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 3m 1179Mi idm-b85f9d978-wnhxb 3m 1239Mi lodemon-54c87f8cbb-jzrjs 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-68c5d5b884-p97w5 11656m 872Mi 03:11:53 DEBUG --- stderr --- 03:11:53 DEBUG 03:11:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665513" 03:11:53 INFO [http_cmd]: http status code OK 03:11:53 DEBUG --- status code --- 03:11:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:11:53 DEBUG --- http response --- 03:11:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:11:53 DEBUG ----- output ----- 03:11:53 DEBUG { 03:11:53 DEBUG "status": "success", 03:11:53 DEBUG "data": { 03:11:53 DEBUG "resultType": "vector", 03:11:53 DEBUG "result": [ 03:11:53 DEBUG { 03:11:53 DEBUG "metric": { 03:11:53 DEBUG "instance": "10.142.0.107:9100" 03:11:53 DEBUG }, 03:11:53 DEBUG "value": [ 03:11:53 DEBUG 03:11:53 DEBUG 03:11:53 DEBUG 03:11:53 DEBUG [...] 03:11:53 DEBUG 03:11:53 DEBUG 03:11:53 DEBUG 03:11:53 DEBUG "instance": "10.142.0.43:9100" 03:11:53 DEBUG }, 03:11:53 DEBUG "value": [ 03:11:53 DEBUG 1713665513, 03:11:53 DEBUG "0.4254333333333136" 03:11:53 DEBUG ] 03:11:53 DEBUG } 03:11:53 DEBUG ] 03:11:53 DEBUG } 03:11:53 DEBUG } 03:11:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665513" 03:11:53 INFO 03:11:53 INFO [loop_until]: kubectl --namespace=xlou top node 03:11:53 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:11:53 INFO [http_cmd]: http status code OK 03:11:53 DEBUG --- status code --- 03:11:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:11:53 DEBUG --- http response --- 03:11:53 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:11:53 DEBUG ----- output ----- 03:11:53 DEBUG { 03:11:53 DEBUG "status": "success", 03:11:53 DEBUG "data": { 03:11:53 DEBUG "resultType": "vector", 03:11:53 DEBUG "result": [ 03:11:53 DEBUG { 03:11:53 DEBUG "metric": { 03:11:53 DEBUG "pod": "ds-cts-0" 03:11:53 DEBUG }, 03:11:53 DEBUG "value": [ 03:11:53 DEBUG 03:11:53 DEBUG 03:11:53 DEBUG 03:11:53 DEBUG [...] 03:11:53 DEBUG 03:11:53 DEBUG 03:11:53 DEBUG 03:11:53 DEBUG "pod": "ds-cts-2" 03:11:53 DEBUG }, 03:11:53 DEBUG "value": [ 03:11:53 DEBUG 1713665513, 03:11:53 DEBUG "0.5604666666666664" 03:11:53 DEBUG ] 03:11:53 DEBUG } 03:11:53 DEBUG ] 03:11:53 DEBUG } 03:11:53 DEBUG } 03:11:54 INFO [loop_until]: OK (rc = 0) 03:11:54 DEBUG --- stdout --- 03:11:54 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 6286Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 10005m 33% 7771Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 21939m 73% 11595Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 83m 0% 3366Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 19925m 66% 15640Mi 14% gke-xlou-cdm-default-pool-1b0769c8-wpwl 160m 0% 5258Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 21641m 72% 12046Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 19717m 66% 3189Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 24673m 82% 11093Mi 9% gke-xlou-cdm-ds-0f601751-j488 10320m 34% 7357Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 19681m 65% 17420Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 10126m 33% 8012Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20255m 67% 15092Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 11999m 75% 11225Mi 19% 03:11:54 DEBUG --- stderr --- 03:11:54 DEBUG 03:11:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713665514" 03:11:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665514" 03:11:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665514" 03:11:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665514" 03:11:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665514" 03:11:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665514" 03:11:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665514" 03:11:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665514" 03:11:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665514" 03:11:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665514" 03:11:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665514" 03:11:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665514" 03:11:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665514" 03:11:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665514" 03:11:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665514" 03:11:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665514" 03:11:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665514" 03:11:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713665514" 03:11:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665514" 03:11:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665514" 03:11:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665514" 03:11:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713665514" 03:11:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713665514" 03:11:54 INFO [http_cmd]: http status code OK 03:11:54 INFO [http_cmd]: http status code OK 03:11:54 INFO [http_cmd]: http status code OK 03:11:54 INFO [http_cmd]: http status code OK 03:11:54 INFO [http_cmd]: http status code OK 03:11:54 INFO [http_cmd]: http status code OK 03:11:54 INFO [http_cmd]: http status code OK 03:11:54 DEBUG --- status code --- 03:11:54 INFO [http_cmd]: http status code OK 03:11:54 INFO [http_cmd]: http status code OK 03:11:54 INFO [http_cmd]: http status code OK 03:11:54 INFO [http_cmd]: http status code OK 03:11:54 INFO [http_cmd]: http status code OK 03:11:54 INFO [http_cmd]: http status code OK 03:11:54 INFO [http_cmd]: http status code OK 03:11:54 INFO [http_cmd]: http status code OK 03:11:54 INFO [http_cmd]: http status code OK 03:11:54 DEBUG --- status code --- 03:11:54 INFO [http_cmd]: http status code OK 03:11:54 DEBUG --- status code --- 03:11:54 INFO [http_cmd]: http status code OK 03:11:54 DEBUG --- status code --- 03:11:54 DEBUG --- status code --- 03:11:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:11:54 DEBUG --- status code --- 03:11:54 DEBUG --- status code --- 03:11:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:11:54 INFO [http_cmd]: http status code OK 03:11:54 DEBUG --- status code --- 03:11:54 DEBUG --- status code --- 03:11:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:11:54 DEBUG --- status code --- 03:11:54 INFO [http_cmd]: http status code OK 03:11:54 DEBUG --- status code --- 03:11:54 DEBUG --- status code --- 03:11:54 DEBUG --- status code --- 03:11:54 DEBUG --- status code --- 03:11:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:11:54 DEBUG --- status code --- 03:11:54 DEBUG --- status code --- 03:11:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:11:54 DEBUG --- status code --- 03:11:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:11:54 INFO [http_cmd]: http status code OK 03:11:54 DEBUG --- status code --- 03:11:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:11:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:11:54 DEBUG --- http response --- 03:11:54 INFO [http_cmd]: http status code OK 03:11:54 DEBUG --- status code --- 03:11:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:11:54 INFO [http_cmd]: http status code OK 03:11:54 DEBUG --- http response --- 03:11:54 DEBUG --- status code --- 03:11:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:11:54 DEBUG --- status code --- 03:11:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:11:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:11:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:11:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:11:54 DEBUG --- http response --- 03:11:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:11:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:11:54 DEBUG --- http response --- 03:11:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:11:54 DEBUG --- http response --- 03:11:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:11:54 DEBUG --- http response --- 03:11:54 DEBUG --- http response --- 03:11:54 DEBUG --- http response --- 03:11:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:11:54 DEBUG --- status code --- 03:11:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:11:54 DEBUG --- http response --- 03:11:54 DEBUG --- status code --- 03:11:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:11:54 DEBUG ----- output ----- 03:11:54 DEBUG --- http response --- 03:11:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:11:54 DEBUG --- http response --- 03:11:54 DEBUG --- http response --- 03:11:54 DEBUG --- http response --- 03:11:54 DEBUG --- http response --- 03:11:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:11:54 DEBUG --- http response --- 03:11:54 DEBUG --- http response --- 03:11:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 03:11:54 DEBUG ----- output ----- 03:11:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:11:54 DEBUG --- http response --- 03:11:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:11:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:11:54 DEBUG ----- output ----- 03:11:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:11:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:11:54 DEBUG --- http response --- 03:11:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:11:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:11:54 DEBUG { 03:11:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:11:54 DEBUG --- http response --- 03:11:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:11:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 03:11:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:11:54 DEBUG ----- output ----- 03:11:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:11:54 DEBUG ----- output ----- 03:11:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:11:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:11:54 DEBUG { 03:11:54 DEBUG ----- output ----- 03:11:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:11:54 DEBUG ----- output ----- 03:11:54 DEBUG ----- output ----- 03:11:54 DEBUG { 03:11:54 DEBUG ----- output ----- 03:11:54 DEBUG --- http response --- 03:11:54 DEBUG ----- output ----- 03:11:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:11:54 DEBUG --- http response --- 03:11:54 DEBUG --- http response --- 03:11:54 DEBUG "status": "success", 03:11:54 DEBUG --- http response --- 03:11:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:11:54 DEBUG ----- output ----- 03:11:54 DEBUG ----- output ----- 03:11:54 DEBUG { 03:11:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:11:54 DEBUG ----- output ----- 03:11:54 DEBUG { 03:11:54 DEBUG ----- output ----- 03:11:54 DEBUG ----- output ----- 03:11:54 DEBUG "status": "success", 03:11:54 DEBUG { 03:11:54 DEBUG ----- output ----- 03:11:54 DEBUG { 03:11:54 DEBUG { 03:11:54 DEBUG "status": "success", 03:11:54 DEBUG { 03:11:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:11:54 DEBUG { 03:11:54 DEBUG ----- output ----- 03:11:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:11:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:11:54 DEBUG "data": { 03:11:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:11:54 DEBUG ----- output ----- 03:11:54 DEBUG { 03:11:54 DEBUG { 03:11:54 DEBUG "status": "success", 03:11:54 DEBUG ----- output ----- 03:11:54 DEBUG { 03:11:54 DEBUG "status": "success", 03:11:54 DEBUG { 03:11:54 DEBUG { 03:11:54 DEBUG "data": { 03:11:54 DEBUG "status": "success", 03:11:54 DEBUG { 03:11:54 DEBUG "status": "success", 03:11:54 DEBUG "status": "success", 03:11:54 DEBUG "data": { 03:11:54 DEBUG "resultType": "vector", 03:11:54 DEBUG ----- output ----- 03:11:54 DEBUG { 03:11:54 DEBUG { 03:11:54 DEBUG ----- output ----- 03:11:54 DEBUG ----- output ----- 03:11:54 DEBUG "resultType": "vector", 03:11:54 DEBUG ----- output ----- 03:11:54 DEBUG { 03:11:54 DEBUG "status": "success", 03:11:54 DEBUG "status": "success", 03:11:54 DEBUG "data": { 03:11:54 DEBUG { 03:11:54 DEBUG "status": "success", 03:11:54 DEBUG "data": { 03:11:54 DEBUG "status": "success", 03:11:54 DEBUG "status": "success", 03:11:54 DEBUG "resultType": "vector", 03:11:54 DEBUG "data": { 03:11:54 DEBUG "status": "success", 03:11:54 DEBUG "data": { 03:11:54 DEBUG "data": { 03:11:54 DEBUG "result": [ 03:11:54 DEBUG "status": "success", 03:11:54 DEBUG "status": "success", 03:11:54 DEBUG "status": "success", 03:11:54 DEBUG "status": "success", 03:11:54 DEBUG { 03:11:54 DEBUG { 03:11:54 DEBUG "result": [ 03:11:54 DEBUG { 03:11:54 DEBUG "data": { 03:11:54 DEBUG "data": { 03:11:54 DEBUG "resultType": "vector", 03:11:54 DEBUG "resultType": "vector", 03:11:54 DEBUG "status": "success", 03:11:54 DEBUG "resultType": "vector", 03:11:54 DEBUG "data": { 03:11:54 DEBUG "data": { 03:11:54 DEBUG "data": { 03:11:54 DEBUG "resultType": "vector", 03:11:54 DEBUG "resultType": "vector", 03:11:54 DEBUG "data": { 03:11:54 DEBUG "resultType": "vector", 03:11:54 DEBUG "resultType": "vector", 03:11:54 DEBUG { 03:11:54 DEBUG "data": { 03:11:54 DEBUG "data": { 03:11:54 DEBUG "data": { 03:11:54 DEBUG "data": { 03:11:54 DEBUG "status": "success", 03:11:54 DEBUG "status": "success", 03:11:54 DEBUG { 03:11:54 DEBUG "status": "success", 03:11:54 DEBUG "resultType": "vector", 03:11:54 DEBUG "result": [ 03:11:54 DEBUG "result": [ 03:11:54 DEBUG "data": { 03:11:54 DEBUG "status": "success", 03:11:54 DEBUG "result": [ 03:11:54 DEBUG "resultType": "vector", 03:11:54 DEBUG "resultType": "vector", 03:11:54 DEBUG "result": [ 03:11:54 DEBUG "result": [ 03:11:54 DEBUG "result": [ 03:11:54 DEBUG "resultType": "vector", 03:11:54 DEBUG "result": [ 03:11:54 DEBUG "result": [ 03:11:54 DEBUG "metric": { 03:11:54 DEBUG "resultType": "vector", 03:11:54 DEBUG "resultType": "vector", 03:11:54 DEBUG "resultType": "vector", 03:11:54 DEBUG "resultType": "vector", 03:11:54 DEBUG "data": { 03:11:54 DEBUG "data": { 03:11:54 DEBUG "metric": { 03:11:54 DEBUG "data": { 03:11:54 DEBUG "result": [ 03:11:54 DEBUG { 03:11:54 DEBUG { 03:11:54 DEBUG "resultType": "vector", 03:11:54 DEBUG "data": { 03:11:54 DEBUG { 03:11:54 DEBUG "result": [ 03:11:54 DEBUG { 03:11:54 DEBUG "result": [ 03:11:54 DEBUG { 03:11:54 DEBUG "metric": { 03:11:54 DEBUG "result": [ 03:11:54 DEBUG { 03:11:54 DEBUG "metric": { 03:11:54 DEBUG "pod": "ds-idrepo-0" 03:11:54 DEBUG "result": [ 03:11:54 DEBUG { 03:11:54 DEBUG "result": [ 03:11:54 DEBUG "result": [ 03:11:54 DEBUG "resultType": "vector", 03:11:54 DEBUG "resultType": "vector", 03:11:54 DEBUG "pod": "ds-cts-0" 03:11:54 DEBUG "resultType": "vector", 03:11:54 DEBUG "result": [ 03:11:54 DEBUG "metric": { 03:11:54 DEBUG "metric": { 03:11:54 DEBUG "result": [ 03:11:54 DEBUG "resultType": "vector", 03:11:54 DEBUG "metric": { 03:11:54 DEBUG { 03:11:54 DEBUG "metric": { 03:11:54 DEBUG { 03:11:54 DEBUG "pod": "am-767f6b67b5-9zw22" 03:11:54 DEBUG "pod": "am-767f6b67b5-9zw22" 03:11:54 DEBUG { 03:11:54 DEBUG { 03:11:54 DEBUG "metric": { 03:11:54 DEBUG "instance": "10.142.0.107:9100" 03:11:54 DEBUG "metric": { 03:11:54 DEBUG "result": [ 03:11:54 DEBUG { 03:11:54 DEBUG { 03:11:54 DEBUG "result": [ 03:11:54 DEBUG "result": [ 03:11:54 DEBUG }, 03:11:54 DEBUG { 03:11:54 DEBUG "pod": "ds-idrepo-1" 03:11:54 DEBUG "pod": "am-767f6b67b5-9zw22" 03:11:54 DEBUG { 03:11:54 DEBUG "metric": { 03:11:54 DEBUG "result": [ 03:11:54 DEBUG "pod": "ds-idrepo-1" 03:11:54 DEBUG }, 03:11:54 DEBUG "pod": "ds-idrepo-0" 03:11:54 DEBUG "metric": { 03:11:54 DEBUG "pod": "ds-idrepo-1" 03:11:54 DEBUG }, 03:11:54 DEBUG "metric": { 03:11:54 DEBUG }, 03:11:54 DEBUG }, 03:11:54 DEBUG { 03:11:54 DEBUG "pod": "ds-cts-0" 03:11:54 DEBUG { 03:11:54 DEBUG "metric": { 03:11:54 DEBUG "metric": { 03:11:54 DEBUG { 03:11:54 DEBUG { 03:11:54 DEBUG "value": [ 03:11:54 DEBUG "metric": { 03:11:54 DEBUG }, 03:11:54 DEBUG }, 03:11:54 DEBUG { 03:11:54 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:11:54 DEBUG { 03:11:54 DEBUG "value": [ 03:11:54 DEBUG }, 03:11:54 DEBUG "metric": { 03:11:54 DEBUG }, 03:11:54 DEBUG "value": [ 03:11:54 DEBUG "value": [ 03:11:54 DEBUG "pod": "am-767f6b67b5-9zw22" 03:11:54 DEBUG "value": [ 03:11:54 DEBUG "value": [ 03:11:54 DEBUG "metric": { 03:11:54 DEBUG }, 03:11:54 DEBUG "metric": { 03:11:54 DEBUG "pod": "am-767f6b67b5-9zw22" 03:11:54 DEBUG "instance": "10.142.0.107:9100" 03:11:54 DEBUG "metric": { 03:11:54 DEBUG "metric": { 03:11:54 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:11:54 DEBUG "pod": "ds-idrepo-0" 03:11:54 DEBUG "value": [ 03:11:54 DEBUG "value": [ 03:11:54 DEBUG "metric": { 03:11:54 DEBUG }, 03:11:54 DEBUG "metric": { 03:11:54 DEBUG 03:11:54 DEBUG "value": [ 03:11:54 DEBUG "pod": "am-767f6b67b5-9zw22" 03:11:54 DEBUG 03:11:54 DEBUG }, 03:11:54 DEBUG 03:11:54 DEBUG }, 03:11:54 DEBUG "value": [ 03:11:54 DEBUG 03:11:54 DEBUG "pod": "ds-idrepo-0" 03:11:54 DEBUG "value": [ 03:11:54 DEBUG "instance": "10.142.0.107:9100" 03:11:54 DEBUG }, 03:11:54 DEBUG }, 03:11:54 DEBUG "pod": "am-767f6b67b5-9zw22" 03:11:54 DEBUG 03:11:54 DEBUG }, 03:11:54 DEBUG }, 03:11:54 DEBUG 03:11:54 DEBUG 03:11:54 DEBUG "instance": "10.142.0.107:9100" 03:11:54 DEBUG "value": [ 03:11:54 DEBUG "pod": "ds-idrepo-1" 03:11:54 DEBUG 03:11:54 DEBUG 03:11:54 DEBUG }, 03:11:54 DEBUG 03:11:54 DEBUG "value": [ 03:11:54 DEBUG 03:11:54 DEBUG 03:11:54 DEBUG 03:11:54 DEBUG 03:11:54 DEBUG }, 03:11:54 DEBUG 03:11:54 DEBUG }, 03:11:54 DEBUG "value": [ 03:11:54 DEBUG "value": [ 03:11:54 DEBUG }, 03:11:54 DEBUG 03:11:54 DEBUG 03:11:54 DEBUG "value": [ 03:11:54 DEBUG 03:11:54 DEBUG 03:11:54 DEBUG }, 03:11:54 DEBUG 03:11:54 DEBUG }, 03:11:54 DEBUG 03:11:54 DEBUG 03:11:54 DEBUG "value": [ 03:11:54 DEBUG 03:11:54 DEBUG 03:11:54 DEBUG 03:11:54 DEBUG 03:11:54 DEBUG [...] 03:11:54 DEBUG 03:11:54 DEBUG "value": [ 03:11:54 DEBUG 03:11:54 DEBUG "value": [ 03:11:54 DEBUG 03:11:54 DEBUG 03:11:54 DEBUG "value": [ 03:11:54 DEBUG [...] 03:11:54 DEBUG 03:11:54 DEBUG "value": [ 03:11:54 DEBUG 03:11:54 DEBUG 03:11:54 DEBUG "value": [ 03:11:54 DEBUG 03:11:54 DEBUG "value": [ 03:11:54 DEBUG [...] 03:11:54 DEBUG 03:11:54 DEBUG 03:11:54 DEBUG [...] 03:11:54 DEBUG 03:11:54 DEBUG 03:11:54 DEBUG 03:11:54 DEBUG [...] 03:11:54 DEBUG 03:11:54 DEBUG 03:11:54 DEBUG 03:11:54 DEBUG 03:11:54 DEBUG 03:11:54 DEBUG 03:11:54 DEBUG [...] 03:11:54 DEBUG 03:11:54 DEBUG 03:11:54 DEBUG 03:11:54 DEBUG [...] 03:11:54 DEBUG 03:11:54 DEBUG 03:11:54 DEBUG 03:11:54 DEBUG 03:11:54 DEBUG 03:11:54 DEBUG [...] 03:11:54 DEBUG 03:11:54 DEBUG 03:11:54 DEBUG 03:11:54 DEBUG 03:11:54 DEBUG 03:11:54 DEBUG 03:11:54 DEBUG 03:11:54 DEBUG [...] 03:11:54 DEBUG 03:11:54 DEBUG 03:11:54 DEBUG 03:11:54 DEBUG 03:11:54 DEBUG 03:11:54 DEBUG 03:11:54 DEBUG 03:11:54 DEBUG 03:11:54 DEBUG 03:11:54 DEBUG [...] 03:11:54 DEBUG 03:11:54 DEBUG [...] 03:11:54 DEBUG 03:11:54 DEBUG 03:11:54 DEBUG 03:11:54 DEBUG 03:11:54 DEBUG [...] 03:11:54 DEBUG 03:11:54 DEBUG "pod": "am-767f6b67b5-vbltz" 03:11:54 DEBUG [...] 03:11:54 DEBUG 03:11:54 DEBUG 03:11:54 DEBUG 03:11:54 DEBUG 03:11:54 DEBUG [...] 03:11:54 DEBUG 03:11:54 DEBUG 03:11:54 DEBUG 03:11:54 DEBUG "pod": "ds-cts-2" 03:11:54 DEBUG 03:11:54 DEBUG [...] 03:11:54 DEBUG 03:11:54 DEBUG 03:11:54 DEBUG 03:11:54 DEBUG 03:11:54 DEBUG 03:11:54 DEBUG 03:11:54 DEBUG 03:11:54 DEBUG [...] 03:11:54 DEBUG 03:11:54 DEBUG }, 03:11:54 DEBUG 03:11:54 DEBUG 03:11:54 DEBUG 03:11:54 DEBUG [...] 03:11:54 DEBUG 03:11:54 DEBUG [...] 03:11:54 DEBUG 03:11:54 DEBUG 03:11:54 DEBUG [...] 03:11:54 DEBUG 03:11:54 DEBUG }, 03:11:54 DEBUG [...] 03:11:54 DEBUG 03:11:54 DEBUG 03:11:54 DEBUG 03:11:54 DEBUG 03:11:54 DEBUG 03:11:54 DEBUG 03:11:54 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:11:54 DEBUG }, 03:11:54 DEBUG 03:11:54 DEBUG 03:11:54 DEBUG "value": [ 03:11:54 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 03:11:54 DEBUG }, 03:11:54 DEBUG "pod": "ds-idrepo-2" 03:11:54 DEBUG 03:11:54 DEBUG 03:11:54 DEBUG 03:11:54 DEBUG 03:11:54 DEBUG 03:11:54 DEBUG 03:11:54 DEBUG [...] 03:11:54 DEBUG "value": [ 03:11:54 DEBUG 03:11:54 DEBUG 03:11:54 DEBUG "pod": "am-767f6b67b5-vbltz" 03:11:54 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 03:11:54 DEBUG }, 03:11:54 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 03:11:54 DEBUG [...] 03:11:54 DEBUG 03:11:54 DEBUG "pod": "ds-idrepo-2" 03:11:54 DEBUG 03:11:54 DEBUG 03:11:54 DEBUG 1713665514, 03:11:54 DEBUG 03:11:54 DEBUG "value": [ 03:11:54 DEBUG }, 03:11:54 DEBUG 03:11:54 DEBUG "pod": "ds-cts-2" 03:11:54 DEBUG 03:11:54 DEBUG "pod": "am-767f6b67b5-vbltz" 03:11:54 DEBUG "pod": "am-767f6b67b5-vbltz" 03:11:54 DEBUG }, 03:11:54 DEBUG 03:11:54 DEBUG 1713665514, 03:11:54 DEBUG 03:11:54 DEBUG 03:11:54 DEBUG }, 03:11:54 DEBUG "value": [ 03:11:54 DEBUG }, 03:11:54 DEBUG [...] 03:11:54 DEBUG 03:11:54 DEBUG }, 03:11:54 DEBUG "value": [ 03:11:54 DEBUG 03:11:54 DEBUG "pod": "am-767f6b67b5-vbltz" 03:11:54 DEBUG "600457.5" 03:11:54 DEBUG 03:11:54 DEBUG 1713665514, 03:11:54 DEBUG "value": [ 03:11:54 DEBUG 03:11:54 DEBUG }, 03:11:54 DEBUG 03:11:54 DEBUG }, 03:11:54 DEBUG 03:11:54 DEBUG "value": [ 03:11:54 DEBUG 03:11:54 DEBUG "0" 03:11:54 DEBUG 03:11:54 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:11:54 DEBUG "value": [ 03:11:54 DEBUG 1713665514, 03:11:54 DEBUG "value": [ 03:11:54 DEBUG 03:11:54 DEBUG 03:11:54 DEBUG "value": [ 03:11:54 DEBUG 1713665514, 03:11:54 DEBUG "pod": "am-767f6b67b5-vbltz" 03:11:54 DEBUG }, 03:11:54 DEBUG "value": [ 03:11:54 DEBUG "instance": "10.142.0.43:9100" 03:11:54 DEBUG "0" 03:11:54 DEBUG 1713665514, 03:11:54 DEBUG "pod": "ds-cts-2" 03:11:54 DEBUG "value": [ 03:11:54 DEBUG "instance": "10.142.0.43:9100" 03:11:54 DEBUG "value": [ 03:11:54 DEBUG 03:11:54 DEBUG 1713665514, 03:11:54 DEBUG 03:11:54 DEBUG ] 03:11:54 DEBUG "pod": "ds-idrepo-2" 03:11:54 DEBUG }, 03:11:54 DEBUG 1713665514, 03:11:54 DEBUG "0" 03:11:54 DEBUG ] 03:11:54 DEBUG 03:11:54 DEBUG 03:11:54 DEBUG 1713665514, 03:11:54 DEBUG "1013411840" 03:11:54 DEBUG 1713665514, 03:11:54 DEBUG }, 03:11:54 DEBUG }, 03:11:54 DEBUG ] 03:11:54 DEBUG ] 03:11:54 DEBUG "0" 03:11:54 DEBUG }, 03:11:54 DEBUG 1713665514, 03:11:54 DEBUG "0" 03:11:54 DEBUG 1713665514, 03:11:54 DEBUG "pod": "am-767f6b67b5-vbltz" 03:11:54 DEBUG "3306.6" 03:11:54 DEBUG "instance": "10.142.0.43:9100" 03:11:54 DEBUG } 03:11:54 DEBUG }, 03:11:54 DEBUG "value": [ 03:11:54 DEBUG 1713665514, 03:11:54 DEBUG "0" 03:11:54 DEBUG 1713665514, 03:11:54 DEBUG 03:11:54 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 03:11:54 DEBUG "0" 03:11:54 DEBUG ] 03:11:54 DEBUG "3306.866666666667" 03:11:54 DEBUG "value": [ 03:11:54 DEBUG "value": [ 03:11:54 DEBUG } 03:11:54 DEBUG } 03:11:54 DEBUG ] 03:11:54 DEBUG } 03:11:54 DEBUG ] 03:11:54 DEBUG }, 03:11:54 DEBUG "3326.266666666667" 03:11:54 DEBUG }, 03:11:54 DEBUG ] 03:11:54 DEBUG }, 03:11:54 DEBUG ] 03:11:54 DEBUG "value": [ 03:11:54 DEBUG "11.771762591267288" 03:11:54 DEBUG ] 03:11:54 DEBUG ] 03:11:54 DEBUG "0.000034468209405764385" 03:11:54 DEBUG "instance": "10.142.0.43:9100" 03:11:54 DEBUG }, 03:11:54 DEBUG ] 03:11:54 DEBUG } 03:11:54 DEBUG ] 03:11:54 DEBUG } 03:11:54 DEBUG 1713665514, 03:11:54 DEBUG ] 03:11:54 DEBUG ] 03:11:54 DEBUG ] 03:11:54 DEBUG "value": [ 03:11:54 DEBUG } 03:11:54 DEBUG "value": [ 03:11:54 DEBUG ] 03:11:54 DEBUG "value": [ 03:11:54 DEBUG } 03:11:54 DEBUG "value": [ 03:11:54 DEBUG } 03:11:54 DEBUG 1713665514, 03:11:54 DEBUG } 03:11:54 DEBUG } 03:11:54 DEBUG } 03:11:54 DEBUG ] 03:11:54 DEBUG }, 03:11:54 DEBUG "value": [ 03:11:54 DEBUG } 03:11:54 DEBUG ] 03:11:54 DEBUG ] 03:11:54 DEBUG 1713665514, 03:11:54 DEBUG "100371803.66666666" 03:11:54 DEBUG } 03:11:54 DEBUG } 03:11:54 DEBUG } 03:11:54 DEBUG 1713665514, 03:11:54 DEBUG "0" 03:11:54 DEBUG 1713665514, 03:11:54 DEBUG } 03:11:54 DEBUG ] 03:11:54 DEBUG ] 03:11:54 DEBUG 1713665514, 03:11:54 DEBUG } 03:11:54 DEBUG "0" 03:11:54 DEBUG ] 03:11:54 DEBUG ] 03:11:54 DEBUG ] 03:11:54 DEBUG } 03:11:54 DEBUG "value": [ 03:11:54 DEBUG 1713665514, 03:11:54 DEBUG ] 03:11:54 DEBUG } 03:11:54 DEBUG } 03:11:54 DEBUG "3323.3333333333335" 03:11:54 DEBUG ] 03:11:54 DEBUG } 03:11:54 DEBUG } 03:11:54 DEBUG } 03:11:54 DEBUG ] 03:11:54 DEBUG ] 03:11:54 DEBUG "4355780164608" 03:11:54 DEBUG } 03:11:54 DEBUG } 03:11:54 DEBUG } 03:11:54 DEBUG } 03:11:54 DEBUG } 03:11:54 DEBUG ] 03:11:54 DEBUG } 03:11:54 DEBUG } 03:11:54 DEBUG ] 03:11:54 DEBUG 1713665514, 03:11:54 DEBUG "0" 03:11:54 DEBUG } 03:11:54 DEBUG } 03:11:54 DEBUG } 03:11:54 DEBUG ] 03:11:54 DEBUG } 03:11:54 DEBUG } 03:11:54 DEBUG } 03:11:54 DEBUG ] 03:11:54 DEBUG 1713665514, 03:11:54 DEBUG "100373385.93333334" 03:11:54 DEBUG ] 03:11:54 DEBUG } 03:11:54 DEBUG } 03:11:54 DEBUG } 03:11:54 DEBUG } 03:11:54 DEBUG } 03:11:54 DEBUG ] 03:11:54 DEBUG } 03:11:54 DEBUG } 03:11:54 DEBUG ] 03:11:54 DEBUG } 03:11:54 DEBUG ] 03:11:54 DEBUG } 03:11:54 DEBUG "149.889" 03:11:54 DEBUG ] 03:11:54 DEBUG } 03:11:54 DEBUG } 03:11:54 DEBUG "0.08999999999999582" 03:11:54 DEBUG } 03:11:54 DEBUG ] 03:11:54 DEBUG } 03:11:54 DEBUG } 03:11:54 DEBUG ] 03:11:54 DEBUG ] 03:11:54 DEBUG } 03:11:54 DEBUG } 03:11:54 DEBUG ] 03:11:54 DEBUG ] 03:11:54 DEBUG } 03:11:54 DEBUG } 03:11:54 DEBUG } 03:11:54 DEBUG } 03:11:54 DEBUG ] 03:11:54 DEBUG } 03:11:54 DEBUG } 03:11:54 DEBUG } 03:11:54 DEBUG } 03:11:54 DEBUG } 03:11:54 DEBUG ] 03:11:54 DEBUG } 03:11:54 DEBUG ] 03:11:54 DEBUG } 03:11:54 DEBUG } 03:11:54 DEBUG } 03:11:54 DEBUG } 03:11:54 DEBUG } 03:11:54 DEBUG } 03:12:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713665572" 03:12:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665572" 03:12:52 INFO [http_cmd]: http status code OK 03:12:52 DEBUG --- status code --- 03:12:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:12:52 ERROR [http_cmd]: ERROR 03:12:52 DEBUG --- http response --- 03:12:52 ERROR --- status code --- 03:12:52 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 03:12:52 ERROR http status code is 400 (expected [200, 401, 403, 404]) 03:12:52 ERROR --- http response --- 03:12:52 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 03:12:52 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 03:12:53 INFO 03:12:53 INFO [loop_until]: kubectl --namespace=xlou top pods 03:12:53 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:12:53 INFO [loop_until]: OK (rc = 0) 03:12:53 DEBUG --- stdout --- 03:12:53 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 21547m 8795Mi am-767f6b67b5-qcqz2 22716m 9129Mi am-767f6b67b5-vbltz 24474m 8824Mi ds-cts-0 19919m 12393Mi ds-cts-1 19489m 12363Mi ds-cts-2 19869m 12372Mi ds-idrepo-0 9485m 3918Mi ds-idrepo-1 9716m 3946Mi ds-idrepo-2 9563m 3915Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 3m 1179Mi idm-b85f9d978-wnhxb 3m 1239Mi lodemon-54c87f8cbb-jzrjs 18m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-68c5d5b884-p97w5 11676m 875Mi 03:12:53 DEBUG --- stderr --- 03:12:53 DEBUG 03:12:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665573" 03:12:54 INFO [http_cmd]: http status code OK 03:12:54 DEBUG --- status code --- 03:12:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:12:54 DEBUG --- http response --- 03:12:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:12:54 DEBUG ----- output ----- 03:12:54 DEBUG { 03:12:54 DEBUG "status": "success", 03:12:54 DEBUG "data": { 03:12:54 DEBUG "resultType": "vector", 03:12:54 DEBUG "result": [ 03:12:54 DEBUG { 03:12:54 DEBUG "metric": { 03:12:54 DEBUG "instance": "10.142.0.107:9100" 03:12:54 DEBUG }, 03:12:54 DEBUG "value": [ 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG [...] 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG "instance": "10.142.0.43:9100" 03:12:54 DEBUG }, 03:12:54 DEBUG "value": [ 03:12:54 DEBUG 1713665573, 03:12:54 DEBUG "0.39089999999998404" 03:12:54 DEBUG ] 03:12:54 DEBUG } 03:12:54 DEBUG ] 03:12:54 DEBUG } 03:12:54 DEBUG } 03:12:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665574" 03:12:54 INFO [http_cmd]: http status code OK 03:12:54 DEBUG --- status code --- 03:12:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:12:54 DEBUG --- http response --- 03:12:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:12:54 DEBUG ----- output ----- 03:12:54 DEBUG { 03:12:54 DEBUG "status": "success", 03:12:54 DEBUG "data": { 03:12:54 DEBUG "resultType": "vector", 03:12:54 DEBUG "result": [ 03:12:54 DEBUG { 03:12:54 DEBUG "metric": { 03:12:54 DEBUG "pod": "ds-cts-0" 03:12:54 DEBUG }, 03:12:54 DEBUG "value": [ 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG [...] 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG "pod": "ds-cts-2" 03:12:54 DEBUG }, 03:12:54 DEBUG "value": [ 03:12:54 DEBUG 1713665574, 03:12:54 DEBUG "0.5660000000000001" 03:12:54 DEBUG ] 03:12:54 DEBUG } 03:12:54 DEBUG ] 03:12:54 DEBUG } 03:12:54 DEBUG } 03:12:54 INFO 03:12:54 INFO [loop_until]: kubectl --namespace=xlou top node 03:12:54 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:12:54 INFO [loop_until]: OK (rc = 0) 03:12:54 DEBUG --- stdout --- 03:12:54 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 65m 0% 6275Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 9948m 33% 7762Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 21412m 71% 11603Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 93m 0% 3371Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 20350m 68% 15715Mi 14% gke-xlou-cdm-default-pool-1b0769c8-wpwl 135m 0% 5279Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 22325m 74% 12063Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 19715m 66% 3180Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 24685m 82% 11084Mi 9% gke-xlou-cdm-ds-0f601751-j488 10355m 34% 7374Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 19273m 64% 17491Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 10191m 34% 8006Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20365m 68% 15131Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 11983m 75% 11228Mi 19% 03:12:54 DEBUG --- stderr --- 03:12:54 DEBUG 03:12:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665574" 03:12:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665574" 03:12:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665574" 03:12:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665574" 03:12:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665574" 03:12:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665574" 03:12:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713665574" 03:12:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665574" 03:12:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665574" 03:12:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665574" 03:12:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665574" 03:12:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665574" 03:12:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665574" 03:12:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665574" 03:12:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713665574" 03:12:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665574" 03:12:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713665574" 03:12:54 INFO [http_cmd]: http status code OK 03:12:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665574" 03:12:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665574" 03:12:54 INFO [http_cmd]: http status code OK 03:12:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665574" 03:12:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665574" 03:12:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713665574" 03:12:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665574" 03:12:54 INFO [http_cmd]: http status code OK 03:12:54 INFO [http_cmd]: http status code OK 03:12:54 INFO [http_cmd]: http status code OK 03:12:54 INFO [http_cmd]: http status code OK 03:12:54 INFO [http_cmd]: http status code OK 03:12:54 INFO [http_cmd]: http status code OK 03:12:54 DEBUG --- status code --- 03:12:54 INFO [http_cmd]: http status code OK 03:12:54 INFO [http_cmd]: http status code OK 03:12:54 DEBUG --- status code --- 03:12:54 INFO [http_cmd]: http status code OK 03:12:54 INFO [http_cmd]: http status code OK 03:12:54 INFO [http_cmd]: http status code OK 03:12:54 INFO [http_cmd]: http status code OK 03:12:54 INFO [http_cmd]: http status code OK 03:12:54 INFO [http_cmd]: http status code OK 03:12:54 INFO [http_cmd]: http status code OK 03:12:54 INFO [http_cmd]: http status code OK 03:12:54 DEBUG --- status code --- 03:12:54 INFO [http_cmd]: http status code OK 03:12:54 DEBUG --- status code --- 03:12:54 DEBUG --- status code --- 03:12:54 DEBUG --- status code --- 03:12:54 DEBUG --- status code --- 03:12:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:12:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:12:54 DEBUG --- status code --- 03:12:54 DEBUG --- status code --- 03:12:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:12:54 DEBUG --- http response --- 03:12:54 DEBUG --- status code --- 03:12:54 DEBUG --- status code --- 03:12:54 INFO [http_cmd]: http status code OK 03:12:54 DEBUG --- status code --- 03:12:54 DEBUG --- status code --- 03:12:54 DEBUG --- status code --- 03:12:54 DEBUG --- status code --- 03:12:54 DEBUG --- status code --- 03:12:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:12:54 DEBUG --- status code --- 03:12:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:12:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:12:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:12:54 DEBUG --- http response --- 03:12:54 DEBUG --- status code --- 03:12:54 DEBUG --- http response --- 03:12:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:12:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:12:54 DEBUG --- status code --- 03:12:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:12:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:12:54 DEBUG ----- output ----- 03:12:54 DEBUG --- status code --- 03:12:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:12:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:12:54 INFO [http_cmd]: http status code OK 03:12:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:12:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:12:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:12:54 DEBUG --- http response --- 03:12:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:12:54 DEBUG --- http response --- 03:12:54 DEBUG --- http response --- 03:12:54 DEBUG --- http response --- 03:12:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:12:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:12:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:12:54 DEBUG --- http response --- 03:12:54 DEBUG --- http response --- 03:12:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:12:54 DEBUG --- http response --- 03:12:54 DEBUG { 03:12:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:12:54 DEBUG --- http response --- 03:12:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:12:54 DEBUG --- http response --- 03:12:54 DEBUG --- status code --- 03:12:54 DEBUG --- http response --- 03:12:54 DEBUG --- http response --- 03:12:54 DEBUG --- http response --- 03:12:54 DEBUG --- http response --- 03:12:54 INFO [http_cmd]: http status code OK 03:12:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:12:54 INFO [http_cmd]: http status code OK 03:12:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:12:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:12:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:12:54 DEBUG ----- output ----- 03:12:54 DEBUG --- http response --- 03:12:54 DEBUG ----- output ----- 03:12:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:12:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 03:12:54 DEBUG --- http response --- 03:12:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:12:54 DEBUG "status": "success", 03:12:54 DEBUG --- http response --- 03:12:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:12:54 DEBUG ----- output ----- 03:12:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:12:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:12:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:12:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:12:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 03:12:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:12:54 DEBUG --- status code --- 03:12:54 DEBUG ----- output ----- 03:12:54 DEBUG --- status code --- 03:12:54 DEBUG ----- output ----- 03:12:54 DEBUG { 03:12:54 DEBUG ----- output ----- 03:12:54 DEBUG { 03:12:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:12:54 DEBUG { 03:12:54 DEBUG "status": "success", 03:12:54 DEBUG ----- output ----- 03:12:54 DEBUG ----- output ----- 03:12:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:12:54 DEBUG "data": { 03:12:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:12:54 DEBUG --- http response --- 03:12:54 DEBUG { 03:12:54 DEBUG --- http response --- 03:12:54 DEBUG ----- output ----- 03:12:54 DEBUG ----- output ----- 03:12:54 DEBUG ----- output ----- 03:12:54 DEBUG ----- output ----- 03:12:54 DEBUG ----- output ----- 03:12:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:12:54 DEBUG { 03:12:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:12:54 DEBUG ----- output ----- 03:12:54 DEBUG "status": "success", 03:12:54 DEBUG { 03:12:54 DEBUG "status": "success", 03:12:54 DEBUG "data": { 03:12:54 DEBUG "resultType": "vector", 03:12:54 DEBUG ----- output ----- 03:12:54 DEBUG { 03:12:54 DEBUG { 03:12:54 DEBUG ----- output ----- 03:12:54 DEBUG "resultType": "vector", 03:12:54 DEBUG ----- output ----- 03:12:54 DEBUG "status": "success", 03:12:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:12:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:12:54 DEBUG { 03:12:54 DEBUG { 03:12:54 DEBUG { 03:12:54 DEBUG { 03:12:54 DEBUG "status": "success", 03:12:54 DEBUG --- http response --- 03:12:54 DEBUG "status": "success", 03:12:54 DEBUG --- http response --- 03:12:54 DEBUG { 03:12:54 DEBUG "data": { 03:12:54 DEBUG "status": "success", 03:12:54 DEBUG "result": [ 03:12:54 DEBUG ----- output ----- 03:12:54 DEBUG "data": { 03:12:54 DEBUG { 03:12:54 DEBUG "status": "success", 03:12:54 DEBUG "status": "success", 03:12:54 DEBUG { 03:12:54 DEBUG "result": [ 03:12:54 DEBUG { 03:12:54 DEBUG "data": { 03:12:54 DEBUG ----- output ----- 03:12:54 DEBUG ----- output ----- 03:12:54 DEBUG "status": "success", 03:12:54 DEBUG "status": "success", 03:12:54 DEBUG "status": "success", 03:12:54 DEBUG "data": { 03:12:54 DEBUG { 03:12:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:12:54 DEBUG ----- output ----- 03:12:54 DEBUG "data": { 03:12:54 DEBUG "status": "success", 03:12:54 DEBUG "resultType": "vector", 03:12:54 DEBUG "data": { 03:12:54 DEBUG { 03:12:54 DEBUG { 03:12:54 DEBUG "resultType": "vector", 03:12:54 DEBUG "data": { 03:12:54 DEBUG "data": { 03:12:54 DEBUG "status": "success", 03:12:54 DEBUG "status": "success", 03:12:54 DEBUG { 03:12:54 DEBUG "status": "success", 03:12:54 DEBUG "resultType": "vector", 03:12:54 DEBUG { 03:12:54 DEBUG { 03:12:54 DEBUG "data": { 03:12:54 DEBUG "data": { 03:12:54 DEBUG "data": { 03:12:54 DEBUG "resultType": "vector", 03:12:54 DEBUG "status": "success", 03:12:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:12:54 DEBUG { 03:12:54 DEBUG "resultType": "vector", 03:12:54 DEBUG "data": { 03:12:54 DEBUG "result": [ 03:12:54 DEBUG "resultType": "vector", 03:12:54 DEBUG "metric": { 03:12:54 DEBUG "status": "success", 03:12:54 DEBUG "result": [ 03:12:54 DEBUG "resultType": "vector", 03:12:54 DEBUG "resultType": "vector", 03:12:54 DEBUG "data": { 03:12:54 DEBUG "data": { 03:12:54 DEBUG "metric": { 03:12:54 DEBUG "data": { 03:12:54 DEBUG "result": [ 03:12:54 DEBUG "status": "success", 03:12:54 DEBUG "status": "success", 03:12:54 DEBUG "resultType": "vector", 03:12:54 DEBUG "resultType": "vector", 03:12:54 DEBUG "resultType": "vector", 03:12:54 DEBUG "result": [ 03:12:54 DEBUG "data": { 03:12:54 DEBUG ----- output ----- 03:12:54 DEBUG "status": "success", 03:12:54 DEBUG "result": [ 03:12:54 DEBUG "resultType": "vector", 03:12:54 DEBUG "result": [ 03:12:54 DEBUG "result": [ 03:12:54 DEBUG "pod": "am-767f6b67b5-9zw22" 03:12:54 DEBUG }, 03:12:54 DEBUG "value": [ 03:12:54 DEBUG "result": [ 03:12:54 DEBUG "result": [ 03:12:54 DEBUG { 03:12:54 DEBUG "resultType": "vector", 03:12:54 DEBUG "pod": "ds-cts-0" 03:12:54 DEBUG "resultType": "vector", 03:12:54 DEBUG "result": [ 03:12:54 DEBUG "data": { 03:12:54 DEBUG "data": { 03:12:54 DEBUG "result": [ 03:12:54 DEBUG "result": [ 03:12:54 DEBUG "result": [ 03:12:54 DEBUG { 03:12:54 DEBUG "resultType": "vector", 03:12:54 DEBUG { 03:12:54 DEBUG "data": { 03:12:54 DEBUG { 03:12:54 DEBUG { 03:12:54 DEBUG { 03:12:54 DEBUG { 03:12:54 DEBUG "data": { 03:12:54 DEBUG 03:12:54 DEBUG { 03:12:54 DEBUG { 03:12:54 DEBUG "resultType": "vector", 03:12:54 DEBUG "metric": { 03:12:54 DEBUG "result": [ 03:12:54 DEBUG }, 03:12:54 DEBUG { 03:12:54 DEBUG { 03:12:54 DEBUG "resultType": "vector", 03:12:54 DEBUG "result": [ 03:12:54 DEBUG { 03:12:54 DEBUG "metric": { 03:12:54 DEBUG { 03:12:54 DEBUG "metric": { 03:12:54 DEBUG "result": [ 03:12:54 DEBUG "status": "success", 03:12:54 DEBUG "resultType": "vector", 03:12:54 DEBUG "metric": { 03:12:54 DEBUG "metric": { 03:12:54 DEBUG "metric": { 03:12:54 DEBUG "metric": { 03:12:54 DEBUG "resultType": "vector", 03:12:54 DEBUG 03:12:54 DEBUG "metric": { 03:12:54 DEBUG "metric": { 03:12:54 DEBUG "result": [ 03:12:54 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:12:54 DEBUG { 03:12:54 DEBUG "value": [ 03:12:54 DEBUG "metric": { 03:12:54 DEBUG "metric": { 03:12:54 DEBUG { 03:12:54 DEBUG "resultType": "vector", 03:12:54 DEBUG { 03:12:54 DEBUG "pod": "ds-idrepo-0" 03:12:54 DEBUG "metric": { 03:12:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:12:54 DEBUG { 03:12:54 DEBUG "data": { 03:12:54 DEBUG "result": [ 03:12:54 DEBUG "instance": "10.142.0.107:9100" 03:12:54 DEBUG "pod": "am-767f6b67b5-9zw22" 03:12:54 DEBUG "pod": "am-767f6b67b5-9zw22" 03:12:54 DEBUG "pod": "am-767f6b67b5-9zw22" 03:12:54 DEBUG "result": [ 03:12:54 DEBUG 03:12:54 DEBUG "pod": "ds-idrepo-0" 03:12:54 DEBUG "pod": "am-767f6b67b5-9zw22" 03:12:54 DEBUG { 03:12:54 DEBUG }, 03:12:54 DEBUG "metric": { 03:12:54 DEBUG 03:12:54 DEBUG "instance": "10.142.0.107:9100" 03:12:54 DEBUG "pod": "ds-idrepo-0" 03:12:54 DEBUG "metric": { 03:12:54 DEBUG "result": [ 03:12:54 DEBUG "metric": { 03:12:54 DEBUG }, 03:12:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:12:54 DEBUG }, 03:12:54 DEBUG "metric": { 03:12:54 DEBUG "resultType": "vector", 03:12:54 DEBUG { 03:12:54 DEBUG }, 03:12:54 DEBUG }, 03:12:54 DEBUG }, 03:12:54 DEBUG }, 03:12:54 DEBUG { 03:12:54 DEBUG [...] 03:12:54 DEBUG }, 03:12:54 DEBUG }, 03:12:54 DEBUG "metric": { 03:12:54 DEBUG "value": [ 03:12:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:12:54 DEBUG 03:12:54 DEBUG }, 03:12:54 DEBUG }, 03:12:54 DEBUG "pod": "ds-cts-0" 03:12:54 DEBUG { 03:12:54 DEBUG "pod": "ds-idrepo-0" 03:12:54 DEBUG "value": [ 03:12:54 DEBUG }, 03:12:54 DEBUG "value": [ 03:12:54 DEBUG "instance": "10.142.0.107:9100" 03:12:54 DEBUG "result": [ 03:12:54 DEBUG "metric": { 03:12:54 DEBUG "value": [ 03:12:54 DEBUG "value": [ 03:12:54 DEBUG "value": [ 03:12:54 DEBUG "value": [ 03:12:54 DEBUG "metric": { 03:12:54 DEBUG 03:12:54 DEBUG "value": [ 03:12:54 DEBUG "value": [ 03:12:54 DEBUG "pod": "am-767f6b67b5-9zw22" 03:12:54 DEBUG 03:12:54 DEBUG }, 03:12:54 DEBUG 03:12:54 DEBUG "value": [ 03:12:54 DEBUG "value": [ 03:12:54 DEBUG }, 03:12:54 DEBUG "metric": { 03:12:54 DEBUG }, 03:12:54 DEBUG 03:12:54 DEBUG "value": [ 03:12:54 DEBUG 03:12:54 DEBUG }, 03:12:54 DEBUG { 03:12:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG }, 03:12:54 DEBUG 03:12:54 DEBUG "value": [ 03:12:54 DEBUG [...] 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG "value": [ 03:12:54 DEBUG "instance": "10.142.0.107:9100" 03:12:54 DEBUG "value": [ 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG "value": [ 03:12:54 DEBUG "metric": { 03:12:54 DEBUG }, 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG }, 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG "value": [ 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG }, 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG "pod": "am-767f6b67b5-9zw22" 03:12:54 DEBUG "value": [ 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG "value": [ 03:12:54 DEBUG "pod": "am-767f6b67b5-vbltz" 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG [...] 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG "value": [ 03:12:54 DEBUG 03:12:54 DEBUG [...] 03:12:54 DEBUG 03:12:54 DEBUG [...] 03:12:54 DEBUG 03:12:54 DEBUG }, 03:12:54 DEBUG 03:12:54 DEBUG [...] 03:12:54 DEBUG [...] 03:12:54 DEBUG [...] 03:12:54 DEBUG [...] 03:12:54 DEBUG 03:12:54 DEBUG }, 03:12:54 DEBUG [...] 03:12:54 DEBUG [...] 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG [...] 03:12:54 DEBUG [...] 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG [...] 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG "value": [ 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG "value": [ 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG [...] 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG [...] 03:12:54 DEBUG 03:12:54 DEBUG [...] 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG [...] 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG 1713665574, 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG [...] 03:12:54 DEBUG 03:12:54 DEBUG "pod": "ds-cts-2" 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG [...] 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG [...] 03:12:54 DEBUG "3326.9333333333334" 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:12:54 DEBUG }, 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG [...] 03:12:54 DEBUG 03:12:54 DEBUG "pod": "ds-idrepo-2" 03:12:54 DEBUG 03:12:54 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG "instance": "10.142.0.43:9100" 03:12:54 DEBUG "pod": "am-767f6b67b5-vbltz" 03:12:54 DEBUG "pod": "am-767f6b67b5-vbltz" 03:12:54 DEBUG "pod": "am-767f6b67b5-vbltz" 03:12:54 DEBUG 03:12:54 DEBUG ] 03:12:54 DEBUG "pod": "ds-idrepo-2" 03:12:54 DEBUG "pod": "am-767f6b67b5-vbltz" 03:12:54 DEBUG 03:12:54 DEBUG }, 03:12:54 DEBUG "value": [ 03:12:54 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 03:12:54 DEBUG "instance": "10.142.0.43:9100" 03:12:54 DEBUG "pod": "ds-cts-2" 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG }, 03:12:54 DEBUG "pod": "ds-cts-0" 03:12:54 DEBUG }, 03:12:54 DEBUG 03:12:54 DEBUG [...] 03:12:54 DEBUG 03:12:54 DEBUG }, 03:12:54 DEBUG }, 03:12:54 DEBUG }, 03:12:54 DEBUG }, 03:12:54 DEBUG 03:12:54 DEBUG } 03:12:54 DEBUG }, 03:12:54 DEBUG }, 03:12:54 DEBUG 03:12:54 DEBUG "value": [ 03:12:54 DEBUG 1713665574, 03:12:54 DEBUG }, 03:12:54 DEBUG }, 03:12:54 DEBUG }, 03:12:54 DEBUG "pod": "ds-cts-2" 03:12:54 DEBUG 03:12:54 DEBUG "pod": "ds-idrepo-2" 03:12:54 DEBUG "value": [ 03:12:54 DEBUG }, 03:12:54 DEBUG "value": [ 03:12:54 DEBUG "instance": "10.142.0.43:9100" 03:12:54 DEBUG 03:12:54 DEBUG 03:12:54 DEBUG "value": [ 03:12:54 DEBUG "value": [ 03:12:54 DEBUG "value": [ 03:12:54 DEBUG "value": [ 03:12:54 DEBUG 03:12:54 DEBUG ] 03:12:54 DEBUG "value": [ 03:12:54 DEBUG "value": [ 03:12:54 DEBUG "pod": "am-767f6b67b5-vbltz" 03:12:54 DEBUG 1713665574, 03:12:54 DEBUG "0" 03:12:54 DEBUG "value": [ 03:12:54 DEBUG "value": [ 03:12:54 DEBUG "value": [ 03:12:54 DEBUG }, 03:12:54 DEBUG 03:12:54 DEBUG }, 03:12:54 DEBUG 1713665574, 03:12:54 DEBUG "value": [ 03:12:54 DEBUG 1713665574, 03:12:54 DEBUG }, 03:12:54 DEBUG 03:12:54 DEBUG "pod": "ds-cts-0" 03:12:54 DEBUG 1713665574, 03:12:54 DEBUG 1713665574, 03:12:54 DEBUG 1713665574, 03:12:54 DEBUG 1713665574, 03:12:54 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:12:54 DEBUG } 03:12:54 DEBUG 1713665574, 03:12:54 DEBUG 1713665574, 03:12:54 DEBUG }, 03:12:54 DEBUG "11.74993467436556" 03:12:54 DEBUG ] 03:12:54 DEBUG 1713665574, 03:12:54 DEBUG 1713665574, 03:12:54 DEBUG 1713665574, 03:12:54 DEBUG "value": [ 03:12:54 DEBUG "instance": "10.142.0.43:9100" 03:12:54 DEBUG "value": [ 03:12:54 DEBUG "0" 03:12:54 DEBUG 1713665574, 03:12:54 DEBUG "0" 03:12:54 DEBUG "value": [ 03:12:54 DEBUG 1713665574, 03:12:54 DEBUG }, 03:12:54 DEBUG "4360647803904" 03:12:54 DEBUG ] 03:12:54 DEBUG "600755.5" 03:12:54 DEBUG "3332.133333333333" 03:12:54 DEBUG }, 03:12:54 DEBUG } 03:12:54 DEBUG "0" 03:12:54 DEBUG "3323.733333333333" 03:12:54 DEBUG "value": [ 03:12:54 DEBUG ] 03:12:54 DEBUG } 03:12:54 DEBUG "0" 03:12:54 DEBUG "100081655" 03:12:54 DEBUG ] 03:12:54 DEBUG 1713665574, 03:12:54 DEBUG }, 03:12:54 DEBUG 1713665574, 03:12:54 DEBUG ] 03:12:54 DEBUG "0" 03:12:54 DEBUG ] 03:12:54 DEBUG "100077947.76666667" 03:12:54 DEBUG "value": [ 03:12:54 DEBUG 03:12:54 DEBUG "3327.9666666666667" 03:12:54 DEBUG } 03:12:54 DEBUG ] 03:12:54 DEBUG ] 03:12:54 DEBUG "value": [ 03:12:54 DEBUG ] 03:12:54 DEBUG } 03:12:54 DEBUG ] 03:12:54 DEBUG } 03:12:54 DEBUG ] 03:12:54 DEBUG ] 03:12:54 DEBUG "0" 03:12:54 DEBUG } 03:12:54 DEBUG "0" 03:12:54 DEBUG "value": [ 03:12:54 DEBUG "0" 03:12:54 DEBUG } 03:12:54 DEBUG ] 03:12:54 DEBUG } 03:12:54 DEBUG ] 03:12:54 DEBUG 1713665574, 03:12:54 DEBUG "pod": "am-767f6b67b5-vbltz" 03:12:54 DEBUG ] 03:12:54 DEBUG ] 03:12:54 DEBUG } 03:12:54 DEBUG } 03:12:54 DEBUG 1713665574, 03:12:54 DEBUG "1015922688" 03:12:54 DEBUG ] 03:12:54 DEBUG } 03:12:54 DEBUG ] 03:12:54 DEBUG } 03:12:54 DEBUG } 03:12:54 DEBUG ] 03:12:54 DEBUG ] 03:12:54 DEBUG ] 03:12:54 DEBUG 1713665574, 03:12:54 DEBUG ] 03:12:54 DEBUG ] 03:12:54 DEBUG } 03:12:54 DEBUG ] 03:12:54 DEBUG } 03:12:54 DEBUG "0" 03:12:54 DEBUG }, 03:12:54 DEBUG } 03:12:54 DEBUG } 03:12:54 DEBUG ] 03:12:54 DEBUG ] 03:12:54 DEBUG ] 03:12:54 DEBUG ] 03:12:54 DEBUG 1713665574, 03:12:54 DEBUG } 03:12:54 DEBUG } 03:12:54 DEBUG ] 03:12:54 DEBUG } 03:12:54 DEBUG } 03:12:54 DEBUG } 03:12:54 DEBUG } 03:12:54 DEBUG "0.07833333333333789" 03:12:54 DEBUG } 03:12:54 DEBUG } 03:12:54 DEBUG } 03:12:54 DEBUG } 03:12:54 DEBUG ] 03:12:54 DEBUG ] 03:12:54 DEBUG "value": [ 03:12:54 DEBUG ] 03:12:54 DEBUG } 03:12:54 DEBUG } 03:12:54 DEBUG } 03:12:54 DEBUG } 03:12:54 DEBUG } 03:12:54 DEBUG "0" 03:12:54 DEBUG } 03:12:54 DEBUG } 03:12:54 DEBUG } 03:12:54 DEBUG ] 03:12:54 DEBUG } 03:12:54 DEBUG ] 03:12:54 DEBUG ] 03:12:54 DEBUG ] 03:12:54 DEBUG } 03:12:54 DEBUG ] 03:12:54 DEBUG } 03:12:54 DEBUG } 03:12:54 DEBUG 1713665574, 03:12:54 DEBUG } 03:12:54 DEBUG } 03:12:54 DEBUG } 03:12:54 DEBUG ] 03:12:54 DEBUG } 03:12:54 DEBUG ] 03:12:54 DEBUG } 03:12:54 DEBUG } 03:12:54 DEBUG } 03:12:54 DEBUG } 03:12:54 DEBUG } 03:12:54 DEBUG } 03:12:54 DEBUG } 03:12:54 DEBUG ] 03:12:54 DEBUG "149.889" 03:12:54 DEBUG } 03:12:54 DEBUG } 03:12:54 DEBUG } 03:12:54 DEBUG } 03:12:54 DEBUG } 03:12:54 DEBUG ] 03:12:54 DEBUG } 03:12:54 DEBUG } 03:12:54 DEBUG } 03:12:54 DEBUG ] 03:12:54 DEBUG } 03:12:54 DEBUG ] 03:12:54 DEBUG } 03:12:54 DEBUG } 03:12:54 DEBUG } 03:12:54 DEBUG } 03:12:54 DEBUG } 03:12:54 DEBUG ] 03:12:54 DEBUG } 03:12:54 DEBUG } 03:12:54 DEBUG } 03:13:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713665632" 03:13:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665632" 03:13:52 ERROR [http_cmd]: ERROR 03:13:52 INFO [http_cmd]: http status code OK 03:13:52 ERROR --- status code --- 03:13:52 DEBUG --- status code --- 03:13:52 ERROR http status code is 400 (expected [200, 401, 403, 404]) 03:13:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:13:52 ERROR --- http response --- 03:13:52 DEBUG --- http response --- 03:13:52 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 03:13:52 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 03:13:52 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 03:13:53 INFO 03:13:53 INFO [loop_until]: kubectl --namespace=xlou top pods 03:13:53 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:13:53 INFO [loop_until]: OK (rc = 0) 03:13:53 DEBUG --- stdout --- 03:13:53 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 22807m 8795Mi am-767f6b67b5-qcqz2 22539m 9125Mi am-767f6b67b5-vbltz 23960m 8818Mi ds-cts-0 19735m 12417Mi ds-cts-1 19939m 12359Mi ds-cts-2 20000m 12471Mi ds-idrepo-0 9633m 3918Mi ds-idrepo-1 9280m 3946Mi ds-idrepo-2 9537m 3916Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1179Mi idm-b85f9d978-wnhxb 3m 1239Mi lodemon-54c87f8cbb-jzrjs 14m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-68c5d5b884-p97w5 11706m 877Mi 03:13:53 DEBUG --- stderr --- 03:13:53 DEBUG 03:13:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665634" 03:13:54 INFO [http_cmd]: http status code OK 03:13:54 DEBUG --- status code --- 03:13:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:13:54 DEBUG --- http response --- 03:13:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:13:54 DEBUG ----- output ----- 03:13:54 DEBUG { 03:13:54 DEBUG "status": "success", 03:13:54 DEBUG "data": { 03:13:54 DEBUG "resultType": "vector", 03:13:54 DEBUG "result": [ 03:13:54 DEBUG { 03:13:54 DEBUG "metric": { 03:13:54 DEBUG "instance": "10.142.0.107:9100" 03:13:54 DEBUG }, 03:13:54 DEBUG "value": [ 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG [...] 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG "instance": "10.142.0.43:9100" 03:13:54 DEBUG }, 03:13:54 DEBUG "value": [ 03:13:54 DEBUG 1713665634, 03:13:54 DEBUG "0.37006666666663174" 03:13:54 DEBUG ] 03:13:54 DEBUG } 03:13:54 DEBUG ] 03:13:54 DEBUG } 03:13:54 DEBUG } 03:13:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665634" 03:13:54 INFO [http_cmd]: http status code OK 03:13:54 DEBUG --- status code --- 03:13:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:13:54 DEBUG --- http response --- 03:13:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:13:54 DEBUG ----- output ----- 03:13:54 DEBUG { 03:13:54 DEBUG "status": "success", 03:13:54 DEBUG "data": { 03:13:54 DEBUG "resultType": "vector", 03:13:54 DEBUG "result": [ 03:13:54 DEBUG { 03:13:54 DEBUG "metric": { 03:13:54 DEBUG "pod": "ds-cts-0" 03:13:54 DEBUG }, 03:13:54 DEBUG "value": [ 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG [...] 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG "pod": "ds-cts-2" 03:13:54 DEBUG }, 03:13:54 DEBUG "value": [ 03:13:54 DEBUG 1713665634, 03:13:54 DEBUG "0.5709999999999998" 03:13:54 DEBUG ] 03:13:54 DEBUG } 03:13:54 DEBUG ] 03:13:54 DEBUG } 03:13:54 DEBUG } 03:13:54 INFO 03:13:54 INFO [loop_until]: kubectl --namespace=xlou top node 03:13:54 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:13:54 INFO [loop_until]: OK (rc = 0) 03:13:54 DEBUG --- stdout --- 03:13:54 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 6288Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 9947m 33% 7749Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 22068m 73% 11604Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 84m 0% 3363Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 20075m 67% 15739Mi 14% gke-xlou-cdm-default-pool-1b0769c8-wpwl 153m 0% 5281Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 22559m 75% 12056Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 19764m 66% 3200Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 24686m 82% 11096Mi 9% gke-xlou-cdm-ds-0f601751-j488 10387m 34% 7364Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 19955m 66% 17488Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 10120m 33% 8005Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20170m 67% 15212Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 12026m 75% 11227Mi 19% 03:13:54 DEBUG --- stderr --- 03:13:54 DEBUG 03:13:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665634" 03:13:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713665634" 03:13:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665634" 03:13:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665634" 03:13:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665634" 03:13:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665634" 03:13:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665634" 03:13:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665634" 03:13:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713665634" 03:13:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665634" 03:13:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665634" 03:13:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665634" 03:13:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665634" 03:13:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665634" 03:13:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665634" 03:13:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713665634" 03:13:54 INFO [http_cmd]: http status code OK 03:13:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665634" 03:13:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665634" 03:13:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665634" 03:13:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665634" 03:13:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665634" 03:13:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665634" 03:13:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713665634" 03:13:54 INFO [http_cmd]: http status code OK 03:13:54 INFO [http_cmd]: http status code OK 03:13:54 INFO [http_cmd]: http status code OK 03:13:54 INFO [http_cmd]: http status code OK 03:13:54 INFO [http_cmd]: http status code OK 03:13:54 INFO [http_cmd]: http status code OK 03:13:54 INFO [http_cmd]: http status code OK 03:13:54 DEBUG --- status code --- 03:13:54 INFO [http_cmd]: http status code OK 03:13:54 INFO [http_cmd]: http status code OK 03:13:54 INFO [http_cmd]: http status code OK 03:13:54 INFO [http_cmd]: http status code OK 03:13:54 INFO [http_cmd]: http status code OK 03:13:54 INFO [http_cmd]: http status code OK 03:13:54 DEBUG --- status code --- 03:13:54 DEBUG --- status code --- 03:13:54 DEBUG --- status code --- 03:13:54 INFO [http_cmd]: http status code OK 03:13:54 DEBUG --- status code --- 03:13:54 INFO [http_cmd]: http status code OK 03:13:54 INFO [http_cmd]: http status code OK 03:13:54 INFO [http_cmd]: http status code OK 03:13:54 INFO [http_cmd]: http status code OK 03:13:54 DEBUG --- status code --- 03:13:54 DEBUG --- status code --- 03:13:54 DEBUG --- status code --- 03:13:54 INFO [http_cmd]: http status code OK 03:13:54 INFO [http_cmd]: http status code OK 03:13:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:13:54 INFO [http_cmd]: http status code OK 03:13:54 DEBUG --- status code --- 03:13:54 DEBUG --- status code --- 03:13:54 DEBUG --- status code --- 03:13:54 DEBUG --- status code --- 03:13:54 DEBUG --- status code --- 03:13:54 INFO [http_cmd]: http status code OK 03:13:54 DEBUG --- status code --- 03:13:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:13:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:13:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:13:54 DEBUG --- status code --- 03:13:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:13:54 DEBUG --- status code --- 03:13:54 DEBUG --- status code --- 03:13:54 DEBUG --- status code --- 03:13:54 DEBUG --- status code --- 03:13:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:13:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:13:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:13:54 DEBUG --- status code --- 03:13:54 DEBUG --- status code --- 03:13:54 DEBUG --- http response --- 03:13:54 DEBUG --- status code --- 03:13:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:13:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:13:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:13:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:13:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:13:54 DEBUG --- status code --- 03:13:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:13:54 DEBUG --- http response --- 03:13:54 DEBUG --- http response --- 03:13:54 DEBUG --- http response --- 03:13:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:13:54 DEBUG --- http response --- 03:13:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:13:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:13:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:13:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:13:54 DEBUG --- http response --- 03:13:54 DEBUG --- http response --- 03:13:54 DEBUG --- http response --- 03:13:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:13:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:13:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:13:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:13:54 DEBUG --- http response --- 03:13:54 DEBUG --- http response --- 03:13:54 DEBUG --- http response --- 03:13:54 DEBUG --- http response --- 03:13:54 DEBUG --- http response --- 03:13:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:13:54 DEBUG --- http response --- 03:13:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:13:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:13:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:13:54 DEBUG --- http response --- 03:13:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 03:13:54 DEBUG --- http response --- 03:13:54 DEBUG --- http response --- 03:13:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:13:54 DEBUG --- http response --- 03:13:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:13:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:13:54 DEBUG --- http response --- 03:13:54 DEBUG --- http response --- 03:13:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:13:54 DEBUG --- http response --- 03:13:54 DEBUG ----- output ----- 03:13:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:13:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:13:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:13:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:13:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 03:13:54 DEBUG --- http response --- 03:13:54 DEBUG ----- output ----- 03:13:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:13:54 DEBUG ----- output ----- 03:13:54 DEBUG ----- output ----- 03:13:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:13:54 DEBUG ----- output ----- 03:13:54 DEBUG --- http response --- 03:13:54 DEBUG ----- output ----- 03:13:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:13:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:13:54 DEBUG ----- output ----- 03:13:54 DEBUG ----- output ----- 03:13:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:13:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 03:13:54 DEBUG ----- output ----- 03:13:54 DEBUG { 03:13:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:13:54 DEBUG ----- output ----- 03:13:54 DEBUG ----- output ----- 03:13:54 DEBUG ----- output ----- 03:13:54 DEBUG ----- output ----- 03:13:54 DEBUG ----- output ----- 03:13:54 DEBUG { 03:13:54 DEBUG { 03:13:54 DEBUG ----- output ----- 03:13:54 DEBUG { 03:13:54 DEBUG { 03:13:54 DEBUG ----- output ----- 03:13:54 DEBUG { 03:13:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 03:13:54 DEBUG { 03:13:54 DEBUG ----- output ----- 03:13:54 DEBUG ----- output ----- 03:13:54 DEBUG { 03:13:54 DEBUG { 03:13:54 DEBUG ----- output ----- 03:13:54 DEBUG ----- output ----- 03:13:54 DEBUG { 03:13:54 DEBUG "status": "success", 03:13:54 DEBUG ----- output ----- 03:13:54 DEBUG { 03:13:54 DEBUG { 03:13:54 DEBUG { 03:13:54 DEBUG { 03:13:54 DEBUG "status": "success", 03:13:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:13:54 DEBUG "status": "success", 03:13:54 DEBUG { 03:13:54 DEBUG "status": "success", 03:13:54 DEBUG "status": "success", 03:13:54 DEBUG { 03:13:54 DEBUG "status": "success", 03:13:54 DEBUG ----- output ----- 03:13:54 DEBUG "status": "success", 03:13:54 DEBUG { 03:13:54 DEBUG { 03:13:54 DEBUG "status": "success", 03:13:54 DEBUG "status": "success", 03:13:54 DEBUG { 03:13:54 DEBUG { 03:13:54 DEBUG "status": "success", 03:13:54 DEBUG "data": { 03:13:54 DEBUG { 03:13:54 DEBUG "status": "success", 03:13:54 DEBUG "status": "success", 03:13:54 DEBUG "status": "success", 03:13:54 DEBUG "status": "success", 03:13:54 DEBUG "data": { 03:13:54 DEBUG ----- output ----- 03:13:54 DEBUG "data": { 03:13:54 DEBUG "status": "success", 03:13:54 DEBUG "data": { 03:13:54 DEBUG "data": { 03:13:54 DEBUG "data": { 03:13:54 DEBUG "status": "success", 03:13:54 DEBUG { 03:13:54 DEBUG "data": { 03:13:54 DEBUG "status": "success", 03:13:54 DEBUG "status": "success", 03:13:54 DEBUG "data": { 03:13:54 DEBUG "data": { 03:13:54 DEBUG "status": "success", 03:13:54 DEBUG "status": "success", 03:13:54 DEBUG "resultType": "vector", 03:13:54 DEBUG "data": { 03:13:54 DEBUG "status": "success", 03:13:54 DEBUG "data": { 03:13:54 DEBUG "data": { 03:13:54 DEBUG "data": { 03:13:54 DEBUG "data": { 03:13:54 DEBUG "resultType": "vector", 03:13:54 DEBUG { 03:13:54 DEBUG "resultType": "vector", 03:13:54 DEBUG "data": { 03:13:54 DEBUG "resultType": "vector", 03:13:54 DEBUG "resultType": "vector", 03:13:54 DEBUG "resultType": "vector", 03:13:54 DEBUG "data": { 03:13:54 DEBUG "status": "success", 03:13:54 DEBUG "resultType": "vector", 03:13:54 DEBUG "data": { 03:13:54 DEBUG "data": { 03:13:54 DEBUG "resultType": "vector", 03:13:54 DEBUG "resultType": "vector", 03:13:54 DEBUG "data": { 03:13:54 DEBUG "data": { 03:13:54 DEBUG "result": [ 03:13:54 DEBUG "resultType": "vector", 03:13:54 DEBUG "data": { 03:13:54 DEBUG "resultType": "vector", 03:13:54 DEBUG "resultType": "vector", 03:13:54 DEBUG "resultType": "vector", 03:13:54 DEBUG "resultType": "vector", 03:13:54 DEBUG "result": [ 03:13:54 DEBUG "status": "success", 03:13:54 DEBUG "result": [ 03:13:54 DEBUG "resultType": "vector", 03:13:54 DEBUG "result": [ 03:13:54 DEBUG "result": [ 03:13:54 DEBUG "result": [ 03:13:54 DEBUG "resultType": "vector", 03:13:54 DEBUG "data": { 03:13:54 DEBUG "result": [ 03:13:54 DEBUG "resultType": "vector", 03:13:54 DEBUG "resultType": "vector", 03:13:54 DEBUG "result": [ 03:13:54 DEBUG "result": [ 03:13:54 DEBUG "resultType": "vector", 03:13:54 DEBUG "resultType": "vector", 03:13:54 DEBUG { 03:13:54 DEBUG "result": [ 03:13:54 DEBUG "resultType": "vector", 03:13:54 DEBUG "result": [ 03:13:54 DEBUG "result": [ 03:13:54 DEBUG "result": [ 03:13:54 DEBUG "result": [ 03:13:54 DEBUG { 03:13:54 DEBUG "data": { 03:13:54 DEBUG { 03:13:54 DEBUG "result": [ 03:13:54 DEBUG { 03:13:54 DEBUG { 03:13:54 DEBUG { 03:13:54 DEBUG "result": [ 03:13:54 DEBUG "resultType": "vector", 03:13:54 DEBUG { 03:13:54 DEBUG "result": [ 03:13:54 DEBUG "result": [ 03:13:54 DEBUG { 03:13:54 DEBUG { 03:13:54 DEBUG "result": [ 03:13:54 DEBUG "result": [ 03:13:54 DEBUG "metric": { 03:13:54 DEBUG { 03:13:54 DEBUG "result": [ 03:13:54 DEBUG { 03:13:54 DEBUG { 03:13:54 DEBUG { 03:13:54 DEBUG { 03:13:54 DEBUG "metric": { 03:13:54 DEBUG "resultType": "vector", 03:13:54 DEBUG "metric": { 03:13:54 DEBUG { 03:13:54 DEBUG "metric": { 03:13:54 DEBUG "metric": { 03:13:54 DEBUG "metric": { 03:13:54 DEBUG { 03:13:54 DEBUG "result": [ 03:13:54 DEBUG "metric": { 03:13:54 DEBUG "pod": "ds-idrepo-0" 03:13:54 DEBUG { 03:13:54 DEBUG "metric": { 03:13:54 DEBUG "metric": { 03:13:54 DEBUG { 03:13:54 DEBUG { 03:13:54 DEBUG "instance": "10.142.0.107:9100" 03:13:54 DEBUG "metric": { 03:13:54 DEBUG { 03:13:54 DEBUG "metric": { 03:13:54 DEBUG "metric": { 03:13:54 DEBUG "metric": { 03:13:54 DEBUG "metric": { 03:13:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:13:54 DEBUG "result": [ 03:13:54 DEBUG "instance": "10.142.0.107:9100" 03:13:54 DEBUG "metric": { 03:13:54 DEBUG "pod": "am-767f6b67b5-9zw22" 03:13:54 DEBUG "pod": "ds-idrepo-0" 03:13:54 DEBUG "pod": "ds-idrepo-0" 03:13:54 DEBUG "metric": { 03:13:54 DEBUG { 03:13:54 DEBUG }, 03:13:54 DEBUG "metric": { 03:13:54 DEBUG { 03:13:54 DEBUG "pod": "ds-cts-0" 03:13:54 DEBUG "pod": "am-767f6b67b5-9zw22" 03:13:54 DEBUG "metric": { 03:13:54 DEBUG "metric": { 03:13:54 DEBUG }, 03:13:54 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:13:54 DEBUG "metric": { 03:13:54 DEBUG "pod": "am-767f6b67b5-9zw22" 03:13:54 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:13:54 DEBUG "instance": "10.142.0.107:9100" 03:13:54 DEBUG "pod": "ds-idrepo-0" 03:13:54 DEBUG }, 03:13:54 DEBUG { 03:13:54 DEBUG }, 03:13:54 DEBUG "pod": "am-767f6b67b5-9zw22" 03:13:54 DEBUG }, 03:13:54 DEBUG }, 03:13:54 DEBUG }, 03:13:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:13:54 DEBUG "metric": { 03:13:54 DEBUG "value": [ 03:13:54 DEBUG "pod": "ds-cts-0" 03:13:54 DEBUG "metric": { 03:13:54 DEBUG }, 03:13:54 DEBUG }, 03:13:54 DEBUG "pod": "am-767f6b67b5-9zw22" 03:13:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:13:54 DEBUG "value": [ 03:13:54 DEBUG }, 03:13:54 DEBUG "pod": "am-767f6b67b5-9zw22" 03:13:54 DEBUG }, 03:13:54 DEBUG }, 03:13:54 DEBUG }, 03:13:54 DEBUG }, 03:13:54 DEBUG "value": [ 03:13:54 DEBUG "metric": { 03:13:54 DEBUG "value": [ 03:13:54 DEBUG }, 03:13:54 DEBUG "value": [ 03:13:54 DEBUG "value": [ 03:13:54 DEBUG "value": [ 03:13:54 DEBUG }, 03:13:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:13:54 DEBUG 03:13:54 DEBUG }, 03:13:54 DEBUG "pod": "am-767f6b67b5-9zw22" 03:13:54 DEBUG "value": [ 03:13:54 DEBUG "value": [ 03:13:54 DEBUG 03:13:54 DEBUG }, 03:13:54 DEBUG 03:13:54 DEBUG "value": [ 03:13:54 DEBUG }, 03:13:54 DEBUG "value": [ 03:13:54 DEBUG "value": [ 03:13:54 DEBUG "value": [ 03:13:54 DEBUG 03:13:54 DEBUG "value": [ 03:13:54 DEBUG "instance": "10.142.0.107:9100" 03:13:54 DEBUG 03:13:54 DEBUG "value": [ 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG "value": [ 03:13:54 DEBUG }, 03:13:54 DEBUG 03:13:54 DEBUG "value": [ 03:13:54 DEBUG }, 03:13:54 DEBUG 03:13:54 DEBUG }, 03:13:54 DEBUG 03:13:54 DEBUG "value": [ 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG "value": [ 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG }, 03:13:54 DEBUG "value": [ 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG "value": [ 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG "value": [ 03:13:54 DEBUG 03:13:54 DEBUG "value": [ 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG [...] 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG [...] 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG [...] 03:13:54 DEBUG 03:13:54 DEBUG [...] 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG [...] 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG [...] 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG [...] 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG [...] 03:13:54 DEBUG 03:13:54 DEBUG [...] 03:13:54 DEBUG [...] 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG [...] 03:13:54 DEBUG [...] 03:13:54 DEBUG 03:13:54 DEBUG [...] 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG [...] 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG [...] 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG [...] 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG [...] 03:13:54 DEBUG 03:13:54 DEBUG [...] 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG [...] 03:13:54 DEBUG [...] 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG [...] 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG [...] 03:13:54 DEBUG 03:13:54 DEBUG "pod": "am-767f6b67b5-vbltz" 03:13:54 DEBUG "pod": "ds-idrepo-2" 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG "pod": "ds-cts-2" 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG "pod": "ds-cts-2" 03:13:54 DEBUG "pod": "am-767f6b67b5-vbltz" 03:13:54 DEBUG [...] 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:13:54 DEBUG "pod": "ds-idrepo-2" 03:13:54 DEBUG 03:13:54 DEBUG "instance": "10.142.0.43:9100" 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG "pod": "am-767f6b67b5-vbltz" 03:13:54 DEBUG }, 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG }, 03:13:54 DEBUG 03:13:54 DEBUG "pod": "ds-cts-2" 03:13:54 DEBUG }, 03:13:54 DEBUG }, 03:13:54 DEBUG "value": [ 03:13:54 DEBUG 1713665634, 03:13:54 DEBUG "600148.5" 03:13:54 DEBUG ] 03:13:54 DEBUG "instance": "10.142.0.43:9100" 03:13:54 DEBUG }, 03:13:54 DEBUG 03:13:54 DEBUG "instance": "10.142.0.43:9100" 03:13:54 DEBUG }, 03:13:54 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:13:54 DEBUG }, 03:13:54 DEBUG 03:13:54 DEBUG }, 03:13:54 DEBUG }, 03:13:54 DEBUG "value": [ 03:13:54 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG "value": [ 03:13:54 DEBUG 03:13:54 DEBUG }, 03:13:54 DEBUG "value": [ 03:13:54 DEBUG 03:13:54 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:13:54 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:13:54 DEBUG 03:13:54 DEBUG } 03:13:54 DEBUG "pod": "am-767f6b67b5-vbltz" 03:13:54 DEBUG "value": [ 03:13:54 DEBUG }, 03:13:54 DEBUG }, 03:13:54 DEBUG "value": [ 03:13:54 DEBUG }, 03:13:54 DEBUG "value": [ 03:13:54 DEBUG 03:13:54 DEBUG "value": [ 03:13:54 DEBUG "value": [ 03:13:54 DEBUG 1713665634, 03:13:54 DEBUG }, 03:13:54 DEBUG "pod": "ds-idrepo-2" 03:13:54 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:13:54 DEBUG 1713665634, 03:13:54 DEBUG 03:13:54 DEBUG "value": [ 03:13:54 DEBUG 1713665634, 03:13:54 DEBUG 03:13:54 DEBUG 03:13:54 DEBUG }, 03:13:54 DEBUG "pod": "am-767f6b67b5-vbltz" 03:13:54 DEBUG ] 03:13:54 DEBUG }, 03:13:54 DEBUG 1713665634, 03:13:54 DEBUG "value": [ 03:13:54 DEBUG "value": [ 03:13:54 DEBUG 1713665634, 03:13:54 DEBUG "value": [ 03:13:54 DEBUG 1713665634, 03:13:54 DEBUG "instance": "10.142.0.43:9100" 03:13:54 DEBUG 1713665634, 03:13:54 DEBUG 1713665634, 03:13:54 DEBUG "0" 03:13:54 DEBUG ] 03:13:54 DEBUG }, 03:13:54 DEBUG }, 03:13:54 DEBUG "value": [ 03:13:54 DEBUG "pod": "am-767f6b67b5-vbltz" 03:13:54 DEBUG 1713665634, 03:13:54 DEBUG "0" 03:13:54 DEBUG "pod": "am-767f6b67b5-vbltz" 03:13:54 DEBUG }, 03:13:54 DEBUG }, 03:13:54 DEBUG }, 03:13:54 DEBUG } 03:13:54 DEBUG "value": [ 03:13:54 DEBUG "100416588.03333333" 03:13:54 DEBUG 1713665634, 03:13:54 DEBUG 1713665634, 03:13:54 DEBUG "1017753600" 03:13:54 DEBUG ] 03:13:54 DEBUG "4365810613248" 03:13:54 DEBUG ] 03:13:54 DEBUG "3340.6666666666665" 03:13:54 DEBUG "3340.8333333333335" 03:13:54 DEBUG "value": [ 03:13:54 DEBUG } 03:13:54 DEBUG ] 03:13:54 DEBUG } 03:13:54 DEBUG } 03:13:54 DEBUG }, 03:13:54 DEBUG "0" 03:13:54 DEBUG ] 03:13:54 DEBUG "value": [ 03:13:54 DEBUG "value": [ 03:13:54 DEBUG "value": [ 03:13:54 DEBUG "value": [ 03:13:54 DEBUG } 03:13:54 DEBUG 1713665634, 03:13:54 DEBUG ] 03:13:54 DEBUG "100414015.63333333" 03:13:54 DEBUG "0" 03:13:54 DEBUG } 03:13:54 DEBUG 1713665634, 03:13:54 DEBUG } 03:13:54 DEBUG }, 03:13:54 DEBUG ] 03:13:54 DEBUG ] 03:13:54 DEBUG 1713665634, 03:13:54 DEBUG "value": [ 03:13:54 DEBUG 1713665634, 03:13:54 DEBUG "0" 03:13:54 DEBUG "value": [ 03:13:54 DEBUG ] 03:13:54 DEBUG } 03:13:54 DEBUG 1713665634, 03:13:54 DEBUG 1713665634, 03:13:54 DEBUG 1713665634, 03:13:54 DEBUG "212672.28172333716" 03:13:54 DEBUG "3347.7" 03:13:54 DEBUG } 03:13:54 DEBUG ] 03:13:54 DEBUG ] 03:13:54 DEBUG } 03:13:54 DEBUG "0" 03:13:54 DEBUG ] 03:13:54 DEBUG "value": [ 03:13:54 DEBUG } 03:13:54 DEBUG } 03:13:54 DEBUG "0" 03:13:54 DEBUG "0" 03:13:54 DEBUG ] 03:13:54 DEBUG ] 03:13:54 DEBUG 1713665634, 03:13:54 DEBUG } 03:13:54 DEBUG ] 03:13:54 DEBUG "149.889" 03:13:54 DEBUG "11.789754239489225" 03:13:54 DEBUG ] 03:13:54 DEBUG } 03:13:54 DEBUG ] 03:13:54 DEBUG } 03:13:54 DEBUG ] 03:13:54 DEBUG ] 03:13:54 DEBUG ] 03:13:54 DEBUG ] 03:13:54 DEBUG } 03:13:54 DEBUG 1713665634, 03:13:54 DEBUG ] 03:13:54 DEBUG ] 03:13:54 DEBUG ] 03:13:54 DEBUG } 03:13:54 DEBUG 1713665634, 03:13:54 DEBUG } 03:13:54 DEBUG "3344.7" 03:13:54 DEBUG ] 03:13:54 DEBUG } 03:13:54 DEBUG ] 03:13:54 DEBUG ] 03:13:54 DEBUG ] 03:13:54 DEBUG } 03:13:54 DEBUG } 03:13:54 DEBUG } 03:13:54 DEBUG 1713665634, 03:13:54 DEBUG } 03:13:54 DEBUG } 03:13:54 DEBUG } 03:13:54 DEBUG } 03:13:54 DEBUG "0.07200000000000842" 03:13:54 DEBUG } 03:13:54 DEBUG } 03:13:54 DEBUG } 03:13:54 DEBUG ] 03:13:54 DEBUG "11.748335242175285" 03:13:54 DEBUG ] 03:13:54 DEBUG } 03:13:54 DEBUG ] 03:13:54 DEBUG } 03:13:54 DEBUG } 03:13:54 DEBUG } 03:13:54 DEBUG ] 03:13:54 DEBUG } 03:13:54 DEBUG "0" 03:13:54 DEBUG } 03:13:54 DEBUG ] 03:13:54 DEBUG ] 03:13:54 DEBUG ] 03:13:54 DEBUG } 03:13:54 DEBUG } 03:13:54 DEBUG ] 03:13:54 DEBUG } 03:13:54 DEBUG ] 03:13:54 DEBUG } 03:13:54 DEBUG ] 03:13:54 DEBUG } 03:13:54 DEBUG ] 03:13:54 DEBUG } 03:13:54 DEBUG } 03:13:54 DEBUG } 03:13:54 DEBUG ] 03:13:54 DEBUG } 03:13:54 DEBUG } 03:13:54 DEBUG } 03:13:54 DEBUG } 03:13:54 DEBUG } 03:13:54 DEBUG } 03:13:54 DEBUG ] 03:13:54 DEBUG } 03:13:54 DEBUG } 03:13:54 DEBUG } 03:13:54 DEBUG } 03:13:54 DEBUG ] 03:13:54 DEBUG } 03:13:54 DEBUG ] 03:13:54 DEBUG } 03:13:54 DEBUG ] 03:13:54 DEBUG } 03:13:54 DEBUG } 03:13:54 DEBUG } 03:13:54 DEBUG } 03:13:54 DEBUG } 03:13:54 DEBUG } 03:13:54 DEBUG } 03:13:54 DEBUG } 03:13:54 DEBUG } 03:13:54 DEBUG } 03:13:54 DEBUG } 03:13:54 DEBUG } 03:13:54 DEBUG } 03:14:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665692" 03:14:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713665692" 03:14:52 ERROR [http_cmd]: ERROR 03:14:52 INFO [http_cmd]: http status code OK 03:14:52 ERROR --- status code --- 03:14:52 DEBUG --- status code --- 03:14:52 ERROR http status code is 400 (expected [200, 401, 403, 404]) 03:14:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:14:52 ERROR --- http response --- 03:14:52 DEBUG --- http response --- 03:14:52 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 03:14:52 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 03:14:52 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 03:14:53 INFO 03:14:53 INFO [loop_until]: kubectl --namespace=xlou top pods 03:14:53 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:14:53 INFO [loop_until]: OK (rc = 0) 03:14:53 DEBUG --- stdout --- 03:14:53 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 22452m 8794Mi am-767f6b67b5-qcqz2 22369m 9126Mi am-767f6b67b5-vbltz 23176m 8844Mi ds-cts-0 20604m 12440Mi ds-cts-1 19610m 12406Mi ds-cts-2 21062m 12493Mi ds-idrepo-0 9781m 3918Mi ds-idrepo-1 9441m 3947Mi ds-idrepo-2 9765m 3915Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 3m 1179Mi idm-b85f9d978-wnhxb 3m 1239Mi lodemon-54c87f8cbb-jzrjs 12m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-68c5d5b884-p97w5 11418m 880Mi 03:14:53 DEBUG --- stderr --- 03:14:53 DEBUG 03:14:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665694" 03:14:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665694" 03:14:54 INFO [http_cmd]: http status code OK 03:14:54 DEBUG --- status code --- 03:14:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:14:54 DEBUG --- http response --- 03:14:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:14:54 DEBUG ----- output ----- 03:14:54 DEBUG { 03:14:54 DEBUG "status": "success", 03:14:54 DEBUG "data": { 03:14:54 DEBUG "resultType": "vector", 03:14:54 DEBUG "result": [ 03:14:54 DEBUG { 03:14:54 DEBUG "metric": { 03:14:54 DEBUG "instance": "10.142.0.107:9100" 03:14:54 DEBUG }, 03:14:54 DEBUG "value": [ 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG [...] 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG "instance": "10.142.0.43:9100" 03:14:54 DEBUG }, 03:14:54 DEBUG "value": [ 03:14:54 DEBUG 1713665694, 03:14:54 DEBUG "0.38216666666658816" 03:14:54 DEBUG ] 03:14:54 DEBUG } 03:14:54 DEBUG ] 03:14:54 DEBUG } 03:14:54 DEBUG } 03:14:54 INFO [http_cmd]: http status code OK 03:14:54 DEBUG --- status code --- 03:14:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:14:54 DEBUG --- http response --- 03:14:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:14:54 DEBUG ----- output ----- 03:14:54 DEBUG { 03:14:54 DEBUG "status": "success", 03:14:54 DEBUG "data": { 03:14:54 DEBUG "resultType": "vector", 03:14:54 DEBUG "result": [ 03:14:54 DEBUG { 03:14:54 DEBUG "metric": { 03:14:54 DEBUG "pod": "ds-cts-0" 03:14:54 DEBUG }, 03:14:54 DEBUG "value": [ 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG [...] 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG "pod": "ds-cts-2" 03:14:54 DEBUG }, 03:14:54 DEBUG "value": [ 03:14:54 DEBUG 1713665694, 03:14:54 DEBUG "0.4618666666666665" 03:14:54 DEBUG ] 03:14:54 DEBUG } 03:14:54 DEBUG ] 03:14:54 DEBUG } 03:14:54 DEBUG } 03:14:54 INFO 03:14:54 INFO [loop_until]: kubectl --namespace=xlou top node 03:14:54 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:14:54 INFO [loop_until]: OK (rc = 0) 03:14:54 DEBUG --- stdout --- 03:14:54 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 65m 0% 6284Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 10012m 33% 7771Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 21974m 73% 11612Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 90m 0% 3364Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 20215m 67% 15751Mi 14% gke-xlou-cdm-default-pool-1b0769c8-wpwl 153m 0% 5274Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 22352m 74% 12049Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 19657m 65% 3176Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 24623m 82% 11092Mi 9% gke-xlou-cdm-ds-0f601751-j488 10385m 34% 7342Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 19741m 66% 17527Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 9795m 32% 8009Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20430m 68% 15240Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 11990m 75% 11230Mi 19% 03:14:54 DEBUG --- stderr --- 03:14:54 DEBUG 03:14:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665694" 03:14:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665694" 03:14:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665694" 03:14:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665694" 03:14:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665694" 03:14:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665694" 03:14:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665694" 03:14:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665694" 03:14:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665694" 03:14:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713665694" 03:14:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713665694" 03:14:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665694" 03:14:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665694" 03:14:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713665694" 03:14:54 INFO [http_cmd]: http status code OK 03:14:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665694" 03:14:54 INFO [http_cmd]: http status code OK 03:14:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665694" 03:14:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665694" 03:14:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665694" 03:14:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665694" 03:14:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665694" 03:14:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665694" 03:14:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665694" 03:14:54 INFO [http_cmd]: http status code OK 03:14:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713665694" 03:14:54 INFO [http_cmd]: http status code OK 03:14:54 INFO [http_cmd]: http status code OK 03:14:54 DEBUG --- status code --- 03:14:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:14:54 DEBUG --- status code --- 03:14:54 INFO [http_cmd]: http status code OK 03:14:54 INFO [http_cmd]: http status code OK 03:14:54 INFO [http_cmd]: http status code OK 03:14:54 INFO [http_cmd]: http status code OK 03:14:54 INFO [http_cmd]: http status code OK 03:14:54 INFO [http_cmd]: http status code OK 03:14:54 DEBUG --- status code --- 03:14:54 INFO [http_cmd]: http status code OK 03:14:54 INFO [http_cmd]: http status code OK 03:14:54 INFO [http_cmd]: http status code OK 03:14:54 DEBUG --- status code --- 03:14:54 DEBUG --- status code --- 03:14:54 INFO [http_cmd]: http status code OK 03:14:54 INFO [http_cmd]: http status code OK 03:14:54 DEBUG --- status code --- 03:14:54 DEBUG --- http response --- 03:14:54 INFO [http_cmd]: http status code OK 03:14:54 DEBUG --- status code --- 03:14:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:14:54 INFO [http_cmd]: http status code OK 03:14:54 INFO [http_cmd]: http status code OK 03:14:54 DEBUG --- status code --- 03:14:54 DEBUG --- status code --- 03:14:54 DEBUG --- status code --- 03:14:54 DEBUG --- status code --- 03:14:54 DEBUG --- status code --- 03:14:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:14:54 INFO [http_cmd]: http status code OK 03:14:54 DEBUG --- status code --- 03:14:54 DEBUG --- status code --- 03:14:54 DEBUG --- status code --- 03:14:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:14:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:14:54 DEBUG --- status code --- 03:14:54 DEBUG --- status code --- 03:14:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:14:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:14:54 INFO [http_cmd]: http status code OK 03:14:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:14:54 DEBUG --- http response --- 03:14:54 DEBUG --- status code --- 03:14:54 DEBUG --- status code --- 03:14:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:14:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:14:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:14:54 INFO [http_cmd]: http status code OK 03:14:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:14:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:14:54 DEBUG --- http response --- 03:14:54 INFO [http_cmd]: http status code OK 03:14:54 DEBUG --- status code --- 03:14:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:14:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:14:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:14:54 DEBUG --- http response --- 03:14:54 DEBUG --- http response --- 03:14:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:14:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:14:54 DEBUG --- http response --- 03:14:54 DEBUG ----- output ----- 03:14:54 DEBUG --- status code --- 03:14:54 DEBUG --- http response --- 03:14:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:14:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:14:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:14:54 DEBUG --- http response --- 03:14:54 DEBUG --- http response --- 03:14:54 DEBUG --- http response --- 03:14:54 DEBUG --- status code --- 03:14:54 DEBUG --- http response --- 03:14:54 DEBUG --- http response --- 03:14:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:14:54 DEBUG --- status code --- 03:14:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:14:54 DEBUG --- http response --- 03:14:54 DEBUG --- http response --- 03:14:54 DEBUG --- http response --- 03:14:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:14:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:14:54 DEBUG --- http response --- 03:14:54 DEBUG --- http response --- 03:14:54 DEBUG { 03:14:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:14:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:14:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:14:54 DEBUG ----- output ----- 03:14:54 DEBUG --- http response --- 03:14:54 DEBUG --- http response --- 03:14:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:14:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:14:54 DEBUG ----- output ----- 03:14:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:14:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:14:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:14:54 DEBUG ----- output ----- 03:14:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:14:54 DEBUG --- http response --- 03:14:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:14:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:14:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 03:14:54 DEBUG ----- output ----- 03:14:54 DEBUG ----- output ----- 03:14:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:14:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 03:14:54 DEBUG "status": "success", 03:14:54 DEBUG ----- output ----- 03:14:54 DEBUG --- http response --- 03:14:54 DEBUG ----- output ----- 03:14:54 DEBUG { 03:14:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:14:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:14:54 DEBUG ----- output ----- 03:14:54 DEBUG { 03:14:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:14:54 DEBUG --- http response --- 03:14:54 DEBUG ----- output ----- 03:14:54 DEBUG ----- output ----- 03:14:54 DEBUG { 03:14:54 DEBUG "status": "success", 03:14:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:14:54 DEBUG ----- output ----- 03:14:54 DEBUG { 03:14:54 DEBUG ----- output ----- 03:14:54 DEBUG { 03:14:54 DEBUG { 03:14:54 DEBUG ----- output ----- 03:14:54 DEBUG ----- output ----- 03:14:54 DEBUG "data": { 03:14:54 DEBUG { 03:14:54 DEBUG { 03:14:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:14:54 DEBUG "status": "success", 03:14:54 DEBUG ----- output ----- 03:14:54 DEBUG ----- output ----- 03:14:54 DEBUG { 03:14:54 DEBUG "status": "success", 03:14:54 DEBUG ----- output ----- 03:14:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:14:54 DEBUG { 03:14:54 DEBUG { 03:14:54 DEBUG "data": { 03:14:54 DEBUG --- http response --- 03:14:54 DEBUG ----- output ----- 03:14:54 DEBUG ----- output ----- 03:14:54 DEBUG "status": "success", 03:14:54 DEBUG { 03:14:54 DEBUG "status": "success", 03:14:54 DEBUG "status": "success", 03:14:54 DEBUG "data": { 03:14:54 DEBUG { 03:14:54 DEBUG "resultType": "vector", 03:14:54 DEBUG "status": "success", 03:14:54 DEBUG "data": { 03:14:54 DEBUG "status": "success", 03:14:54 DEBUG "data": { 03:14:54 DEBUG { 03:14:54 DEBUG { 03:14:54 DEBUG "status": "success", 03:14:54 DEBUG "data": { 03:14:54 DEBUG "resultType": "vector", 03:14:54 DEBUG ----- output ----- 03:14:54 DEBUG "status": "success", 03:14:54 DEBUG "status": "success", 03:14:54 DEBUG { 03:14:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:14:54 DEBUG { 03:14:54 DEBUG { 03:14:54 DEBUG "data": { 03:14:54 DEBUG "status": "success", 03:14:54 DEBUG "data": { 03:14:54 DEBUG { 03:14:54 DEBUG "resultType": "vector", 03:14:54 DEBUG "status": "success", 03:14:54 DEBUG "result": [ 03:14:54 DEBUG ----- output ----- 03:14:54 DEBUG "resultType": "vector", 03:14:54 DEBUG "data": { 03:14:54 DEBUG "resultType": "vector", 03:14:54 DEBUG "status": "success", 03:14:54 DEBUG "status": "success", 03:14:54 DEBUG "data": { 03:14:54 DEBUG "result": [ 03:14:54 DEBUG { 03:14:54 DEBUG "data": { 03:14:54 DEBUG "resultType": "vector", 03:14:54 DEBUG "data": { 03:14:54 DEBUG "status": "success", 03:14:54 DEBUG ----- output ----- 03:14:54 DEBUG "status": "success", 03:14:54 DEBUG "status": "success", 03:14:54 DEBUG "resultType": "vector", 03:14:54 DEBUG "data": { 03:14:54 DEBUG "resultType": "vector", 03:14:54 DEBUG "status": "success", 03:14:54 DEBUG "result": [ 03:14:54 DEBUG "data": { 03:14:54 DEBUG { 03:14:54 DEBUG { 03:14:54 DEBUG "result": [ 03:14:54 DEBUG "resultType": "vector", 03:14:54 DEBUG "result": [ 03:14:54 DEBUG "data": { 03:14:54 DEBUG "data": { 03:14:54 DEBUG "resultType": "vector", 03:14:54 DEBUG { 03:14:54 DEBUG "status": "success", 03:14:54 DEBUG "resultType": "vector", 03:14:54 DEBUG "result": [ 03:14:54 DEBUG { 03:14:54 DEBUG "metric": { 03:14:54 DEBUG { 03:14:54 DEBUG "data": { 03:14:54 DEBUG "data": { 03:14:54 DEBUG "result": [ 03:14:54 DEBUG "resultType": "vector", 03:14:54 DEBUG "result": [ 03:14:54 DEBUG "data": { 03:14:54 DEBUG { 03:14:54 DEBUG "resultType": "vector", 03:14:54 DEBUG "metric": { 03:14:54 DEBUG "status": "success", 03:14:54 DEBUG { 03:14:54 DEBUG "result": [ 03:14:54 DEBUG { 03:14:54 DEBUG "resultType": "vector", 03:14:54 DEBUG "resultType": "vector", 03:14:54 DEBUG "result": [ 03:14:54 DEBUG "metric": { 03:14:54 DEBUG "data": { 03:14:54 DEBUG "result": [ 03:14:54 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:14:54 DEBUG "resultType": "vector", 03:14:54 DEBUG "status": "success", 03:14:54 DEBUG "resultType": "vector", 03:14:54 DEBUG "resultType": "vector", 03:14:54 DEBUG { 03:14:54 DEBUG "result": [ 03:14:54 DEBUG "data": { 03:14:54 DEBUG { 03:14:54 DEBUG "resultType": "vector", 03:14:54 DEBUG "metric": { 03:14:54 DEBUG "result": [ 03:14:54 DEBUG "pod": "ds-idrepo-0" 03:14:54 DEBUG "data": { 03:14:54 DEBUG "metric": { 03:14:54 DEBUG { 03:14:54 DEBUG "metric": { 03:14:54 DEBUG "result": [ 03:14:54 DEBUG "result": [ 03:14:54 DEBUG { 03:14:54 DEBUG "pod": "am-767f6b67b5-9zw22" 03:14:54 DEBUG "resultType": "vector", 03:14:54 DEBUG { 03:14:54 DEBUG }, 03:14:54 DEBUG "result": [ 03:14:54 DEBUG "data": { 03:14:54 DEBUG "result": [ 03:14:54 DEBUG "result": [ 03:14:54 DEBUG "metric": { 03:14:54 DEBUG { 03:14:54 DEBUG "resultType": "vector", 03:14:54 DEBUG "metric": { 03:14:54 DEBUG "result": [ 03:14:54 DEBUG "instance": "10.142.0.107:9100" 03:14:54 DEBUG { 03:14:54 DEBUG }, 03:14:54 DEBUG "resultType": "vector", 03:14:54 DEBUG "instance": "10.142.0.107:9100" 03:14:54 DEBUG "metric": { 03:14:54 DEBUG "pod": "am-767f6b67b5-9zw22" 03:14:54 DEBUG { 03:14:54 DEBUG { 03:14:54 DEBUG "metric": { 03:14:54 DEBUG }, 03:14:54 DEBUG "result": [ 03:14:54 DEBUG "metric": { 03:14:54 DEBUG "value": [ 03:14:54 DEBUG { 03:14:54 DEBUG "resultType": "vector", 03:14:54 DEBUG { 03:14:54 DEBUG { 03:14:54 DEBUG "pod": "ds-cts-0" 03:14:54 DEBUG "metric": { 03:14:54 DEBUG "result": [ 03:14:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:14:54 DEBUG { 03:14:54 DEBUG }, 03:14:54 DEBUG "metric": { 03:14:54 DEBUG "value": [ 03:14:54 DEBUG "result": [ 03:14:54 DEBUG }, 03:14:54 DEBUG "pod": "ds-idrepo-0" 03:14:54 DEBUG }, 03:14:54 DEBUG "metric": { 03:14:54 DEBUG "metric": { 03:14:54 DEBUG "pod": "am-767f6b67b5-9zw22" 03:14:54 DEBUG "value": [ 03:14:54 DEBUG 03:14:54 DEBUG { 03:14:54 DEBUG 03:14:54 DEBUG "metric": { 03:14:54 DEBUG "result": [ 03:14:54 DEBUG "metric": { 03:14:54 DEBUG "metric": { 03:14:54 DEBUG }, 03:14:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:14:54 DEBUG { 03:14:54 DEBUG }, 03:14:54 DEBUG "metric": { 03:14:54 DEBUG "value": [ 03:14:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:14:54 DEBUG 03:14:54 DEBUG { 03:14:54 DEBUG "value": [ 03:14:54 DEBUG }, 03:14:54 DEBUG "value": [ 03:14:54 DEBUG "pod": "am-767f6b67b5-9zw22" 03:14:54 DEBUG "pod": "am-767f6b67b5-9zw22" 03:14:54 DEBUG }, 03:14:54 DEBUG 03:14:54 DEBUG "pod": "am-767f6b67b5-9zw22" 03:14:54 DEBUG "metric": { 03:14:54 DEBUG 03:14:54 DEBUG "pod": "ds-idrepo-0" 03:14:54 DEBUG { 03:14:54 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:14:54 DEBUG "pod": "am-767f6b67b5-9zw22" 03:14:54 DEBUG "value": [ 03:14:54 DEBUG }, 03:14:54 DEBUG "metric": { 03:14:54 DEBUG "value": [ 03:14:54 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:14:54 DEBUG 03:14:54 DEBUG }, 03:14:54 DEBUG 03:14:54 DEBUG "metric": { 03:14:54 DEBUG 03:14:54 DEBUG "value": [ 03:14:54 DEBUG 03:14:54 DEBUG }, 03:14:54 DEBUG }, 03:14:54 DEBUG "value": [ 03:14:54 DEBUG 03:14:54 DEBUG }, 03:14:54 DEBUG "instance": "10.142.0.107:9100" 03:14:54 DEBUG 03:14:54 DEBUG }, 03:14:54 DEBUG "metric": { 03:14:54 DEBUG }, 03:14:54 DEBUG }, 03:14:54 DEBUG 03:14:54 DEBUG "value": [ 03:14:54 DEBUG "pod": "ds-cts-0" 03:14:54 DEBUG 03:14:54 DEBUG }, 03:14:54 DEBUG 03:14:54 DEBUG "value": [ 03:14:54 DEBUG 03:14:54 DEBUG "instance": "10.142.0.107:9100" 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG "value": [ 03:14:54 DEBUG "value": [ 03:14:54 DEBUG 03:14:54 DEBUG [...] 03:14:54 DEBUG "value": [ 03:14:54 DEBUG }, 03:14:54 DEBUG [...] 03:14:54 DEBUG "value": [ 03:14:54 DEBUG "pod": "ds-idrepo-0" 03:14:54 DEBUG "value": [ 03:14:54 DEBUG "value": [ 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG }, 03:14:54 DEBUG 03:14:54 DEBUG "value": [ 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG }, 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG "value": [ 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG }, 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG "value": [ 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG [...] 03:14:54 DEBUG [...] 03:14:54 DEBUG 03:14:54 DEBUG "value": [ 03:14:54 DEBUG [...] 03:14:54 DEBUG 03:14:54 DEBUG [...] 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG "value": [ 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG [...] 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG [...] 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG [...] 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG [...] 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG [...] 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG [...] 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG [...] 03:14:54 DEBUG [...] 03:14:54 DEBUG 03:14:54 DEBUG "pod": "am-767f6b67b5-vbltz" 03:14:54 DEBUG [...] 03:14:54 DEBUG 03:14:54 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:14:54 DEBUG [...] 03:14:54 DEBUG 03:14:54 DEBUG [...] 03:14:54 DEBUG 03:14:54 DEBUG [...] 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG [...] 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG }, 03:14:54 DEBUG 03:14:54 DEBUG [...] 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG [...] 03:14:54 DEBUG 03:14:54 DEBUG "instance": "10.142.0.43:9100" 03:14:54 DEBUG }, 03:14:54 DEBUG 03:14:54 DEBUG [...] 03:14:54 DEBUG "instance": "10.142.0.43:9100" 03:14:54 DEBUG }, 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG "value": [ 03:14:54 DEBUG 03:14:54 DEBUG }, 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG [...] 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:14:54 DEBUG 03:14:54 DEBUG "pod": "ds-cts-0" 03:14:54 DEBUG "pod": "ds-idrepo-2" 03:14:54 DEBUG "value": [ 03:14:54 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:14:54 DEBUG 03:14:54 DEBUG "pod": "am-767f6b67b5-vbltz" 03:14:54 DEBUG "value": [ 03:14:54 DEBUG "pod": "ds-idrepo-2" 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG "pod": "am-767f6b67b5-vbltz" 03:14:54 DEBUG 1713665694, 03:14:54 DEBUG 03:14:54 DEBUG "value": [ 03:14:54 DEBUG 03:14:54 DEBUG "instance": "10.142.0.43:9100" 03:14:54 DEBUG 03:14:54 DEBUG "pod": "ds-cts-2" 03:14:54 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:14:54 DEBUG 03:14:54 DEBUG 03:14:54 DEBUG }, 03:14:54 DEBUG 03:14:54 DEBUG }, 03:14:54 DEBUG }, 03:14:54 DEBUG "value": [ 03:14:54 DEBUG }, 03:14:54 DEBUG 03:14:54 DEBUG }, 03:14:54 DEBUG 1713665694, 03:14:54 DEBUG }, 03:14:54 DEBUG "pod": "am-767f6b67b5-vbltz" 03:14:54 DEBUG "pod": "am-767f6b67b5-vbltz" 03:14:54 DEBUG }, 03:14:54 DEBUG "3325.633333333333" 03:14:54 DEBUG "pod": "am-767f6b67b5-vbltz" 03:14:54 DEBUG 1713665694, 03:14:54 DEBUG 03:14:54 DEBUG }, 03:14:54 DEBUG "pod": "ds-idrepo-2" 03:14:54 DEBUG }, 03:14:54 DEBUG }, 03:14:54 DEBUG "pod": "am-767f6b67b5-vbltz" 03:14:54 DEBUG 03:14:54 DEBUG "value": [ 03:14:54 DEBUG 03:14:54 DEBUG "value": [ 03:14:54 DEBUG 1713665694, 03:14:54 DEBUG 1713665694, 03:14:54 DEBUG "value": [ 03:14:54 DEBUG 03:14:54 DEBUG "value": [ 03:14:54 DEBUG "4370598949888" 03:14:54 DEBUG "value": [ 03:14:54 DEBUG }, 03:14:54 DEBUG }, 03:14:54 DEBUG "value": [ 03:14:54 DEBUG ] 03:14:54 DEBUG }, 03:14:54 DEBUG "1020354560" 03:14:54 DEBUG 03:14:54 DEBUG "value": [ 03:14:54 DEBUG }, 03:14:54 DEBUG "value": [ 03:14:54 DEBUG "value": [ 03:14:54 DEBUG }, 03:14:54 DEBUG "pod": "ds-cts-2" 03:14:54 DEBUG 1713665694, 03:14:54 DEBUG "pod": "ds-cts-0" 03:14:54 DEBUG 1713665694, 03:14:54 DEBUG "100386632.33333334" 03:14:54 DEBUG "0" 03:14:54 DEBUG 1713665694, 03:14:54 DEBUG "instance": "10.142.0.43:9100" 03:14:54 DEBUG }, 03:14:54 DEBUG ] 03:14:54 DEBUG 1713665694, 03:14:54 DEBUG "value": [ 03:14:54 DEBUG "value": [ 03:14:54 DEBUG 1713665694, 03:14:54 DEBUG } 03:14:54 DEBUG "value": [ 03:14:54 DEBUG ] 03:14:54 DEBUG 03:14:54 DEBUG 1713665694, 03:14:54 DEBUG "value": [ 03:14:54 DEBUG 1713665694, 03:14:54 DEBUG 1713665694, 03:14:54 DEBUG "value": [ 03:14:54 DEBUG }, 03:14:54 DEBUG "0" 03:14:54 DEBUG }, 03:14:54 DEBUG "0" 03:14:54 DEBUG ] 03:14:54 DEBUG ] 03:14:54 DEBUG "0" 03:14:54 DEBUG "value": [ 03:14:54 DEBUG 1713665694, 03:14:54 DEBUG } 03:14:54 DEBUG "0" 03:14:54 DEBUG 1713665694, 03:14:54 DEBUG 1713665694, 03:14:54 DEBUG "3326.5333333333333" 03:14:54 DEBUG ] 03:14:54 DEBUG 1713665694, 03:14:54 DEBUG } 03:14:54 DEBUG "pod": "ds-cts-2" 03:14:54 DEBUG "0.07566666666667553" 03:14:54 DEBUG 1713665694, 03:14:54 DEBUG "0" 03:14:54 DEBUG ] 03:14:54 DEBUG 1713665694, 03:14:54 DEBUG "0" 03:14:54 DEBUG ] 03:14:54 DEBUG "value": [ 03:14:54 DEBUG ] 03:14:54 DEBUG } 03:14:54 DEBUG } 03:14:54 DEBUG ] 03:14:54 DEBUG 1713665694, 03:14:54 DEBUG "600599.5" 03:14:54 DEBUG ] 03:14:54 DEBUG ] 03:14:54 DEBUG "3306.0666666666666" 03:14:54 DEBUG "3325.9666666666667" 03:14:54 DEBUG ] 03:14:54 DEBUG } 03:14:54 DEBUG "149.889" 03:14:54 DEBUG ] 03:14:54 DEBUG }, 03:14:54 DEBUG ] 03:14:54 DEBUG "22.366666666666667" 03:14:54 DEBUG } 03:14:54 DEBUG "11.777218733416495" 03:14:54 DEBUG ] 03:14:54 DEBUG "value": [ 03:14:54 DEBUG } 03:14:54 DEBUG 1713665694, 03:14:54 DEBUG } 03:14:54 DEBUG ] 03:14:54 DEBUG ] 03:14:54 DEBUG } 03:14:54 DEBUG "100383300.69999999" 03:14:54 DEBUG ] 03:14:54 DEBUG } 03:14:54 DEBUG } 03:14:54 DEBUG ] 03:14:54 DEBUG ] 03:14:54 DEBUG } 03:14:54 DEBUG } 03:14:54 DEBUG ] 03:14:54 DEBUG } 03:14:54 DEBUG "value": [ 03:14:54 DEBUG } 03:14:54 DEBUG ] 03:14:54 DEBUG ] 03:14:54 DEBUG ] 03:14:54 DEBUG } 03:14:54 DEBUG 1713665694, 03:14:54 DEBUG ] 03:14:54 DEBUG "0" 03:14:54 DEBUG ] 03:14:54 DEBUG } 03:14:54 DEBUG } 03:14:54 DEBUG ] 03:14:54 DEBUG ] 03:14:54 DEBUG } 03:14:54 DEBUG } 03:14:54 DEBUG } 03:14:54 DEBUG ] 03:14:54 DEBUG } 03:14:54 DEBUG ] 03:14:54 DEBUG } 03:14:54 DEBUG } 03:14:54 DEBUG 1713665694, 03:14:54 DEBUG } 03:14:54 DEBUG } 03:14:54 DEBUG ] 03:14:54 DEBUG } 03:14:54 DEBUG ] 03:14:54 DEBUG "0" 03:14:54 DEBUG } 03:14:54 DEBUG ] 03:14:54 DEBUG } 03:14:54 DEBUG } 03:14:54 DEBUG } 03:14:54 DEBUG } 03:14:54 DEBUG } 03:14:54 DEBUG ] 03:14:54 DEBUG } 03:14:54 DEBUG } 03:14:54 DEBUG ] 03:14:54 DEBUG } 03:14:54 DEBUG ] 03:14:54 DEBUG "0" 03:14:54 DEBUG } 03:14:54 DEBUG } 03:14:54 DEBUG } 03:14:54 DEBUG ] 03:14:54 DEBUG } 03:14:54 DEBUG ] 03:14:54 DEBUG } 03:14:54 DEBUG } 03:14:54 DEBUG } 03:14:54 DEBUG } 03:14:54 DEBUG } 03:14:54 DEBUG } 03:14:54 DEBUG ] 03:14:54 DEBUG } 03:14:54 DEBUG } 03:14:54 DEBUG } 03:14:54 DEBUG ] 03:14:54 DEBUG ] 03:14:54 DEBUG } 03:14:54 DEBUG } 03:14:54 DEBUG } 03:14:54 DEBUG ] 03:14:54 DEBUG ] 03:14:54 DEBUG } 03:14:54 DEBUG } 03:14:54 DEBUG } 03:14:54 DEBUG } 03:14:54 DEBUG } 03:14:54 DEBUG } 03:14:54 DEBUG ] 03:14:54 DEBUG } 03:14:54 DEBUG } 03:14:54 DEBUG } 03:14:54 DEBUG ] 03:14:54 DEBUG } 03:14:54 DEBUG } 03:14:54 DEBUG } 03:14:54 DEBUG } 03:14:54 DEBUG } 03:14:54 DEBUG } 03:14:54 DEBUG } 03:15:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713665752" 03:15:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665752" 03:15:52 INFO [http_cmd]: http status code OK 03:15:52 DEBUG --- status code --- 03:15:52 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:15:52 DEBUG --- http response --- 03:15:52 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 03:15:52 ERROR [http_cmd]: ERROR 03:15:52 ERROR --- status code --- 03:15:52 ERROR http status code is 400 (expected [200, 401, 403, 404]) 03:15:52 ERROR --- http response --- 03:15:52 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 03:15:52 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 03:15:53 INFO 03:15:53 INFO [loop_until]: kubectl --namespace=xlou top pods 03:15:53 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:15:54 INFO [loop_until]: OK (rc = 0) 03:15:54 DEBUG --- stdout --- 03:15:54 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 22364m 8795Mi am-767f6b67b5-qcqz2 22140m 9128Mi am-767f6b67b5-vbltz 24320m 8816Mi ds-cts-0 19810m 12437Mi ds-cts-1 19549m 12462Mi ds-cts-2 19807m 12510Mi ds-idrepo-0 9552m 3918Mi ds-idrepo-1 9472m 3945Mi ds-idrepo-2 9852m 3915Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1179Mi idm-b85f9d978-wnhxb 3m 1239Mi lodemon-54c87f8cbb-jzrjs 11m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-68c5d5b884-p97w5 11489m 883Mi 03:15:54 DEBUG --- stderr --- 03:15:54 DEBUG 03:15:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665754" 03:15:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665754" 03:15:54 INFO [http_cmd]: http status code OK 03:15:54 DEBUG --- status code --- 03:15:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:15:54 DEBUG --- http response --- 03:15:54 INFO [http_cmd]: http status code OK 03:15:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:15:54 DEBUG --- status code --- 03:15:54 DEBUG ----- output ----- 03:15:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:15:54 DEBUG { 03:15:54 DEBUG --- http response --- 03:15:54 DEBUG "status": "success", 03:15:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:15:54 DEBUG "data": { 03:15:54 DEBUG ----- output ----- 03:15:54 DEBUG "resultType": "vector", 03:15:54 DEBUG { 03:15:54 DEBUG "result": [ 03:15:54 DEBUG "status": "success", 03:15:54 DEBUG { 03:15:54 DEBUG "data": { 03:15:54 DEBUG "metric": { 03:15:54 DEBUG "resultType": "vector", 03:15:54 DEBUG "pod": "ds-cts-0" 03:15:54 DEBUG "result": [ 03:15:54 DEBUG }, 03:15:54 DEBUG { 03:15:54 DEBUG "value": [ 03:15:54 DEBUG "metric": { 03:15:54 DEBUG "instance": "10.142.0.107:9100" 03:15:54 DEBUG }, 03:15:54 DEBUG 03:15:54 DEBUG "value": [ 03:15:54 DEBUG 03:15:54 DEBUG 03:15:54 DEBUG 03:15:54 DEBUG 03:15:54 DEBUG [...] 03:15:54 DEBUG 03:15:54 DEBUG 03:15:54 DEBUG [...] 03:15:54 DEBUG 03:15:54 DEBUG 03:15:54 DEBUG 03:15:54 DEBUG 03:15:54 DEBUG "pod": "ds-cts-2" 03:15:54 DEBUG 03:15:54 DEBUG }, 03:15:54 DEBUG "instance": "10.142.0.43:9100" 03:15:54 DEBUG "value": [ 03:15:54 DEBUG }, 03:15:54 DEBUG 1713665754, 03:15:54 DEBUG "value": [ 03:15:54 DEBUG "0.5660000000000006" 03:15:54 DEBUG 1713665754, 03:15:54 DEBUG ] 03:15:54 DEBUG "0.39496666666665686" 03:15:54 DEBUG } 03:15:54 DEBUG ] 03:15:54 DEBUG ] 03:15:54 DEBUG } 03:15:54 DEBUG } 03:15:54 DEBUG ] 03:15:54 DEBUG } 03:15:54 DEBUG } 03:15:54 DEBUG } 03:15:54 INFO 03:15:54 INFO [loop_until]: kubectl --namespace=xlou top node 03:15:54 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:15:54 INFO [loop_until]: OK (rc = 0) 03:15:54 DEBUG --- stdout --- 03:15:54 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 66m 0% 6294Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 10006m 33% 7767Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 21980m 73% 11604Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 90m 0% 3360Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 19845m 66% 15754Mi 14% gke-xlou-cdm-default-pool-1b0769c8-wpwl 138m 0% 5290Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 22527m 75% 12068Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 19743m 66% 3174Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 24112m 80% 11087Mi 9% gke-xlou-cdm-ds-0f601751-j488 10385m 34% 7353Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 19693m 65% 17564Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 10147m 33% 8002Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20250m 67% 15233Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 12027m 75% 11229Mi 19% 03:15:54 DEBUG --- stderr --- 03:15:54 DEBUG 03:15:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665754" 03:15:55 INFO [http_cmd]: http status code OK 03:15:55 DEBUG --- status code --- 03:15:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:15:55 DEBUG --- http response --- 03:15:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:15:55 DEBUG ----- output ----- 03:15:55 DEBUG { 03:15:55 DEBUG "status": "success", 03:15:55 DEBUG "data": { 03:15:55 DEBUG "resultType": "vector", 03:15:55 DEBUG "result": [ 03:15:55 DEBUG { 03:15:55 DEBUG "metric": { 03:15:55 DEBUG "pod": "am-767f6b67b5-9zw22" 03:15:55 DEBUG }, 03:15:55 DEBUG "value": [ 03:15:55 DEBUG 03:15:55 DEBUG 03:15:55 DEBUG 03:15:55 DEBUG [...] 03:15:55 DEBUG 03:15:55 DEBUG 03:15:55 DEBUG 03:15:55 DEBUG "pod": "am-767f6b67b5-vbltz" 03:15:55 DEBUG }, 03:15:55 DEBUG "value": [ 03:15:55 DEBUG 1713665754, 03:15:55 DEBUG "3363.5" 03:15:55 DEBUG ] 03:15:55 DEBUG } 03:15:55 DEBUG ] 03:15:55 DEBUG } 03:15:55 DEBUG } 03:15:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665755" 03:15:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713665755" 03:15:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665755" 03:15:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665755" 03:15:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665755" 03:15:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665755" 03:15:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665755" 03:15:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665755" 03:15:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665755" 03:15:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665755" 03:15:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665755" 03:15:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665755" 03:15:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665755" 03:15:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665755" 03:15:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713665755" 03:15:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665755" 03:15:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713665755" 03:15:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665755" 03:15:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713665755" 03:15:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665755" 03:15:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665755" 03:15:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665755" 03:15:55 INFO [http_cmd]: http status code OK 03:15:55 INFO [http_cmd]: http status code OK 03:15:55 INFO [http_cmd]: http status code OK 03:15:55 INFO [http_cmd]: http status code OK 03:15:55 DEBUG --- status code --- 03:15:55 INFO [http_cmd]: http status code OK 03:15:55 INFO [http_cmd]: http status code OK 03:15:55 DEBUG --- status code --- 03:15:55 INFO [http_cmd]: http status code OK 03:15:55 INFO [http_cmd]: http status code OK 03:15:55 INFO [http_cmd]: http status code OK 03:15:55 INFO [http_cmd]: http status code OK 03:15:55 INFO [http_cmd]: http status code OK 03:15:55 INFO [http_cmd]: http status code OK 03:15:55 INFO [http_cmd]: http status code OK 03:15:55 DEBUG --- status code --- 03:15:55 INFO [http_cmd]: http status code OK 03:15:55 INFO [http_cmd]: http status code OK 03:15:55 DEBUG --- status code --- 03:15:55 DEBUG --- status code --- 03:15:55 INFO [http_cmd]: http status code OK 03:15:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:15:55 INFO [http_cmd]: http status code OK 03:15:55 INFO [http_cmd]: http status code OK 03:15:55 DEBUG --- status code --- 03:15:55 INFO [http_cmd]: http status code OK 03:15:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:15:55 DEBUG --- status code --- 03:15:55 INFO [http_cmd]: http status code OK 03:15:55 DEBUG --- status code --- 03:15:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:15:55 DEBUG --- status code --- 03:15:55 DEBUG --- status code --- 03:15:55 DEBUG --- status code --- 03:15:55 DEBUG --- status code --- 03:15:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:15:55 DEBUG --- status code --- 03:15:55 DEBUG --- status code --- 03:15:55 INFO [http_cmd]: http status code OK 03:15:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:15:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:15:55 DEBUG --- http response --- 03:15:55 DEBUG --- http response --- 03:15:55 DEBUG --- status code --- 03:15:55 DEBUG --- status code --- 03:15:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:15:55 DEBUG --- status code --- 03:15:55 INFO [http_cmd]: http status code OK 03:15:55 DEBUG --- http response --- 03:15:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:15:55 DEBUG --- status code --- 03:15:55 DEBUG --- http response --- 03:15:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:15:55 DEBUG --- status code --- 03:15:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:15:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:15:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:15:55 DEBUG --- http response --- 03:15:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:15:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:15:55 DEBUG --- status code --- 03:15:55 DEBUG --- http response --- 03:15:55 DEBUG --- status code --- 03:15:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:15:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:15:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:15:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:15:55 DEBUG --- http response --- 03:15:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:15:55 DEBUG --- status code --- 03:15:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:15:55 DEBUG --- http response --- 03:15:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:15:55 DEBUG --- http response --- 03:15:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 03:15:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:15:55 DEBUG --- http response --- 03:15:55 DEBUG --- http response --- 03:15:55 DEBUG --- http response --- 03:15:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:15:55 DEBUG --- http response --- 03:15:55 DEBUG --- http response --- 03:15:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:15:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 03:15:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:15:55 DEBUG ----- output ----- 03:15:55 DEBUG ----- output ----- 03:15:55 DEBUG --- http response --- 03:15:55 DEBUG --- http response --- 03:15:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:15:55 DEBUG --- http response --- 03:15:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:15:55 DEBUG ----- output ----- 03:15:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 03:15:55 DEBUG --- http response --- 03:15:55 DEBUG ----- output ----- 03:15:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:15:55 DEBUG --- http response --- 03:15:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:15:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:15:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:15:55 DEBUG ----- output ----- 03:15:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:15:55 DEBUG --- http response --- 03:15:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:15:55 DEBUG ----- output ----- 03:15:55 DEBUG --- http response --- 03:15:55 DEBUG { 03:15:55 DEBUG { 03:15:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 03:15:55 DEBUG ----- output ----- 03:15:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:15:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:15:55 DEBUG --- http response --- 03:15:55 DEBUG { 03:15:55 DEBUG ----- output ----- 03:15:55 DEBUG { 03:15:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:15:55 DEBUG ----- output ----- 03:15:55 DEBUG ----- output ----- 03:15:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:15:55 DEBUG ----- output ----- 03:15:55 DEBUG ----- output ----- 03:15:55 DEBUG { 03:15:55 DEBUG ----- output ----- 03:15:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:15:55 DEBUG ----- output ----- 03:15:55 DEBUG { 03:15:55 DEBUG "status": "success", 03:15:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:15:55 DEBUG "status": "success", 03:15:55 DEBUG ----- output ----- 03:15:55 DEBUG { 03:15:55 DEBUG ----- output ----- 03:15:55 DEBUG ----- output ----- 03:15:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:15:55 DEBUG ----- output ----- 03:15:55 DEBUG { 03:15:55 DEBUG "status": "success", 03:15:55 DEBUG ----- output ----- 03:15:55 DEBUG { 03:15:55 DEBUG { 03:15:55 DEBUG ----- output ----- 03:15:55 DEBUG { 03:15:55 DEBUG { 03:15:55 DEBUG "status": "success", 03:15:55 DEBUG { 03:15:55 DEBUG ----- output ----- 03:15:55 DEBUG { 03:15:55 DEBUG "status": "success", 03:15:55 DEBUG "data": { 03:15:55 DEBUG ----- output ----- 03:15:55 DEBUG "data": { 03:15:55 DEBUG { 03:15:55 DEBUG "status": "success", 03:15:55 DEBUG { 03:15:55 DEBUG { 03:15:55 DEBUG { 03:15:55 DEBUG "status": "success", 03:15:55 DEBUG "status": "success", 03:15:55 DEBUG "data": { 03:15:55 DEBUG { 03:15:55 DEBUG "status": "success", 03:15:55 DEBUG "status": "success", 03:15:55 DEBUG { 03:15:55 DEBUG "status": "success", 03:15:55 DEBUG "status": "success", 03:15:55 DEBUG "data": { 03:15:55 DEBUG "status": "success", 03:15:55 DEBUG { 03:15:55 DEBUG "status": "success", 03:15:55 DEBUG "data": { 03:15:55 DEBUG "resultType": "vector", 03:15:55 DEBUG { 03:15:55 DEBUG "status": "success", 03:15:55 DEBUG "status": "success", 03:15:55 DEBUG "data": { 03:15:55 DEBUG "status": "success", 03:15:55 DEBUG "status": "success", 03:15:55 DEBUG "status": "success", 03:15:55 DEBUG "data": { 03:15:55 DEBUG "data": { 03:15:55 DEBUG "resultType": "vector", 03:15:55 DEBUG "status": "success", 03:15:55 DEBUG "data": { 03:15:55 DEBUG "data": { 03:15:55 DEBUG "status": "success", 03:15:55 DEBUG "data": { 03:15:55 DEBUG "data": { 03:15:55 DEBUG "resultType": "vector", 03:15:55 DEBUG "data": { 03:15:55 DEBUG "status": "success", 03:15:55 DEBUG "data": { 03:15:55 DEBUG "resultType": "vector", 03:15:55 DEBUG "result": [ 03:15:55 DEBUG { 03:15:55 DEBUG "metric": { 03:15:55 DEBUG "resultType": "vector", 03:15:55 DEBUG "resultType": "vector", 03:15:55 DEBUG "data": { 03:15:55 DEBUG "data": { 03:15:55 DEBUG "data": { 03:15:55 DEBUG "resultType": "vector", 03:15:55 DEBUG "resultType": "vector", 03:15:55 DEBUG "result": [ 03:15:55 DEBUG "data": { 03:15:55 DEBUG "resultType": "vector", 03:15:55 DEBUG "resultType": "vector", 03:15:55 DEBUG "data": { 03:15:55 DEBUG "resultType": "vector", 03:15:55 DEBUG "resultType": "vector", 03:15:55 DEBUG "result": [ 03:15:55 DEBUG "resultType": "vector", 03:15:55 DEBUG "data": { 03:15:55 DEBUG "resultType": "vector", 03:15:55 DEBUG "result": [ 03:15:55 DEBUG "data": { 03:15:55 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:15:55 DEBUG "data": { 03:15:55 DEBUG "result": [ 03:15:55 DEBUG { 03:15:55 DEBUG "metric": { 03:15:55 DEBUG "resultType": "vector", 03:15:55 DEBUG "resultType": "vector", 03:15:55 DEBUG "result": [ 03:15:55 DEBUG "result": [ 03:15:55 DEBUG { 03:15:55 DEBUG "resultType": "vector", 03:15:55 DEBUG "result": [ 03:15:55 DEBUG "result": [ 03:15:55 DEBUG "resultType": "vector", 03:15:55 DEBUG "result": [ 03:15:55 DEBUG { 03:15:55 DEBUG { 03:15:55 DEBUG "result": [ 03:15:55 DEBUG "resultType": "vector", 03:15:55 DEBUG "result": [ 03:15:55 DEBUG { 03:15:55 DEBUG "resultType": "vector", 03:15:55 DEBUG }, 03:15:55 DEBUG "resultType": "vector", 03:15:55 DEBUG "instance": "10.142.0.107:9100" 03:15:55 DEBUG "result": [ 03:15:55 DEBUG "resultType": "vector", 03:15:55 DEBUG "result": [ 03:15:55 DEBUG "result": [ 03:15:55 DEBUG { 03:15:55 DEBUG { 03:15:55 DEBUG "metric": { 03:15:55 DEBUG "result": [ 03:15:55 DEBUG { 03:15:55 DEBUG { 03:15:55 DEBUG "metric": { 03:15:55 DEBUG "result": [ 03:15:55 DEBUG "result": [ 03:15:55 DEBUG "metric": { 03:15:55 DEBUG { 03:15:55 DEBUG "result": [ 03:15:55 DEBUG { 03:15:55 DEBUG "metric": { 03:15:55 DEBUG "result": [ 03:15:55 DEBUG "value": [ 03:15:55 DEBUG "result": [ 03:15:55 DEBUG }, 03:15:55 DEBUG { 03:15:55 DEBUG "result": [ 03:15:55 DEBUG { 03:15:55 DEBUG { 03:15:55 DEBUG "metric": { 03:15:55 DEBUG "metric": { 03:15:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:15:55 DEBUG { 03:15:55 DEBUG "metric": { 03:15:55 DEBUG "metric": { 03:15:55 DEBUG "pod": "am-767f6b67b5-9zw22" 03:15:55 DEBUG { 03:15:55 DEBUG { 03:15:55 DEBUG "instance": "10.142.0.107:9100" 03:15:55 DEBUG "metric": { 03:15:55 DEBUG { 03:15:55 DEBUG "metric": { 03:15:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:15:55 DEBUG { 03:15:55 DEBUG 03:15:55 DEBUG { 03:15:55 DEBUG "value": [ 03:15:55 DEBUG "metric": { 03:15:55 DEBUG { 03:15:55 DEBUG "metric": { 03:15:55 DEBUG "metric": { 03:15:55 DEBUG "pod": "ds-idrepo-0" 03:15:55 DEBUG }, 03:15:55 DEBUG }, 03:15:55 DEBUG "metric": { 03:15:55 DEBUG "pod": "am-767f6b67b5-9zw22" 03:15:55 DEBUG "pod": "ds-cts-0" 03:15:55 DEBUG }, 03:15:55 DEBUG "metric": { 03:15:55 DEBUG "metric": { 03:15:55 DEBUG "pod": "am-767f6b67b5-9zw22" 03:15:55 DEBUG }, 03:15:55 DEBUG "metric": { 03:15:55 DEBUG "pod": "am-767f6b67b5-9zw22" 03:15:55 DEBUG }, 03:15:55 DEBUG "metric": { 03:15:55 DEBUG 03:15:55 DEBUG "metric": { 03:15:55 DEBUG 03:15:55 DEBUG "pod": "ds-idrepo-0" 03:15:55 DEBUG "metric": { 03:15:55 DEBUG "pod": "ds-cts-0" 03:15:55 DEBUG "pod": "am-767f6b67b5-9zw22" 03:15:55 DEBUG "value": [ 03:15:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:15:55 DEBUG "value": [ 03:15:55 DEBUG "pod": "ds-idrepo-0" 03:15:55 DEBUG }, 03:15:55 DEBUG "value": [ 03:15:55 DEBUG "value": [ 03:15:55 DEBUG "pod": "am-767f6b67b5-9zw22" 03:15:55 DEBUG }, 03:15:55 DEBUG }, 03:15:55 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:15:55 DEBUG "instance": "10.142.0.107:9100" 03:15:55 DEBUG "value": [ 03:15:55 DEBUG }, 03:15:55 DEBUG "value": [ 03:15:55 DEBUG 03:15:55 DEBUG "instance": "10.142.0.107:9100" 03:15:55 DEBUG 03:15:55 DEBUG }, 03:15:55 DEBUG "pod": "ds-idrepo-0" 03:15:55 DEBUG }, 03:15:55 DEBUG }, 03:15:55 DEBUG 03:15:55 DEBUG }, 03:15:55 DEBUG "value": [ 03:15:55 DEBUG }, 03:15:55 DEBUG 03:15:55 DEBUG }, 03:15:55 DEBUG 03:15:55 DEBUG "value": [ 03:15:55 DEBUG "value": [ 03:15:55 DEBUG "value": [ 03:15:55 DEBUG }, 03:15:55 DEBUG }, 03:15:55 DEBUG 03:15:55 DEBUG 03:15:55 DEBUG 03:15:55 DEBUG [...] 03:15:55 DEBUG }, 03:15:55 DEBUG 03:15:55 DEBUG "value": [ 03:15:55 DEBUG }, 03:15:55 DEBUG "value": [ 03:15:55 DEBUG "value": [ 03:15:55 DEBUG 03:15:55 DEBUG 03:15:55 DEBUG 03:15:55 DEBUG 03:15:55 DEBUG 03:15:55 DEBUG "value": [ 03:15:55 DEBUG 03:15:55 DEBUG 03:15:55 DEBUG 03:15:55 DEBUG 03:15:55 DEBUG "value": [ 03:15:55 DEBUG 03:15:55 DEBUG 03:15:55 DEBUG 03:15:55 DEBUG [...] 03:15:55 DEBUG 03:15:55 DEBUG "value": [ 03:15:55 DEBUG [...] 03:15:55 DEBUG 03:15:55 DEBUG "value": [ 03:15:55 DEBUG 03:15:55 DEBUG 03:15:55 DEBUG 03:15:55 DEBUG [...] 03:15:55 DEBUG 03:15:55 DEBUG 03:15:55 DEBUG "value": [ 03:15:55 DEBUG 03:15:55 DEBUG 03:15:55 DEBUG 03:15:55 DEBUG 03:15:55 DEBUG 03:15:55 DEBUG 03:15:55 DEBUG "value": [ 03:15:55 DEBUG 03:15:55 DEBUG 03:15:55 DEBUG 03:15:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:15:55 DEBUG }, 03:15:55 DEBUG 03:15:55 DEBUG 03:15:55 DEBUG 03:15:55 DEBUG 03:15:55 DEBUG 03:15:55 DEBUG [...] 03:15:55 DEBUG 03:15:55 DEBUG 03:15:55 DEBUG 03:15:55 DEBUG 03:15:55 DEBUG 03:15:55 DEBUG [...] 03:15:55 DEBUG 03:15:55 DEBUG [...] 03:15:55 DEBUG 03:15:55 DEBUG 03:15:55 DEBUG 03:15:55 DEBUG [...] 03:15:55 DEBUG 03:15:55 DEBUG 03:15:55 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:15:55 DEBUG 03:15:55 DEBUG 03:15:55 DEBUG 03:15:55 DEBUG [...] 03:15:55 DEBUG 03:15:55 DEBUG 03:15:55 DEBUG 03:15:55 DEBUG [...] 03:15:55 DEBUG 03:15:55 DEBUG 03:15:55 DEBUG 03:15:55 DEBUG 03:15:55 DEBUG 03:15:55 DEBUG 03:15:55 DEBUG 03:15:55 DEBUG [...] 03:15:55 DEBUG [...] 03:15:55 DEBUG 03:15:55 DEBUG 03:15:55 DEBUG 03:15:55 DEBUG }, 03:15:55 DEBUG "value": [ 03:15:55 DEBUG 03:15:55 DEBUG 03:15:55 DEBUG [...] 03:15:55 DEBUG 03:15:55 DEBUG 03:15:55 DEBUG 03:15:55 DEBUG 03:15:55 DEBUG 03:15:55 DEBUG [...] 03:15:55 DEBUG [...] 03:15:55 DEBUG 03:15:55 DEBUG [...] 03:15:55 DEBUG 03:15:55 DEBUG 03:15:55 DEBUG [...] 03:15:55 DEBUG 03:15:55 DEBUG 03:15:55 DEBUG 03:15:55 DEBUG 03:15:55 DEBUG "pod": "idm-b85f9d978-wnhxb" 03:15:55 DEBUG "value": [ 03:15:55 DEBUG 03:15:55 DEBUG 03:15:55 DEBUG "instance": "10.142.0.43:9100" 03:15:55 DEBUG }, 03:15:55 DEBUG 03:15:55 DEBUG [...] 03:15:55 DEBUG 03:15:55 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:15:55 DEBUG 03:15:55 DEBUG 03:15:55 DEBUG 03:15:55 DEBUG [...] 03:15:55 DEBUG 03:15:55 DEBUG 03:15:55 DEBUG 03:15:55 DEBUG 03:15:55 DEBUG 03:15:55 DEBUG 03:15:55 DEBUG [...] 03:15:55 DEBUG 03:15:55 DEBUG }, 03:15:55 DEBUG 1713665755, 03:15:55 DEBUG 03:15:55 DEBUG [...] 03:15:55 DEBUG 03:15:55 DEBUG "value": [ 03:15:55 DEBUG 03:15:55 DEBUG 03:15:55 DEBUG "pod": "ds-idrepo-2" 03:15:55 DEBUG }, 03:15:55 DEBUG 03:15:55 DEBUG 03:15:55 DEBUG 03:15:55 DEBUG 03:15:55 DEBUG 03:15:55 DEBUG "pod": "am-767f6b67b5-vbltz" 03:15:55 DEBUG "pod": "am-767f6b67b5-vbltz" 03:15:55 DEBUG 03:15:55 DEBUG "instance": "10.142.0.43:9100" 03:15:55 DEBUG 03:15:55 DEBUG "pod": "am-767f6b67b5-vbltz" 03:15:55 DEBUG "value": [ 03:15:55 DEBUG "1022824448" 03:15:55 DEBUG 03:15:55 DEBUG 03:15:55 DEBUG 03:15:55 DEBUG 1713665755, 03:15:55 DEBUG 03:15:55 DEBUG "pod": "ds-cts-2" 03:15:55 DEBUG 03:15:55 DEBUG }, 03:15:55 DEBUG "value": [ 03:15:55 DEBUG "pod": "am-767f6b67b5-vbltz" 03:15:55 DEBUG 03:15:55 DEBUG 03:15:55 DEBUG 03:15:55 DEBUG 03:15:55 DEBUG }, 03:15:55 DEBUG }, 03:15:55 DEBUG 03:15:55 DEBUG }, 03:15:55 DEBUG 03:15:55 DEBUG }, 03:15:55 DEBUG 1713665755, 03:15:55 DEBUG ] 03:15:55 DEBUG [...] 03:15:55 DEBUG 03:15:55 DEBUG 03:15:55 DEBUG "0.07733333333331606" 03:15:55 DEBUG 03:15:55 DEBUG }, 03:15:55 DEBUG 03:15:55 DEBUG "value": [ 03:15:55 DEBUG 1713665755, 03:15:55 DEBUG }, 03:15:55 DEBUG "pod": "idm-b85f9d978-wnhxb" 03:15:55 DEBUG "pod": "am-767f6b67b5-vbltz" 03:15:55 DEBUG 03:15:55 DEBUG "pod": "ds-cts-2" 03:15:55 DEBUG "value": [ 03:15:55 DEBUG "value": [ 03:15:55 DEBUG "pod": "am-767f6b67b5-vbltz" 03:15:55 DEBUG "value": [ 03:15:55 DEBUG 03:15:55 DEBUG "value": [ 03:15:55 DEBUG "0" 03:15:55 DEBUG } 03:15:55 DEBUG 03:15:55 DEBUG 03:15:55 DEBUG "pod": "ds-idrepo-2" 03:15:55 DEBUG ] 03:15:55 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:15:55 DEBUG "value": [ 03:15:55 DEBUG "pod": "ds-cts-2" 03:15:55 DEBUG 1713665755, 03:15:55 DEBUG "0" 03:15:55 DEBUG "value": [ 03:15:55 DEBUG }, 03:15:55 DEBUG }, 03:15:55 DEBUG "pod": "ds-idrepo-2" 03:15:55 DEBUG }, 03:15:55 DEBUG 1713665755, 03:15:55 DEBUG 1713665755, 03:15:55 DEBUG "599961" 03:15:55 DEBUG 1713665755, 03:15:55 DEBUG "instance": "10.142.0.43:9100" 03:15:55 DEBUG 1713665755, 03:15:55 DEBUG ] 03:15:55 DEBUG ] 03:15:55 DEBUG 03:15:55 DEBUG "instance": "10.142.0.43:9100" 03:15:55 DEBUG }, 03:15:55 DEBUG } 03:15:55 DEBUG }, 03:15:55 DEBUG 1713665755, 03:15:55 DEBUG }, 03:15:55 DEBUG "0" 03:15:55 DEBUG ] 03:15:55 DEBUG 1713665755, 03:15:55 DEBUG "0" 03:15:55 DEBUG "value": [ 03:15:55 DEBUG }, 03:15:55 DEBUG "value": [ 03:15:55 DEBUG "3332.9" 03:15:55 DEBUG ] 03:15:55 DEBUG }, 03:15:55 DEBUG "4375971888128" 03:15:55 DEBUG }, 03:15:55 DEBUG "149.889" 03:15:55 DEBUG } 03:15:55 DEBUG } 03:15:55 DEBUG 03:15:55 DEBUG }, 03:15:55 DEBUG "value": [ 03:15:55 DEBUG ] 03:15:55 DEBUG "value": [ 03:15:55 DEBUG "0" 03:15:55 DEBUG "value": [ 03:15:55 DEBUG ] 03:15:55 DEBUG } 03:15:55 DEBUG ] 03:15:55 DEBUG "value": [ 03:15:55 DEBUG 1713665755, 03:15:55 DEBUG "value": [ 03:15:55 DEBUG 1713665755, 03:15:55 DEBUG ] 03:15:55 DEBUG } 03:15:55 DEBUG "value": [ 03:15:55 DEBUG ] 03:15:55 DEBUG "value": [ 03:15:55 DEBUG ] 03:15:55 DEBUG ] 03:15:55 DEBUG } 03:15:55 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:15:55 DEBUG "value": [ 03:15:55 DEBUG 1713665755, 03:15:55 DEBUG } 03:15:55 DEBUG 1713665755, 03:15:55 DEBUG "11.768344066744994" 03:15:55 DEBUG ] 03:15:55 DEBUG } 03:15:55 DEBUG ] 03:15:55 DEBUG } 03:15:55 DEBUG } 03:15:55 DEBUG "3347.5333333333333" 03:15:55 DEBUG } 03:15:55 DEBUG "30.5" 03:15:55 DEBUG } 03:15:55 DEBUG ] 03:15:55 DEBUG 1713665755, 03:15:55 DEBUG } 03:15:55 DEBUG 1713665755, 03:15:55 DEBUG } 03:15:55 DEBUG } 03:15:55 DEBUG }, 03:15:55 DEBUG 1713665755, 03:15:55 DEBUG "100373577.2" 03:15:55 DEBUG ] 03:15:55 DEBUG 1713665755, 03:15:55 DEBUG } 03:15:55 DEBUG 1713665755, 03:15:55 DEBUG 1713665755, 03:15:55 DEBUG ] 03:15:55 DEBUG ] 03:15:55 DEBUG ] 03:15:55 DEBUG } 03:15:55 DEBUG ] 03:15:55 DEBUG ] 03:15:55 DEBUG } 03:15:55 DEBUG "3361.9666666666667" 03:15:55 DEBUG ] 03:15:55 DEBUG "100370559.4" 03:15:55 DEBUG ] 03:15:55 DEBUG } 03:15:55 DEBUG "value": [ 03:15:55 DEBUG } 03:15:55 DEBUG "0" 03:15:55 DEBUG } 03:15:55 DEBUG "0" 03:15:55 DEBUG ] 03:15:55 DEBUG "1217.145583086394" 03:15:55 DEBUG "0" 03:15:55 DEBUG } 03:15:55 DEBUG } 03:15:55 DEBUG ] 03:15:55 DEBUG ] 03:15:55 DEBUG } 03:15:55 DEBUG } 03:15:55 DEBUG } 03:15:55 DEBUG ] 03:15:55 DEBUG } 03:15:55 DEBUG ] 03:15:55 DEBUG } 03:15:55 DEBUG } 03:15:55 DEBUG ] 03:15:55 DEBUG ] 03:15:55 DEBUG ] 03:15:55 DEBUG } 03:15:55 DEBUG ] 03:15:55 DEBUG ] 03:15:55 DEBUG ] 03:15:55 DEBUG } 03:15:55 DEBUG } 03:15:55 DEBUG } 03:15:55 DEBUG ] 03:15:55 DEBUG } 03:15:55 DEBUG } 03:15:55 DEBUG } 03:15:55 DEBUG } 03:15:55 DEBUG 1713665755, 03:15:55 DEBUG } 03:15:55 DEBUG } 03:15:55 DEBUG } 03:15:55 DEBUG ] 03:15:55 DEBUG } 03:15:55 DEBUG } 03:15:55 DEBUG } 03:15:55 DEBUG } 03:15:55 DEBUG } 03:15:55 DEBUG } 03:15:55 DEBUG ] 03:15:55 DEBUG ] 03:15:55 DEBUG "0" 03:15:55 DEBUG } 03:15:55 DEBUG ] 03:15:55 DEBUG } 03:15:55 DEBUG } 03:15:55 DEBUG ] 03:15:55 DEBUG ] 03:15:55 DEBUG } 03:15:55 DEBUG } 03:15:55 DEBUG } 03:15:55 DEBUG } 03:15:55 DEBUG ] 03:15:55 DEBUG } 03:15:55 DEBUG } 03:15:55 DEBUG } 03:15:55 DEBUG } 03:15:55 DEBUG } 03:15:55 DEBUG } 03:15:55 DEBUG } 03:15:55 DEBUG } 03:15:55 DEBUG } 03:15:55 DEBUG } 03:15:55 DEBUG ] 03:15:55 DEBUG } 03:15:55 DEBUG } 03:16:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665812" 03:16:52 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713665812" 03:16:53 ERROR [http_cmd]: ERROR 03:16:53 INFO [http_cmd]: http status code OK 03:16:53 ERROR --- status code --- 03:16:53 DEBUG --- status code --- 03:16:53 ERROR http status code is 400 (expected [200, 401, 403, 404]) 03:16:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:16:53 ERROR --- http response --- 03:16:53 DEBUG --- http response --- 03:16:53 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 03:16:53 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 03:16:53 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 03:16:54 INFO 03:16:54 INFO [loop_until]: kubectl --namespace=xlou top pods 03:16:54 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:16:54 INFO [loop_until]: OK (rc = 0) 03:16:54 DEBUG --- stdout --- 03:16:54 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 22506m 8795Mi am-767f6b67b5-qcqz2 22893m 9127Mi am-767f6b67b5-vbltz 24659m 8830Mi ds-cts-0 19882m 12464Mi ds-cts-1 20198m 12471Mi ds-cts-2 20656m 12547Mi ds-idrepo-0 9651m 3918Mi ds-idrepo-1 9437m 3946Mi ds-idrepo-2 9692m 3915Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1179Mi idm-b85f9d978-wnhxb 3m 1239Mi lodemon-54c87f8cbb-jzrjs 19m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-68c5d5b884-p97w5 11612m 885Mi 03:16:54 DEBUG --- stderr --- 03:16:54 DEBUG 03:16:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665814" 03:16:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665814" 03:16:54 INFO [http_cmd]: http status code OK 03:16:54 INFO [http_cmd]: http status code OK 03:16:54 DEBUG --- status code --- 03:16:54 DEBUG --- status code --- 03:16:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:16:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:16:54 DEBUG --- http response --- 03:16:54 DEBUG --- http response --- 03:16:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:16:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:16:54 DEBUG ----- output ----- 03:16:54 DEBUG ----- output ----- 03:16:54 DEBUG { 03:16:54 DEBUG { 03:16:54 DEBUG "status": "success", 03:16:54 DEBUG "data": { 03:16:54 DEBUG "resultType": "vector", 03:16:54 DEBUG "result": [ 03:16:54 DEBUG { 03:16:54 DEBUG "status": "success", 03:16:54 DEBUG "metric": { 03:16:54 DEBUG "data": { 03:16:54 DEBUG "pod": "ds-cts-0" 03:16:54 DEBUG "resultType": "vector", 03:16:54 DEBUG }, 03:16:54 DEBUG "result": [ 03:16:54 DEBUG "value": [ 03:16:54 DEBUG { 03:16:54 DEBUG 03:16:54 DEBUG 03:16:54 DEBUG "metric": { 03:16:54 DEBUG 03:16:54 DEBUG "instance": "10.142.0.107:9100" 03:16:54 DEBUG [...] 03:16:54 DEBUG }, 03:16:54 DEBUG 03:16:54 DEBUG "value": [ 03:16:54 DEBUG 03:16:54 DEBUG 03:16:54 DEBUG 03:16:54 DEBUG 03:16:54 DEBUG "pod": "ds-cts-2" 03:16:54 DEBUG 03:16:54 DEBUG }, 03:16:54 DEBUG [...] 03:16:54 DEBUG "value": [ 03:16:54 DEBUG 03:16:54 DEBUG 1713665814, 03:16:54 DEBUG 03:16:54 DEBUG "0.5516333333333326" 03:16:54 DEBUG 03:16:54 DEBUG ] 03:16:54 DEBUG "instance": "10.142.0.43:9100" 03:16:54 DEBUG } 03:16:54 DEBUG }, 03:16:54 DEBUG ] 03:16:54 DEBUG "value": [ 03:16:54 DEBUG } 03:16:54 DEBUG 1713665814, 03:16:54 DEBUG "0.41763333333343555" 03:16:54 DEBUG ] 03:16:54 DEBUG } 03:16:54 DEBUG ] 03:16:54 DEBUG } 03:16:54 DEBUG } 03:16:54 DEBUG } 03:16:54 INFO 03:16:54 INFO [loop_until]: kubectl --namespace=xlou top node 03:16:54 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:16:54 INFO [loop_until]: OK (rc = 0) 03:16:54 DEBUG --- stdout --- 03:16:54 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 6290Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 9956m 33% 7763Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 22012m 73% 11597Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 85m 0% 3351Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 20170m 67% 15766Mi 14% gke-xlou-cdm-default-pool-1b0769c8-wpwl 155m 0% 5266Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 22396m 75% 12057Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 19705m 66% 3186Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 24636m 82% 11082Mi 9% gke-xlou-cdm-ds-0f601751-j488 10344m 34% 7359Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 19788m 66% 17594Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 10164m 34% 7991Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20152m 67% 15249Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 12007m 75% 11227Mi 19% 03:16:54 DEBUG --- stderr --- 03:16:54 DEBUG 03:16:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665815" 03:16:55 INFO [http_cmd]: http status code OK 03:16:55 DEBUG --- status code --- 03:16:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:16:55 DEBUG --- http response --- 03:16:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:16:55 DEBUG ----- output ----- 03:16:55 DEBUG { 03:16:55 DEBUG "status": "success", 03:16:55 DEBUG "data": { 03:16:55 DEBUG "resultType": "vector", 03:16:55 DEBUG "result": [ 03:16:55 DEBUG { 03:16:55 DEBUG "metric": { 03:16:55 DEBUG "pod": "am-767f6b67b5-9zw22" 03:16:55 DEBUG }, 03:16:55 DEBUG "value": [ 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG [...] 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG "pod": "am-767f6b67b5-vbltz" 03:16:55 DEBUG }, 03:16:55 DEBUG "value": [ 03:16:55 DEBUG 1713665815, 03:16:55 DEBUG "3311.233333333333" 03:16:55 DEBUG ] 03:16:55 DEBUG } 03:16:55 DEBUG ] 03:16:55 DEBUG } 03:16:55 DEBUG } 03:16:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713665815" 03:16:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713665815" 03:16:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665815" 03:16:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665815" 03:16:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665815" 03:16:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713665815" 03:16:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665815" 03:16:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665815" 03:16:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665815" 03:16:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665815" 03:16:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665815" 03:16:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665815" 03:16:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665815" 03:16:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713665815" 03:16:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665815" 03:16:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665815" 03:16:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665815" 03:16:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665815" 03:16:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665815" 03:16:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665815" 03:16:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665815" 03:16:55 INFO [http_cmd]: http status code OK 03:16:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665815" 03:16:55 INFO [http_cmd]: http status code OK 03:16:55 INFO [http_cmd]: http status code OK 03:16:55 INFO [http_cmd]: http status code OK 03:16:55 INFO [http_cmd]: http status code OK 03:16:55 INFO [http_cmd]: http status code OK 03:16:55 INFO [http_cmd]: http status code OK 03:16:55 INFO [http_cmd]: http status code OK 03:16:55 INFO [http_cmd]: http status code OK 03:16:55 INFO [http_cmd]: http status code OK 03:16:55 INFO [http_cmd]: http status code OK 03:16:55 INFO [http_cmd]: http status code OK 03:16:55 INFO [http_cmd]: http status code OK 03:16:55 DEBUG --- status code --- 03:16:55 INFO [http_cmd]: http status code OK 03:16:55 INFO [http_cmd]: http status code OK 03:16:55 INFO [http_cmd]: http status code OK 03:16:55 DEBUG --- status code --- 03:16:55 INFO [http_cmd]: http status code OK 03:16:55 DEBUG --- status code --- 03:16:55 INFO [http_cmd]: http status code OK 03:16:55 DEBUG --- status code --- 03:16:55 DEBUG --- status code --- 03:16:55 DEBUG --- status code --- 03:16:55 DEBUG --- status code --- 03:16:55 INFO [http_cmd]: http status code OK 03:16:55 DEBUG --- status code --- 03:16:55 DEBUG --- status code --- 03:16:55 DEBUG --- status code --- 03:16:55 DEBUG --- status code --- 03:16:55 DEBUG --- status code --- 03:16:55 DEBUG --- status code --- 03:16:55 DEBUG --- status code --- 03:16:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:16:55 DEBUG --- status code --- 03:16:55 DEBUG --- status code --- 03:16:55 DEBUG --- status code --- 03:16:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:16:55 DEBUG --- status code --- 03:16:55 INFO [http_cmd]: http status code OK 03:16:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:16:55 DEBUG --- status code --- 03:16:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:16:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:16:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:16:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:16:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:16:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:16:55 INFO [http_cmd]: http status code OK 03:16:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:16:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:16:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:16:55 INFO [http_cmd]: http status code OK 03:16:55 DEBUG --- status code --- 03:16:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:16:55 DEBUG --- http response --- 03:16:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:16:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:16:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:16:55 DEBUG --- http response --- 03:16:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:16:55 DEBUG --- status code --- 03:16:55 DEBUG --- http response --- 03:16:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:16:55 DEBUG --- http response --- 03:16:55 DEBUG --- http response --- 03:16:55 DEBUG --- http response --- 03:16:55 DEBUG --- http response --- 03:16:55 DEBUG --- http response --- 03:16:55 DEBUG --- http response --- 03:16:55 DEBUG --- status code --- 03:16:55 DEBUG --- http response --- 03:16:55 DEBUG --- http response --- 03:16:55 DEBUG --- http response --- 03:16:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:16:55 DEBUG --- http response --- 03:16:55 DEBUG --- http response --- 03:16:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:16:55 DEBUG --- http response --- 03:16:55 DEBUG --- http response --- 03:16:55 DEBUG --- http response --- 03:16:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:16:55 DEBUG --- http response --- 03:16:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:16:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:16:55 DEBUG --- http response --- 03:16:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:16:55 DEBUG ----- output ----- 03:16:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:16:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:16:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:16:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:16:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:16:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:16:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:16:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:16:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:16:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:16:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:16:55 DEBUG ----- output ----- 03:16:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 03:16:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:16:55 DEBUG ----- output ----- 03:16:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:16:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:16:55 DEBUG --- http response --- 03:16:55 DEBUG ----- output ----- 03:16:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:16:55 DEBUG { 03:16:55 DEBUG ----- output ----- 03:16:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 03:16:55 DEBUG ----- output ----- 03:16:55 DEBUG ----- output ----- 03:16:55 DEBUG ----- output ----- 03:16:55 DEBUG --- http response --- 03:16:55 DEBUG ----- output ----- 03:16:55 DEBUG ----- output ----- 03:16:55 DEBUG ----- output ----- 03:16:55 DEBUG --- http response --- 03:16:55 DEBUG ----- output ----- 03:16:55 DEBUG ----- output ----- 03:16:55 DEBUG { 03:16:55 DEBUG ----- output ----- 03:16:55 DEBUG ----- output ----- 03:16:55 DEBUG { 03:16:55 DEBUG ----- output ----- 03:16:55 DEBUG ----- output ----- 03:16:55 DEBUG { 03:16:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:16:55 DEBUG ----- output ----- 03:16:55 DEBUG { 03:16:55 DEBUG { 03:16:55 DEBUG ----- output ----- 03:16:55 DEBUG { 03:16:55 DEBUG { 03:16:55 DEBUG { 03:16:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:16:55 DEBUG { 03:16:55 DEBUG { 03:16:55 DEBUG { 03:16:55 DEBUG { 03:16:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:16:55 DEBUG { 03:16:55 DEBUG "status": "success", 03:16:55 DEBUG { 03:16:55 DEBUG { 03:16:55 DEBUG "status": "success", 03:16:55 DEBUG { 03:16:55 DEBUG { 03:16:55 DEBUG "status": "success", 03:16:55 DEBUG ----- output ----- 03:16:55 DEBUG "status": "success", 03:16:55 DEBUG "status": "success", 03:16:55 DEBUG "status": "success", 03:16:55 DEBUG { 03:16:55 DEBUG "status": "success", 03:16:55 DEBUG "status": "success", 03:16:55 DEBUG "status": "success", 03:16:55 DEBUG ----- output ----- 03:16:55 DEBUG "status": "success", 03:16:55 DEBUG "status": "success", 03:16:55 DEBUG "status": "success", 03:16:55 DEBUG "status": "success", 03:16:55 DEBUG ----- output ----- 03:16:55 DEBUG "status": "success", 03:16:55 DEBUG "data": { 03:16:55 DEBUG "status": "success", 03:16:55 DEBUG "status": "success", 03:16:55 DEBUG "data": { 03:16:55 DEBUG "resultType": "vector", 03:16:55 DEBUG "result": [ 03:16:55 DEBUG "data": { 03:16:55 DEBUG "resultType": "vector", 03:16:55 DEBUG "data": { 03:16:55 DEBUG "data": { 03:16:55 DEBUG "data": { 03:16:55 DEBUG "status": "success", 03:16:55 DEBUG "data": { 03:16:55 DEBUG "data": { 03:16:55 DEBUG "data": { 03:16:55 DEBUG { 03:16:55 DEBUG "data": { 03:16:55 DEBUG "data": { 03:16:55 DEBUG "data": { 03:16:55 DEBUG "data": { 03:16:55 DEBUG { 03:16:55 DEBUG "data": { 03:16:55 DEBUG "resultType": "vector", 03:16:55 DEBUG "data": { 03:16:55 DEBUG "data": { 03:16:55 DEBUG "resultType": "vector", 03:16:55 DEBUG { 03:16:55 DEBUG "metric": { 03:16:55 DEBUG "pod": "am-767f6b67b5-9zw22" 03:16:55 DEBUG "result": [ 03:16:55 DEBUG "resultType": "vector", 03:16:55 DEBUG "resultType": "vector", 03:16:55 DEBUG "resultType": "vector", 03:16:55 DEBUG "data": { 03:16:55 DEBUG "resultType": "vector", 03:16:55 DEBUG "resultType": "vector", 03:16:55 DEBUG "result": [ 03:16:55 DEBUG "status": "success", 03:16:55 DEBUG "resultType": "vector", 03:16:55 DEBUG "resultType": "vector", 03:16:55 DEBUG "resultType": "vector", 03:16:55 DEBUG "resultType": "vector", 03:16:55 DEBUG "status": "success", 03:16:55 DEBUG "data": { 03:16:55 DEBUG "result": [ 03:16:55 DEBUG "resultType": "vector", 03:16:55 DEBUG "status": "success", 03:16:55 DEBUG "result": [ 03:16:55 DEBUG { 03:16:55 DEBUG }, 03:16:55 DEBUG "status": "success", 03:16:55 DEBUG { 03:16:55 DEBUG "result": [ 03:16:55 DEBUG "result": [ 03:16:55 DEBUG "result": [ 03:16:55 DEBUG "resultType": "vector", 03:16:55 DEBUG "result": [ 03:16:55 DEBUG "resultType": "vector", 03:16:55 DEBUG { 03:16:55 DEBUG "data": { 03:16:55 DEBUG "result": [ 03:16:55 DEBUG "result": [ 03:16:55 DEBUG "result": [ 03:16:55 DEBUG "result": [ 03:16:55 DEBUG "resultType": "vector", 03:16:55 DEBUG "resultType": "vector", 03:16:55 DEBUG { 03:16:55 DEBUG "result": [ 03:16:55 DEBUG "data": { 03:16:55 DEBUG { 03:16:55 DEBUG "status": "success", 03:16:55 DEBUG "value": [ 03:16:55 DEBUG "data": { 03:16:55 DEBUG "metric": { 03:16:55 DEBUG { 03:16:55 DEBUG { 03:16:55 DEBUG { 03:16:55 DEBUG "result": [ 03:16:55 DEBUG { 03:16:55 DEBUG "result": [ 03:16:55 DEBUG "metric": { 03:16:55 DEBUG "resultType": "vector", 03:16:55 DEBUG { 03:16:55 DEBUG { 03:16:55 DEBUG { 03:16:55 DEBUG { 03:16:55 DEBUG "result": [ 03:16:55 DEBUG "result": [ 03:16:55 DEBUG { 03:16:55 DEBUG { 03:16:55 DEBUG "metric": { 03:16:55 DEBUG "metric": { 03:16:55 DEBUG "data": { 03:16:55 DEBUG 03:16:55 DEBUG "resultType": "vector", 03:16:55 DEBUG "instance": "10.142.0.107:9100" 03:16:55 DEBUG }, 03:16:55 DEBUG "metric": { 03:16:55 DEBUG "metric": { 03:16:55 DEBUG { 03:16:55 DEBUG "metric": { 03:16:55 DEBUG { 03:16:55 DEBUG "pod": "am-767f6b67b5-9zw22" 03:16:55 DEBUG "result": [ 03:16:55 DEBUG "metric": { 03:16:55 DEBUG "metric": { 03:16:55 DEBUG "metric": { 03:16:55 DEBUG "metric": { 03:16:55 DEBUG { 03:16:55 DEBUG "metric": { 03:16:55 DEBUG "metric": { 03:16:55 DEBUG "resultType": "vector", 03:16:55 DEBUG "result": [ 03:16:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:16:55 DEBUG "resultType": "vector", 03:16:55 DEBUG "result": [ 03:16:55 DEBUG "result": [ 03:16:55 DEBUG "value": [ 03:16:55 DEBUG "metric": { 03:16:55 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:16:55 DEBUG "pod": "am-767f6b67b5-9zw22" 03:16:55 DEBUG "metric": { 03:16:55 DEBUG "instance": "10.142.0.107:9100" 03:16:55 DEBUG "metric": { 03:16:55 DEBUG }, 03:16:55 DEBUG "value": [ 03:16:55 DEBUG "pod": "am-767f6b67b5-9zw22" 03:16:55 DEBUG "pod": "am-767f6b67b5-9zw22" 03:16:55 DEBUG "pod": "am-767f6b67b5-9zw22" 03:16:55 DEBUG "pod": "ds-idrepo-0" 03:16:55 DEBUG "metric": { 03:16:55 DEBUG { 03:16:55 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:16:55 DEBUG "pod": "ds-idrepo-0" 03:16:55 DEBUG }, 03:16:55 DEBUG "value": [ 03:16:55 DEBUG 03:16:55 DEBUG { 03:16:55 DEBUG { 03:16:55 DEBUG 03:16:55 DEBUG "pod": "ds-idrepo-0" 03:16:55 DEBUG }, 03:16:55 DEBUG }, 03:16:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:16:55 DEBUG }, 03:16:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:16:55 DEBUG 03:16:55 DEBUG }, 03:16:55 DEBUG }, 03:16:55 DEBUG }, 03:16:55 DEBUG }, 03:16:55 DEBUG "instance": "10.142.0.107:9100" 03:16:55 DEBUG "pod": "ds-idrepo-0" 03:16:55 DEBUG "metric": { 03:16:55 DEBUG }, 03:16:55 DEBUG }, 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG "metric": { 03:16:55 DEBUG "metric": { 03:16:55 DEBUG { 03:16:55 DEBUG 03:16:55 DEBUG }, 03:16:55 DEBUG "value": [ 03:16:55 DEBUG "value": [ 03:16:55 DEBUG }, 03:16:55 DEBUG "value": [ 03:16:55 DEBUG }, 03:16:55 DEBUG 03:16:55 DEBUG "value": [ 03:16:55 DEBUG "value": [ 03:16:55 DEBUG "value": [ 03:16:55 DEBUG "value": [ 03:16:55 DEBUG }, 03:16:55 DEBUG }, 03:16:55 DEBUG "instance": "10.142.0.107:9100" 03:16:55 DEBUG "value": [ 03:16:55 DEBUG "value": [ 03:16:55 DEBUG 03:16:55 DEBUG [...] 03:16:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:16:55 DEBUG "pod": "ds-cts-0" 03:16:55 DEBUG "metric": { 03:16:55 DEBUG 03:16:55 DEBUG "value": [ 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG "value": [ 03:16:55 DEBUG 03:16:55 DEBUG "value": [ 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG "value": [ 03:16:55 DEBUG "value": [ 03:16:55 DEBUG }, 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG }, 03:16:55 DEBUG }, 03:16:55 DEBUG "pod": "ds-cts-0" 03:16:55 DEBUG [...] 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG [...] 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG "value": [ 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG [...] 03:16:55 DEBUG 03:16:55 DEBUG "value": [ 03:16:55 DEBUG "value": [ 03:16:55 DEBUG }, 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG "value": [ 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG [...] 03:16:55 DEBUG [...] 03:16:55 DEBUG 03:16:55 DEBUG [...] 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG [...] 03:16:55 DEBUG [...] 03:16:55 DEBUG 03:16:55 DEBUG [...] 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG [...] 03:16:55 DEBUG [...] 03:16:55 DEBUG 03:16:55 DEBUG "pod": "am-767f6b67b5-vbltz" 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG [...] 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG [...] 03:16:55 DEBUG 03:16:55 DEBUG [...] 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG [...] 03:16:55 DEBUG 03:16:55 DEBUG [...] 03:16:55 DEBUG [...] 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG }, 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG "instance": "10.142.0.43:9100" 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG "pod": "am-767f6b67b5-vbltz" 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG "pod": "am-767f6b67b5-vbltz" 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG [...] 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG "value": [ 03:16:55 DEBUG "pod": "ds-idrepo-2" 03:16:55 DEBUG [...] 03:16:55 DEBUG 03:16:55 DEBUG }, 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG "pod": "am-767f6b67b5-vbltz" 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG }, 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG }, 03:16:55 DEBUG "pod": "ds-cts-2" 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:16:55 DEBUG [...] 03:16:55 DEBUG 1713665815, 03:16:55 DEBUG }, 03:16:55 DEBUG 03:16:55 DEBUG [...] 03:16:55 DEBUG "value": [ 03:16:55 DEBUG 03:16:55 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:16:55 DEBUG }, 03:16:55 DEBUG 03:16:55 DEBUG "instance": "10.142.0.43:9100" 03:16:55 DEBUG 03:16:55 DEBUG "value": [ 03:16:55 DEBUG "pod": "am-767f6b67b5-vbltz" 03:16:55 DEBUG 03:16:55 DEBUG "value": [ 03:16:55 DEBUG }, 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:16:55 DEBUG }, 03:16:55 DEBUG "value": [ 03:16:55 DEBUG "599809.5" 03:16:55 DEBUG "value": [ 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG 1713665815, 03:16:55 DEBUG "pod": "ds-idrepo-2" 03:16:55 DEBUG }, 03:16:55 DEBUG "value": [ 03:16:55 DEBUG 03:16:55 DEBUG }, 03:16:55 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:16:55 DEBUG 1713665815, 03:16:55 DEBUG }, 03:16:55 DEBUG 03:16:55 DEBUG 1713665815, 03:16:55 DEBUG "value": [ 03:16:55 DEBUG 03:16:55 DEBUG "pod": "ds-idrepo-2" 03:16:55 DEBUG 03:16:55 DEBUG 1713665815, 03:16:55 DEBUG 03:16:55 DEBUG }, 03:16:55 DEBUG "value": [ 03:16:55 DEBUG 1713665815, 03:16:55 DEBUG 03:16:55 DEBUG 03:16:55 DEBUG "0.08166666666666628" 03:16:55 DEBUG "value": [ 03:16:55 DEBUG }, 03:16:55 DEBUG 1713665815, 03:16:55 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:16:55 DEBUG "value": [ 03:16:55 DEBUG }, 03:16:55 DEBUG "3312.5666666666666" 03:16:55 DEBUG "value": [ 03:16:55 DEBUG "pod": "am-767f6b67b5-vbltz" 03:16:55 DEBUG "0" 03:16:55 DEBUG 1713665815, 03:16:55 DEBUG "instance": "10.142.0.43:9100" 03:16:55 DEBUG }, 03:16:55 DEBUG "instance": "10.142.0.43:9100" 03:16:55 DEBUG "1025400832" 03:16:55 DEBUG 03:16:55 DEBUG 1713665815, 03:16:55 DEBUG ] 03:16:55 DEBUG "0" 03:16:55 DEBUG "pod": "ds-cts-2" 03:16:55 DEBUG 03:16:55 DEBUG ] 03:16:55 DEBUG 1713665815, 03:16:55 DEBUG "value": [ 03:16:55 DEBUG "149.889" 03:16:55 DEBUG }, 03:16:55 DEBUG 1713665815, 03:16:55 DEBUG "value": [ 03:16:55 DEBUG ] 03:16:55 DEBUG 1713665815, 03:16:55 DEBUG }, 03:16:55 DEBUG ] 03:16:55 DEBUG } 03:16:55 DEBUG }, 03:16:55 DEBUG "value": [ 03:16:55 DEBUG }, 03:16:55 DEBUG ] 03:16:55 DEBUG 03:16:55 DEBUG "0" 03:16:55 DEBUG ] 03:16:55 DEBUG ] 03:16:55 DEBUG }, 03:16:55 DEBUG "pod": "ds-cts-2" 03:16:55 DEBUG } 03:16:55 DEBUG "0" 03:16:55 DEBUG 1713665815, 03:16:55 DEBUG ] 03:16:55 DEBUG "value": [ 03:16:55 DEBUG "4381071043584" 03:16:55 DEBUG 1713665815, 03:16:55 DEBUG } 03:16:55 DEBUG "3327.4666666666667" 03:16:55 DEBUG "value": [ 03:16:55 DEBUG ] 03:16:55 DEBUG "0" 03:16:55 DEBUG "value": [ 03:16:55 DEBUG 1713665815, 03:16:55 DEBUG "value": [ 03:16:55 DEBUG } 03:16:55 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:16:55 DEBUG } 03:16:55 DEBUG } 03:16:55 DEBUG } 03:16:55 DEBUG "value": [ 03:16:55 DEBUG }, 03:16:55 DEBUG "value": [ 03:16:55 DEBUG ] 03:16:55 DEBUG "11.760005095389584" 03:16:55 DEBUG } 03:16:55 DEBUG 1713665815, 03:16:55 DEBUG "0" 03:16:55 DEBUG "25447043.922718495" 03:16:55 DEBUG ] 03:16:55 DEBUG ] 03:16:55 DEBUG 1713665815, 03:16:55 DEBUG } 03:16:55 DEBUG ] 03:16:55 DEBUG 1713665815, 03:16:55 DEBUG "0" 03:16:55 DEBUG 1713665815, 03:16:55 DEBUG ] 03:16:55 DEBUG }, 03:16:55 DEBUG ] 03:16:55 DEBUG ] 03:16:55 DEBUG ] 03:16:55 DEBUG 1713665815, 03:16:55 DEBUG ] 03:16:55 DEBUG 1713665815, 03:16:55 DEBUG } 03:16:55 DEBUG ] 03:16:55 DEBUG ] 03:16:55 DEBUG ] 03:16:55 DEBUG ] 03:16:55 DEBUG ] 03:16:55 DEBUG } 03:16:55 DEBUG } 03:16:55 DEBUG "3327.9333333333334" 03:16:55 DEBUG } 03:16:55 DEBUG } 03:16:55 DEBUG "100201518.83333333" 03:16:55 DEBUG ] 03:16:55 DEBUG "100200107.19999999" 03:16:55 DEBUG } 03:16:55 DEBUG "value": [ 03:16:55 DEBUG } 03:16:55 DEBUG } 03:16:55 DEBUG } 03:16:55 DEBUG "0" 03:16:55 DEBUG } 03:16:55 DEBUG "29.833333333333332" 03:16:55 DEBUG ] 03:16:55 DEBUG } 03:16:55 DEBUG } 03:16:55 DEBUG } 03:16:55 DEBUG } 03:16:55 DEBUG } 03:16:55 DEBUG } 03:16:55 DEBUG ] 03:16:55 DEBUG ] 03:16:55 DEBUG ] 03:16:55 DEBUG ] 03:16:55 DEBUG } 03:16:55 DEBUG ] 03:16:55 DEBUG } 03:16:55 DEBUG 1713665815, 03:16:55 DEBUG "5.936037927583378" 03:16:55 DEBUG } 03:16:55 DEBUG } 03:16:55 DEBUG ] 03:16:55 DEBUG } 03:16:55 DEBUG ] 03:16:55 DEBUG } 03:16:55 DEBUG ] 03:16:55 DEBUG } 03:16:55 DEBUG ] 03:16:55 DEBUG ] 03:16:55 DEBUG ] 03:16:55 DEBUG } 03:16:55 DEBUG } 03:16:55 DEBUG } 03:16:55 DEBUG ] 03:16:55 DEBUG } 03:16:55 DEBUG } 03:16:55 DEBUG } 03:16:55 DEBUG } 03:16:55 DEBUG ] 03:16:55 DEBUG } 03:16:55 DEBUG } 03:16:55 DEBUG } 03:16:55 DEBUG } 03:16:55 DEBUG } 03:16:55 DEBUG } 03:16:55 DEBUG } 03:16:55 DEBUG } 03:16:55 DEBUG ] 03:16:55 DEBUG } 03:16:55 DEBUG } 03:16:55 DEBUG ] 03:16:55 DEBUG } 03:16:55 DEBUG ] 03:16:55 DEBUG ] 03:16:55 DEBUG } 03:16:55 DEBUG } 03:16:55 DEBUG } 03:16:55 DEBUG } 03:16:55 DEBUG } 03:16:55 DEBUG ] 03:16:55 DEBUG } 03:16:55 DEBUG ] 03:16:55 DEBUG } 03:16:55 DEBUG } 03:16:55 DEBUG } 03:16:55 DEBUG } 03:16:55 DEBUG } 03:16:55 DEBUG } 03:16:55 DEBUG } 03:16:55 DEBUG } 03:16:55 DEBUG } 03:16:55 DEBUG } 03:17:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713665873" 03:17:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665873" 03:17:53 ERROR [http_cmd]: ERROR 03:17:53 ERROR --- status code --- 03:17:53 ERROR http status code is 400 (expected [200, 401, 403, 404]) 03:17:53 ERROR --- http response --- 03:17:53 INFO [http_cmd]: http status code OK 03:17:53 DEBUG --- status code --- 03:17:53 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 03:17:53 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 03:17:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:17:53 DEBUG --- http response --- 03:17:53 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 03:17:54 INFO 03:17:54 INFO [loop_until]: kubectl --namespace=xlou top pods 03:17:54 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:17:54 INFO [loop_until]: OK (rc = 0) 03:17:54 DEBUG --- stdout --- 03:17:54 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 21855m 8795Mi am-767f6b67b5-qcqz2 22371m 9127Mi am-767f6b67b5-vbltz 24464m 8823Mi ds-cts-0 19428m 12454Mi ds-cts-1 19677m 12481Mi ds-cts-2 19838m 12543Mi ds-idrepo-0 9586m 3918Mi ds-idrepo-1 9493m 3946Mi ds-idrepo-2 9570m 3915Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1179Mi idm-b85f9d978-wnhxb 3m 1239Mi lodemon-54c87f8cbb-jzrjs 17m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-68c5d5b884-p97w5 12150m 888Mi 03:17:54 DEBUG --- stderr --- 03:17:54 DEBUG 03:17:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665874" 03:17:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665874" 03:17:54 INFO [http_cmd]: http status code OK 03:17:54 DEBUG --- status code --- 03:17:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:17:54 DEBUG --- http response --- 03:17:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:17:54 DEBUG ----- output ----- 03:17:54 DEBUG { 03:17:54 DEBUG "status": "success", 03:17:54 DEBUG "data": { 03:17:54 DEBUG "resultType": "vector", 03:17:54 DEBUG "result": [ 03:17:54 DEBUG { 03:17:54 DEBUG "metric": { 03:17:54 DEBUG "pod": "ds-cts-0" 03:17:54 DEBUG }, 03:17:54 DEBUG "value": [ 03:17:54 DEBUG 03:17:54 DEBUG 03:17:54 DEBUG 03:17:54 DEBUG [...] 03:17:54 DEBUG 03:17:54 DEBUG 03:17:54 DEBUG 03:17:54 DEBUG "pod": "ds-cts-2" 03:17:54 DEBUG }, 03:17:54 DEBUG "value": [ 03:17:54 DEBUG 1713665874, 03:17:54 DEBUG "0.5400333333333349" 03:17:54 DEBUG ] 03:17:54 DEBUG } 03:17:54 DEBUG ] 03:17:54 DEBUG } 03:17:54 DEBUG } 03:17:54 INFO [http_cmd]: http status code OK 03:17:54 DEBUG --- status code --- 03:17:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:17:54 DEBUG --- http response --- 03:17:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:17:54 DEBUG ----- output ----- 03:17:54 DEBUG { 03:17:54 DEBUG "status": "success", 03:17:54 DEBUG "data": { 03:17:54 DEBUG "resultType": "vector", 03:17:54 DEBUG "result": [ 03:17:54 DEBUG { 03:17:54 DEBUG "metric": { 03:17:54 DEBUG "instance": "10.142.0.107:9100" 03:17:54 DEBUG }, 03:17:54 DEBUG "value": [ 03:17:54 DEBUG 03:17:54 DEBUG 03:17:54 DEBUG 03:17:54 DEBUG [...] 03:17:54 DEBUG 03:17:54 DEBUG 03:17:54 DEBUG 03:17:54 DEBUG "instance": "10.142.0.43:9100" 03:17:54 DEBUG }, 03:17:54 DEBUG "value": [ 03:17:54 DEBUG 1713665874, 03:17:54 DEBUG "0.3790333333333384" 03:17:54 DEBUG ] 03:17:54 DEBUG } 03:17:54 DEBUG ] 03:17:54 DEBUG } 03:17:54 DEBUG } 03:17:54 INFO 03:17:54 INFO [loop_until]: kubectl --namespace=xlou top node 03:17:54 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:17:55 INFO [loop_until]: OK (rc = 0) 03:17:55 DEBUG --- stdout --- 03:17:55 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 6271Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 9940m 33% 7765Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 22039m 73% 11603Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 83m 0% 3356Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 19089m 63% 15777Mi 14% gke-xlou-cdm-default-pool-1b0769c8-wpwl 156m 0% 5267Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 21685m 72% 12065Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 19086m 63% 3175Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 24726m 82% 11088Mi 9% gke-xlou-cdm-ds-0f601751-j488 10412m 34% 7345Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 19682m 65% 17612Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 10161m 34% 8011Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20098m 67% 15268Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 11978m 75% 11239Mi 19% 03:17:55 DEBUG --- stderr --- 03:17:55 DEBUG 03:17:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665875" 03:17:55 INFO [http_cmd]: http status code OK 03:17:55 DEBUG --- status code --- 03:17:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:17:55 DEBUG --- http response --- 03:17:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:17:55 DEBUG ----- output ----- 03:17:55 DEBUG { 03:17:55 DEBUG "status": "success", 03:17:55 DEBUG "data": { 03:17:55 DEBUG "resultType": "vector", 03:17:55 DEBUG "result": [ 03:17:55 DEBUG { 03:17:55 DEBUG "metric": { 03:17:55 DEBUG "pod": "am-767f6b67b5-9zw22" 03:17:55 DEBUG }, 03:17:55 DEBUG "value": [ 03:17:55 DEBUG 03:17:55 DEBUG 03:17:55 DEBUG 03:17:55 DEBUG [...] 03:17:55 DEBUG 03:17:55 DEBUG 03:17:55 DEBUG 03:17:55 DEBUG "pod": "am-767f6b67b5-vbltz" 03:17:55 DEBUG }, 03:17:55 DEBUG "value": [ 03:17:55 DEBUG 1713665875, 03:17:55 DEBUG "3345.7" 03:17:55 DEBUG ] 03:17:55 DEBUG } 03:17:55 DEBUG ] 03:17:55 DEBUG } 03:17:55 DEBUG } 03:17:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665875" 03:17:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665875" 03:17:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665875" 03:17:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665875" 03:17:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665875" 03:17:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713665875" 03:17:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713665875" 03:17:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665875" 03:17:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665875" 03:17:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665875" 03:17:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665875" 03:17:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665875" 03:17:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665875" 03:17:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713665875" 03:17:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665875" 03:17:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665875" 03:17:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665875" 03:17:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713665875" 03:17:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665875" 03:17:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665875" 03:17:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665875" 03:17:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665875" 03:17:55 INFO [http_cmd]: http status code OK 03:17:55 INFO [http_cmd]: http status code OK 03:17:55 INFO [http_cmd]: http status code OK 03:17:55 INFO [http_cmd]: http status code OK 03:17:55 DEBUG --- status code --- 03:17:55 INFO [http_cmd]: http status code OK 03:17:55 INFO [http_cmd]: http status code OK 03:17:55 INFO [http_cmd]: http status code OK 03:17:55 INFO [http_cmd]: http status code OK 03:17:55 DEBUG --- status code --- 03:17:55 INFO [http_cmd]: http status code OK 03:17:55 INFO [http_cmd]: http status code OK 03:17:55 INFO [http_cmd]: http status code OK 03:17:55 INFO [http_cmd]: http status code OK 03:17:55 INFO [http_cmd]: http status code OK 03:17:55 INFO [http_cmd]: http status code OK 03:17:55 DEBUG --- status code --- 03:17:55 INFO [http_cmd]: http status code OK 03:17:55 DEBUG --- status code --- 03:17:55 INFO [http_cmd]: http status code OK 03:17:55 DEBUG --- status code --- 03:17:55 INFO [http_cmd]: http status code OK 03:17:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:17:55 INFO [http_cmd]: http status code OK 03:17:55 DEBUG --- status code --- 03:17:55 DEBUG --- status code --- 03:17:55 DEBUG --- status code --- 03:17:55 INFO [http_cmd]: http status code OK 03:17:55 INFO [http_cmd]: http status code OK 03:17:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:17:55 DEBUG --- status code --- 03:17:55 INFO [http_cmd]: http status code OK 03:17:55 DEBUG --- status code --- 03:17:55 DEBUG --- status code --- 03:17:55 DEBUG --- status code --- 03:17:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:17:55 DEBUG --- status code --- 03:17:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:17:55 DEBUG --- status code --- 03:17:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:17:55 DEBUG --- status code --- 03:17:55 INFO [http_cmd]: http status code OK 03:17:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:17:55 DEBUG --- status code --- 03:17:55 DEBUG --- http response --- 03:17:55 DEBUG --- status code --- 03:17:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:17:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:17:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:17:55 DEBUG --- status code --- 03:17:55 DEBUG --- status code --- 03:17:55 DEBUG --- http response --- 03:17:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:17:55 DEBUG --- status code --- 03:17:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:17:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:17:55 DEBUG --- http response --- 03:17:55 DEBUG --- status code --- 03:17:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:17:55 DEBUG --- http response --- 03:17:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:17:55 DEBUG --- http response --- 03:17:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:17:55 DEBUG --- status code --- 03:17:55 DEBUG --- http response --- 03:17:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:17:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:17:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:17:55 DEBUG --- http response --- 03:17:55 DEBUG --- http response --- 03:17:55 DEBUG --- http response --- 03:17:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:17:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:17:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:17:55 DEBUG --- http response --- 03:17:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:17:55 DEBUG --- http response --- 03:17:55 DEBUG --- http response --- 03:17:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 03:17:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:17:55 DEBUG --- http response --- 03:17:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:17:55 DEBUG --- http response --- 03:17:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:17:55 DEBUG --- http response --- 03:17:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:17:55 DEBUG --- http response --- 03:17:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:17:55 DEBUG ----- output ----- 03:17:55 DEBUG --- http response --- 03:17:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:17:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:17:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:17:55 DEBUG --- http response --- 03:17:55 DEBUG --- http response --- 03:17:55 DEBUG ----- output ----- 03:17:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 03:17:55 DEBUG --- http response --- 03:17:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:17:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 03:17:55 DEBUG ----- output ----- 03:17:55 DEBUG --- http response --- 03:17:55 DEBUG ----- output ----- 03:17:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:17:55 DEBUG ----- output ----- 03:17:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:17:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:17:55 DEBUG --- http response --- 03:17:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:17:55 DEBUG { 03:17:55 DEBUG ----- output ----- 03:17:55 DEBUG ----- output ----- 03:17:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:17:55 DEBUG ----- output ----- 03:17:55 DEBUG ----- output ----- 03:17:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:17:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:17:55 DEBUG { 03:17:55 DEBUG ----- output ----- 03:17:55 DEBUG ----- output ----- 03:17:55 DEBUG ----- output ----- 03:17:55 DEBUG { 03:17:55 DEBUG { 03:17:55 DEBUG { 03:17:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:17:55 DEBUG ----- output ----- 03:17:55 DEBUG { 03:17:55 DEBUG ----- output ----- 03:17:55 DEBUG ----- output ----- 03:17:55 DEBUG ----- output ----- 03:17:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 03:17:55 DEBUG "status": "success", 03:17:55 DEBUG { 03:17:55 DEBUG { 03:17:55 DEBUG ----- output ----- 03:17:55 DEBUG { 03:17:55 DEBUG "status": "success", 03:17:55 DEBUG ----- output ----- 03:17:55 DEBUG ----- output ----- 03:17:55 DEBUG "status": "success", 03:17:55 DEBUG { 03:17:55 DEBUG "status": "success", 03:17:55 DEBUG "data": { 03:17:55 DEBUG "status": "success", 03:17:55 DEBUG "status": "success", 03:17:55 DEBUG "status": "success", 03:17:55 DEBUG ----- output ----- 03:17:55 DEBUG { 03:17:55 DEBUG "status": "success", 03:17:55 DEBUG { 03:17:55 DEBUG "status": "success", 03:17:55 DEBUG { 03:17:55 DEBUG ----- output ----- 03:17:55 DEBUG "data": { 03:17:55 DEBUG "status": "success", 03:17:55 DEBUG "status": "success", 03:17:55 DEBUG { 03:17:55 DEBUG "data": { 03:17:55 DEBUG { 03:17:55 DEBUG { 03:17:55 DEBUG { 03:17:55 DEBUG "data": { 03:17:55 DEBUG "resultType": "vector", 03:17:55 DEBUG { 03:17:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:17:55 DEBUG "data": { 03:17:55 DEBUG "data": { 03:17:55 DEBUG "data": { 03:17:55 DEBUG { 03:17:55 DEBUG "status": "success", 03:17:55 DEBUG "data": { 03:17:55 DEBUG { 03:17:55 DEBUG "status": "success", 03:17:55 DEBUG "status": "success", 03:17:55 DEBUG { 03:17:55 DEBUG "resultType": "vector", 03:17:55 DEBUG "data": { 03:17:55 DEBUG "resultType": "vector", 03:17:55 DEBUG "status": "success", 03:17:55 DEBUG "resultType": "vector", 03:17:55 DEBUG "status": "success", 03:17:55 DEBUG "data": { 03:17:55 DEBUG "resultType": "vector", 03:17:55 DEBUG "resultType": "vector", 03:17:55 DEBUG "result": [ 03:17:55 DEBUG "status": "success", 03:17:55 DEBUG ----- output ----- 03:17:55 DEBUG "resultType": "vector", 03:17:55 DEBUG "resultType": "vector", 03:17:55 DEBUG "resultType": "vector", 03:17:55 DEBUG "status": "success", 03:17:55 DEBUG "data": { 03:17:55 DEBUG "resultType": "vector", 03:17:55 DEBUG "data": { 03:17:55 DEBUG "data": { 03:17:55 DEBUG "data": { 03:17:55 DEBUG "status": "success", 03:17:55 DEBUG "result": [ 03:17:55 DEBUG "result": [ 03:17:55 DEBUG "data": { 03:17:55 DEBUG "data": { 03:17:55 DEBUG "result": [ 03:17:55 DEBUG "status": "success", 03:17:55 DEBUG "result": [ 03:17:55 DEBUG "status": "success", 03:17:55 DEBUG "result": [ 03:17:55 DEBUG { 03:17:55 DEBUG "data": { 03:17:55 DEBUG { 03:17:55 DEBUG "result": [ 03:17:55 DEBUG "result": [ 03:17:55 DEBUG "result": [ 03:17:55 DEBUG "data": { 03:17:55 DEBUG "resultType": "vector", 03:17:55 DEBUG "result": [ 03:17:55 DEBUG "resultType": "vector", 03:17:55 DEBUG "result": [ 03:17:55 DEBUG "resultType": "vector", 03:17:55 DEBUG "data": { 03:17:55 DEBUG "resultType": "vector", 03:17:55 DEBUG { 03:17:55 DEBUG "resultType": "vector", 03:17:55 DEBUG "resultType": "vector", 03:17:55 DEBUG { 03:17:55 DEBUG "data": { 03:17:55 DEBUG { 03:17:55 DEBUG "data": { 03:17:55 DEBUG { 03:17:55 DEBUG "metric": { 03:17:55 DEBUG "resultType": "vector", 03:17:55 DEBUG "status": "success", 03:17:55 DEBUG { 03:17:55 DEBUG { 03:17:55 DEBUG { 03:17:55 DEBUG "resultType": "vector", 03:17:55 DEBUG "result": [ 03:17:55 DEBUG { 03:17:55 DEBUG { 03:17:55 DEBUG "resultType": "vector", 03:17:55 DEBUG "result": [ 03:17:55 DEBUG { 03:17:55 DEBUG "result": [ 03:17:55 DEBUG "metric": { 03:17:55 DEBUG "result": [ 03:17:55 DEBUG "result": [ 03:17:55 DEBUG "metric": { 03:17:55 DEBUG "resultType": "vector", 03:17:55 DEBUG "metric": { 03:17:55 DEBUG "resultType": "vector", 03:17:55 DEBUG "metric": { 03:17:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:17:55 DEBUG }, 03:17:55 DEBUG "data": { 03:17:55 DEBUG "metric": { 03:17:55 DEBUG "metric": { 03:17:55 DEBUG "metric": { 03:17:55 DEBUG "result": [ 03:17:55 DEBUG { 03:17:55 DEBUG "metric": { 03:17:55 DEBUG "metric": { 03:17:55 DEBUG "result": [ 03:17:55 DEBUG { 03:17:55 DEBUG "metric": { 03:17:55 DEBUG { 03:17:55 DEBUG "pod": "ds-idrepo-0" 03:17:55 DEBUG { 03:17:55 DEBUG { 03:17:55 DEBUG "pod": "am-767f6b67b5-9zw22" 03:17:55 DEBUG "result": [ 03:17:55 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:17:55 DEBUG }, 03:17:55 DEBUG "instance": "10.142.0.107:9100" 03:17:55 DEBUG "value": [ 03:17:55 DEBUG "resultType": "vector", 03:17:55 DEBUG "result": [ 03:17:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:17:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:17:55 DEBUG "pod": "am-767f6b67b5-9zw22" 03:17:55 DEBUG { 03:17:55 DEBUG "metric": { 03:17:55 DEBUG "pod": "am-767f6b67b5-9zw22" 03:17:55 DEBUG "pod": "am-767f6b67b5-9zw22" 03:17:55 DEBUG { 03:17:55 DEBUG "metric": { 03:17:55 DEBUG "instance": "10.142.0.107:9100" 03:17:55 DEBUG "metric": { 03:17:55 DEBUG }, 03:17:55 DEBUG "metric": { 03:17:55 DEBUG "metric": { 03:17:55 DEBUG "pod": "am-767f6b67b5-9zw22" 03:17:55 DEBUG { 03:17:55 DEBUG "value": [ 03:17:55 DEBUG "result": [ 03:17:55 DEBUG }, 03:17:55 DEBUG 03:17:55 DEBUG "result": [ 03:17:55 DEBUG { 03:17:55 DEBUG "metric": { 03:17:55 DEBUG }, 03:17:55 DEBUG }, 03:17:55 DEBUG "metric": { 03:17:55 DEBUG "pod": "ds-idrepo-0" 03:17:55 DEBUG }, 03:17:55 DEBUG }, 03:17:55 DEBUG "metric": { 03:17:55 DEBUG "pod": "ds-idrepo-0" 03:17:55 DEBUG }, 03:17:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:17:55 DEBUG "value": [ 03:17:55 DEBUG "pod": "am-767f6b67b5-9zw22" 03:17:55 DEBUG }, 03:17:55 DEBUG }, 03:17:55 DEBUG "metric": { 03:17:55 DEBUG 03:17:55 DEBUG { 03:17:55 DEBUG "value": [ 03:17:55 DEBUG 03:17:55 DEBUG }, 03:17:55 DEBUG "instance": "10.142.0.107:9100" 03:17:55 DEBUG { 03:17:55 DEBUG "value": [ 03:17:55 DEBUG "value": [ 03:17:55 DEBUG "pod": "ds-cts-0" 03:17:55 DEBUG }, 03:17:55 DEBUG "value": [ 03:17:55 DEBUG "value": [ 03:17:55 DEBUG "pod": "ds-idrepo-0" 03:17:55 DEBUG }, 03:17:55 DEBUG "value": [ 03:17:55 DEBUG }, 03:17:55 DEBUG 03:17:55 DEBUG }, 03:17:55 DEBUG "value": [ 03:17:55 DEBUG "value": [ 03:17:55 DEBUG "instance": "10.142.0.107:9100" 03:17:55 DEBUG 03:17:55 DEBUG "metric": { 03:17:55 DEBUG 03:17:55 DEBUG 03:17:55 DEBUG "value": [ 03:17:55 DEBUG }, 03:17:55 DEBUG "value": [ 03:17:55 DEBUG 03:17:55 DEBUG 03:17:55 DEBUG }, 03:17:55 DEBUG "value": [ 03:17:55 DEBUG 03:17:55 DEBUG 03:17:55 DEBUG }, 03:17:55 DEBUG "value": [ 03:17:55 DEBUG 03:17:55 DEBUG "value": [ 03:17:55 DEBUG 03:17:55 DEBUG "value": [ 03:17:55 DEBUG 03:17:55 DEBUG 03:17:55 DEBUG }, 03:17:55 DEBUG 03:17:55 DEBUG "pod": "ds-cts-0" 03:17:55 DEBUG 03:17:55 DEBUG [...] 03:17:55 DEBUG 03:17:55 DEBUG "metric": { 03:17:55 DEBUG 03:17:55 DEBUG 03:17:55 DEBUG 03:17:55 DEBUG "value": [ 03:17:55 DEBUG 03:17:55 DEBUG 03:17:55 DEBUG 03:17:55 DEBUG "value": [ 03:17:55 DEBUG 03:17:55 DEBUG 03:17:55 DEBUG 03:17:55 DEBUG 03:17:55 DEBUG [...] 03:17:55 DEBUG 03:17:55 DEBUG 03:17:55 DEBUG "value": [ 03:17:55 DEBUG [...] 03:17:55 DEBUG }, 03:17:55 DEBUG 03:17:55 DEBUG 03:17:55 DEBUG 03:17:55 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:17:55 DEBUG 03:17:55 DEBUG 03:17:55 DEBUG 03:17:55 DEBUG 03:17:55 DEBUG 03:17:55 DEBUG 03:17:55 DEBUG 03:17:55 DEBUG 03:17:55 DEBUG 03:17:55 DEBUG 03:17:55 DEBUG 03:17:55 DEBUG 03:17:55 DEBUG 03:17:55 DEBUG 03:17:55 DEBUG 03:17:55 DEBUG 03:17:55 DEBUG 03:17:55 DEBUG 03:17:55 DEBUG [...] 03:17:55 DEBUG 03:17:55 DEBUG 03:17:55 DEBUG }, 03:17:55 DEBUG [...] 03:17:55 DEBUG [...] 03:17:55 DEBUG 03:17:55 DEBUG 03:17:55 DEBUG 03:17:55 DEBUG [...] 03:17:55 DEBUG [...] 03:17:55 DEBUG 03:17:55 DEBUG 03:17:55 DEBUG [...] 03:17:55 DEBUG 03:17:55 DEBUG 03:17:55 DEBUG 03:17:55 DEBUG [...] 03:17:55 DEBUG [...] 03:17:55 DEBUG 03:17:55 DEBUG 03:17:55 DEBUG 03:17:55 DEBUG "value": [ 03:17:55 DEBUG 03:17:55 DEBUG [...] 03:17:55 DEBUG "value": [ 03:17:55 DEBUG 03:17:55 DEBUG 03:17:55 DEBUG 03:17:55 DEBUG 03:17:55 DEBUG [...] 03:17:55 DEBUG 03:17:55 DEBUG 03:17:55 DEBUG 03:17:55 DEBUG [...] 03:17:55 DEBUG 03:17:55 DEBUG [...] 03:17:55 DEBUG 03:17:55 DEBUG 03:17:55 DEBUG 03:17:55 DEBUG 03:17:55 DEBUG 03:17:55 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:17:55 DEBUG 03:17:55 DEBUG 03:17:55 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:17:55 DEBUG }, 03:17:55 DEBUG 03:17:55 DEBUG 03:17:55 DEBUG [...] 03:17:55 DEBUG 03:17:55 DEBUG "pod": "am-767f6b67b5-vbltz" 03:17:55 DEBUG 03:17:55 DEBUG 03:17:55 DEBUG 03:17:55 DEBUG [...] 03:17:55 DEBUG 03:17:55 DEBUG 03:17:55 DEBUG 03:17:55 DEBUG "pod": "ds-idrepo-2" 03:17:55 DEBUG [...] 03:17:55 DEBUG 03:17:55 DEBUG 03:17:55 DEBUG [...] 03:17:55 DEBUG }, 03:17:55 DEBUG 03:17:55 DEBUG 03:17:55 DEBUG "value": [ 03:17:55 DEBUG 03:17:55 DEBUG 03:17:55 DEBUG 03:17:55 DEBUG 03:17:55 DEBUG [...] 03:17:55 DEBUG }, 03:17:55 DEBUG 03:17:55 DEBUG 03:17:55 DEBUG 03:17:55 DEBUG 03:17:55 DEBUG 03:17:55 DEBUG 03:17:55 DEBUG 03:17:55 DEBUG }, 03:17:55 DEBUG 03:17:55 DEBUG 03:17:55 DEBUG 03:17:55 DEBUG 03:17:55 DEBUG "value": [ 03:17:55 DEBUG "instance": "10.142.0.43:9100" 03:17:55 DEBUG 03:17:55 DEBUG 1713665875, 03:17:55 DEBUG 03:17:55 DEBUG 03:17:55 DEBUG "instance": "10.142.0.43:9100" 03:17:55 DEBUG 03:17:55 DEBUG "value": [ 03:17:55 DEBUG 03:17:55 DEBUG "pod": "am-767f6b67b5-vbltz" 03:17:55 DEBUG "pod": "ds-idrepo-2" 03:17:55 DEBUG "pod": "am-767f6b67b5-vbltz" 03:17:55 DEBUG 03:17:55 DEBUG 03:17:55 DEBUG "instance": "10.142.0.43:9100" 03:17:55 DEBUG 03:17:55 DEBUG "value": [ 03:17:55 DEBUG 03:17:55 DEBUG 03:17:55 DEBUG "pod": "am-767f6b67b5-vbltz" 03:17:55 DEBUG }, 03:17:55 DEBUG 1713665875, 03:17:55 DEBUG }, 03:17:55 DEBUG [...] 03:17:55 DEBUG 03:17:55 DEBUG 03:17:55 DEBUG [...] 03:17:55 DEBUG }, 03:17:55 DEBUG 03:17:55 DEBUG 1713665875, 03:17:55 DEBUG 03:17:55 DEBUG }, 03:17:55 DEBUG }, 03:17:55 DEBUG }, 03:17:55 DEBUG 03:17:55 DEBUG "pod": "ds-idrepo-2" 03:17:55 DEBUG }, 03:17:55 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:17:55 DEBUG }, 03:17:55 DEBUG "pod": "am-767f6b67b5-vbltz" 03:17:55 DEBUG "pod": "am-767f6b67b5-vbltz" 03:17:55 DEBUG "value": [ 03:17:55 DEBUG 03:17:55 DEBUG "1027039232" 03:17:55 DEBUG "value": [ 03:17:55 DEBUG 03:17:55 DEBUG "0" 03:17:55 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:17:55 DEBUG 03:17:55 DEBUG "value": [ 03:17:55 DEBUG 03:17:55 DEBUG "0" 03:17:55 DEBUG 03:17:55 DEBUG "value": [ 03:17:55 DEBUG "value": [ 03:17:55 DEBUG "value": [ 03:17:55 DEBUG "pod": "ds-cts-2" 03:17:55 DEBUG }, 03:17:55 DEBUG "value": [ 03:17:55 DEBUG "value": [ 03:17:55 DEBUG 1713665875, 03:17:55 DEBUG }, 03:17:55 DEBUG }, 03:17:55 DEBUG 1713665875, 03:17:55 DEBUG 03:17:55 DEBUG ] 03:17:55 DEBUG 1713665875, 03:17:55 DEBUG 03:17:55 DEBUG ] 03:17:55 DEBUG }, 03:17:55 DEBUG 03:17:55 DEBUG 1713665875, 03:17:55 DEBUG "pod": "ds-cts-2" 03:17:55 DEBUG ] 03:17:55 DEBUG } 03:17:55 DEBUG 1713665875, 03:17:55 DEBUG 1713665875, 03:17:55 DEBUG 1713665875, 03:17:55 DEBUG }, 03:17:55 DEBUG "value": [ 03:17:55 DEBUG 1713665875, 03:17:55 DEBUG 1713665875, 03:17:55 DEBUG "0" 03:17:55 DEBUG "value": [ 03:17:55 DEBUG "value": [ 03:17:55 DEBUG "149.889" 03:17:55 DEBUG "instance": "10.142.0.43:9100" 03:17:55 DEBUG } 03:17:55 DEBUG "100426113.86666667" 03:17:55 DEBUG "pod": "ds-cts-2" 03:17:55 DEBUG } 03:17:55 DEBUG "value": [ 03:17:55 DEBUG 03:17:55 DEBUG "100423230.83333334" 03:17:55 DEBUG }, 03:17:55 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:17:55 DEBUG ] 03:17:55 DEBUG "3343.6" 03:17:55 DEBUG "0" 03:17:55 DEBUG "3328.7999999999997" 03:17:55 DEBUG ] 03:17:55 DEBUG 1713665875, 03:17:55 DEBUG "0" 03:17:55 DEBUG "11.72906705178104" 03:17:55 DEBUG ] 03:17:55 DEBUG 1713665875, 03:17:55 DEBUG 1713665875, 03:17:55 DEBUG ] 03:17:55 DEBUG }, 03:17:55 DEBUG ] 03:17:55 DEBUG ] 03:17:55 DEBUG }, 03:17:55 DEBUG ] 03:17:55 DEBUG 1713665875, 03:17:55 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:17:55 DEBUG ] 03:17:55 DEBUG "value": [ 03:17:55 DEBUG }, 03:17:55 DEBUG } 03:17:55 DEBUG ] 03:17:55 DEBUG ] 03:17:55 DEBUG } 03:17:55 DEBUG "value": [ 03:17:55 DEBUG "0.07133333333334425" 03:17:55 DEBUG ] 03:17:55 DEBUG ] 03:17:55 DEBUG } 03:17:55 DEBUG "3346.0333333333333" 03:17:55 DEBUG "599998" 03:17:55 DEBUG } 03:17:55 DEBUG "value": [ 03:17:55 DEBUG } 03:17:55 DEBUG } 03:17:55 DEBUG "value": [ 03:17:55 DEBUG } 03:17:55 DEBUG "0" 03:17:55 DEBUG }, 03:17:55 DEBUG } 03:17:55 DEBUG 1713665875, 03:17:55 DEBUG "value": [ 03:17:55 DEBUG } 03:17:55 DEBUG } 03:17:55 DEBUG } 03:17:55 DEBUG ] 03:17:55 DEBUG 1713665875, 03:17:55 DEBUG ] 03:17:55 DEBUG } 03:17:55 DEBUG } 03:17:55 DEBUG ] 03:17:55 DEBUG ] 03:17:55 DEBUG ] 03:17:55 DEBUG ] 03:17:55 DEBUG 1713665875, 03:17:55 DEBUG } 03:17:55 DEBUG ] 03:17:55 DEBUG 1713665875, 03:17:55 DEBUG } 03:17:55 DEBUG ] 03:17:55 DEBUG "value": [ 03:17:55 DEBUG ] 03:17:55 DEBUG "0" 03:17:55 DEBUG 1713665875, 03:17:55 DEBUG ] 03:17:55 DEBUG } 03:17:55 DEBUG ] 03:17:55 DEBUG "0" 03:17:55 DEBUG } 03:17:55 DEBUG ] 03:17:55 DEBUG ] 03:17:55 DEBUG } 03:17:55 DEBUG } 03:17:55 DEBUG } 03:17:55 DEBUG } 03:17:55 DEBUG "4386134860800" 03:17:55 DEBUG } 03:17:55 DEBUG "31.433333333333334" 03:17:55 DEBUG } 03:17:55 DEBUG 1713665875, 03:17:55 DEBUG } 03:17:55 DEBUG ] 03:17:55 DEBUG } 03:17:55 DEBUG } 03:17:55 DEBUG } 03:17:55 DEBUG } 03:17:55 DEBUG ] 03:17:55 DEBUG ] 03:17:55 DEBUG } 03:17:55 DEBUG } 03:17:55 DEBUG } 03:17:55 DEBUG ] 03:17:55 DEBUG ] 03:17:55 DEBUG } 03:17:55 DEBUG ] 03:17:55 DEBUG } 03:17:55 DEBUG ] 03:17:55 DEBUG ] 03:17:55 DEBUG "11.771354030623357" 03:17:55 DEBUG } 03:17:55 DEBUG ] 03:17:55 DEBUG "55482.26187916577" 03:17:55 DEBUG } 03:17:55 DEBUG } 03:17:55 DEBUG } 03:17:55 DEBUG } 03:17:55 DEBUG } 03:17:55 DEBUG } 03:17:55 DEBUG } 03:17:55 DEBUG } 03:17:55 DEBUG } 03:17:55 DEBUG } 03:17:55 DEBUG } 03:17:55 DEBUG ] 03:17:55 DEBUG } 03:17:55 DEBUG } 03:17:55 DEBUG ] 03:17:55 DEBUG } 03:17:55 DEBUG } 03:17:55 DEBUG } 03:17:55 DEBUG ] 03:17:55 DEBUG ] 03:17:55 DEBUG } 03:17:55 DEBUG } 03:17:55 DEBUG } 03:17:55 DEBUG ] 03:17:55 DEBUG ] 03:17:55 DEBUG } 03:17:55 DEBUG } 03:17:55 DEBUG } 03:17:55 DEBUG } 03:17:55 DEBUG } 03:17:55 DEBUG } 03:17:55 DEBUG ] 03:17:55 DEBUG } 03:17:55 DEBUG } 03:17:55 DEBUG } 03:17:55 DEBUG } 03:18:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713665933" 03:18:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665933" 03:18:53 ERROR [http_cmd]: ERROR 03:18:53 INFO [http_cmd]: http status code OK 03:18:53 ERROR --- status code --- 03:18:53 ERROR http status code is 400 (expected [200, 401, 403, 404]) 03:18:53 DEBUG --- status code --- 03:18:53 ERROR --- http response --- 03:18:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:18:53 DEBUG --- http response --- 03:18:53 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 03:18:53 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 03:18:53 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 03:18:54 INFO 03:18:54 INFO [loop_until]: kubectl --namespace=xlou top pods 03:18:54 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:18:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665934" 03:18:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665934" 03:18:54 INFO [loop_until]: OK (rc = 0) 03:18:54 DEBUG --- stdout --- 03:18:54 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 22313m 8793Mi am-767f6b67b5-qcqz2 22052m 9126Mi am-767f6b67b5-vbltz 24000m 8824Mi ds-cts-0 19751m 12458Mi ds-cts-1 19464m 12478Mi ds-cts-2 19957m 12539Mi ds-idrepo-0 9487m 3918Mi ds-idrepo-1 9633m 3946Mi ds-idrepo-2 9705m 3915Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1179Mi idm-b85f9d978-wnhxb 3m 1239Mi lodemon-54c87f8cbb-jzrjs 29m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-68c5d5b884-p97w5 11702m 890Mi 03:18:54 DEBUG --- stderr --- 03:18:54 DEBUG 03:18:54 INFO [http_cmd]: http status code OK 03:18:54 INFO [http_cmd]: http status code OK 03:18:54 DEBUG --- status code --- 03:18:54 DEBUG --- status code --- 03:18:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:18:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:18:54 DEBUG --- http response --- 03:18:54 DEBUG --- http response --- 03:18:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:18:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:18:54 DEBUG ----- output ----- 03:18:54 DEBUG ----- output ----- 03:18:54 DEBUG { 03:18:54 DEBUG { 03:18:54 DEBUG "status": "success", 03:18:54 DEBUG "status": "success", 03:18:54 DEBUG "data": { 03:18:54 DEBUG "data": { 03:18:54 DEBUG "resultType": "vector", 03:18:54 DEBUG "resultType": "vector", 03:18:54 DEBUG "result": [ 03:18:54 DEBUG "result": [ 03:18:54 DEBUG { 03:18:54 DEBUG { 03:18:54 DEBUG "metric": { 03:18:54 DEBUG "metric": { 03:18:54 DEBUG "pod": "ds-cts-0" 03:18:54 DEBUG "instance": "10.142.0.107:9100" 03:18:54 DEBUG }, 03:18:54 DEBUG }, 03:18:54 DEBUG "value": [ 03:18:54 DEBUG "value": [ 03:18:54 DEBUG 03:18:54 DEBUG 03:18:54 DEBUG 03:18:54 DEBUG 03:18:54 DEBUG 03:18:54 DEBUG 03:18:54 DEBUG [...] 03:18:54 DEBUG [...] 03:18:54 DEBUG 03:18:54 DEBUG 03:18:54 DEBUG 03:18:54 DEBUG 03:18:54 DEBUG 03:18:54 DEBUG 03:18:54 DEBUG "pod": "ds-cts-2" 03:18:54 DEBUG "instance": "10.142.0.43:9100" 03:18:54 DEBUG }, 03:18:54 DEBUG }, 03:18:54 DEBUG "value": [ 03:18:54 DEBUG "value": [ 03:18:54 DEBUG 1713665934, 03:18:54 DEBUG 1713665934, 03:18:54 DEBUG "0.5588999999999998" 03:18:54 DEBUG "0.40863333333342666" 03:18:54 DEBUG ] 03:18:54 DEBUG ] 03:18:54 DEBUG } 03:18:54 DEBUG } 03:18:54 DEBUG ] 03:18:54 DEBUG ] 03:18:54 DEBUG } 03:18:54 DEBUG } 03:18:54 DEBUG } 03:18:54 DEBUG } 03:18:55 INFO 03:18:55 INFO [loop_until]: kubectl --namespace=xlou top node 03:18:55 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:18:55 INFO [loop_until]: OK (rc = 0) 03:18:55 DEBUG --- stdout --- 03:18:55 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 62m 0% 6285Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 9952m 33% 7759Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 22031m 73% 11599Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 86m 0% 3350Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 20037m 67% 15781Mi 14% gke-xlou-cdm-default-pool-1b0769c8-wpwl 143m 0% 5280Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 22426m 75% 12071Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 19830m 66% 3181Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 24791m 83% 11084Mi 9% gke-xlou-cdm-ds-0f601751-j488 10453m 35% 7355Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 19623m 65% 17605Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 10176m 34% 8011Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20444m 68% 15259Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 11668m 73% 11237Mi 19% 03:18:55 DEBUG --- stderr --- 03:18:55 DEBUG 03:18:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665935" 03:18:55 INFO [http_cmd]: http status code OK 03:18:55 DEBUG --- status code --- 03:18:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:18:55 DEBUG --- http response --- 03:18:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:18:55 DEBUG ----- output ----- 03:18:55 DEBUG { 03:18:55 DEBUG "status": "success", 03:18:55 DEBUG "data": { 03:18:55 DEBUG "resultType": "vector", 03:18:55 DEBUG "result": [ 03:18:55 DEBUG { 03:18:55 DEBUG "metric": { 03:18:55 DEBUG "pod": "am-767f6b67b5-9zw22" 03:18:55 DEBUG }, 03:18:55 DEBUG "value": [ 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG [...] 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG "pod": "am-767f6b67b5-vbltz" 03:18:55 DEBUG }, 03:18:55 DEBUG "value": [ 03:18:55 DEBUG 1713665935, 03:18:55 DEBUG "3325.1" 03:18:55 DEBUG ] 03:18:55 DEBUG } 03:18:55 DEBUG ] 03:18:55 DEBUG } 03:18:55 DEBUG } 03:18:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665935" 03:18:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713665935" 03:18:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665935" 03:18:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665935" 03:18:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665935" 03:18:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665935" 03:18:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665935" 03:18:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665935" 03:18:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665935" 03:18:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665935" 03:18:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665935" 03:18:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665935" 03:18:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713665935" 03:18:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665935" 03:18:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665935" 03:18:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665935" 03:18:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665935" 03:18:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665935" 03:18:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713665935" 03:18:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713665935" 03:18:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665935" 03:18:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665935" 03:18:55 INFO [http_cmd]: http status code OK 03:18:55 INFO [http_cmd]: http status code OK 03:18:55 INFO [http_cmd]: http status code OK 03:18:55 INFO [http_cmd]: http status code OK 03:18:55 INFO [http_cmd]: http status code OK 03:18:55 INFO [http_cmd]: http status code OK 03:18:55 INFO [http_cmd]: http status code OK 03:18:55 INFO [http_cmd]: http status code OK 03:18:55 INFO [http_cmd]: http status code OK 03:18:55 INFO [http_cmd]: http status code OK 03:18:55 INFO [http_cmd]: http status code OK 03:18:55 INFO [http_cmd]: http status code OK 03:18:55 INFO [http_cmd]: http status code OK 03:18:55 DEBUG --- status code --- 03:18:55 INFO [http_cmd]: http status code OK 03:18:55 DEBUG --- status code --- 03:18:55 DEBUG --- status code --- 03:18:55 DEBUG --- status code --- 03:18:55 INFO [http_cmd]: http status code OK 03:18:55 INFO [http_cmd]: http status code OK 03:18:55 DEBUG --- status code --- 03:18:55 INFO [http_cmd]: http status code OK 03:18:55 INFO [http_cmd]: http status code OK 03:18:55 INFO [http_cmd]: http status code OK 03:18:55 INFO [http_cmd]: http status code OK 03:18:55 DEBUG --- status code --- 03:18:55 INFO [http_cmd]: http status code OK 03:18:55 DEBUG --- status code --- 03:18:55 DEBUG --- status code --- 03:18:55 DEBUG --- status code --- 03:18:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:18:55 DEBUG --- status code --- 03:18:55 DEBUG --- status code --- 03:18:55 DEBUG --- status code --- 03:18:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:18:55 DEBUG --- status code --- 03:18:55 INFO [http_cmd]: http status code OK 03:18:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:18:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:18:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:18:55 DEBUG --- status code --- 03:18:55 DEBUG --- status code --- 03:18:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:18:55 DEBUG --- status code --- 03:18:55 DEBUG --- status code --- 03:18:55 DEBUG --- status code --- 03:18:55 DEBUG --- status code --- 03:18:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:18:55 DEBUG --- status code --- 03:18:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:18:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:18:55 DEBUG --- http response --- 03:18:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:18:55 DEBUG --- status code --- 03:18:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:18:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:18:55 DEBUG --- http response --- 03:18:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:18:55 DEBUG --- status code --- 03:18:55 DEBUG --- http response --- 03:18:55 DEBUG --- http response --- 03:18:55 DEBUG --- http response --- 03:18:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:18:55 DEBUG --- http response --- 03:18:55 DEBUG --- http response --- 03:18:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:18:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:18:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:18:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:18:55 DEBUG --- http response --- 03:18:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:18:55 DEBUG --- http response --- 03:18:55 DEBUG --- http response --- 03:18:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:18:55 DEBUG --- http response --- 03:18:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:18:55 DEBUG --- http response --- 03:18:55 DEBUG --- http response --- 03:18:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 03:18:55 DEBUG --- http response --- 03:18:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:18:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:18:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:18:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:18:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:18:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:18:55 DEBUG --- http response --- 03:18:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:18:55 DEBUG --- http response --- 03:18:55 DEBUG --- http response --- 03:18:55 DEBUG --- http response --- 03:18:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:18:55 DEBUG --- http response --- 03:18:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:18:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:18:55 DEBUG ----- output ----- 03:18:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:18:55 DEBUG --- http response --- 03:18:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:18:55 DEBUG ----- output ----- 03:18:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:18:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:18:55 DEBUG --- http response --- 03:18:55 DEBUG ----- output ----- 03:18:55 DEBUG ----- output ----- 03:18:55 DEBUG ----- output ----- 03:18:55 DEBUG --- http response --- 03:18:55 DEBUG ----- output ----- 03:18:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:18:55 DEBUG ----- output ----- 03:18:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:18:55 DEBUG ----- output ----- 03:18:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:18:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 03:18:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:18:55 DEBUG ----- output ----- 03:18:55 DEBUG ----- output ----- 03:18:55 DEBUG { 03:18:55 DEBUG ----- output ----- 03:18:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 03:18:55 DEBUG ----- output ----- 03:18:55 DEBUG { 03:18:55 DEBUG ----- output ----- 03:18:55 DEBUG ----- output ----- 03:18:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 03:18:55 DEBUG ----- output ----- 03:18:55 DEBUG { 03:18:55 DEBUG { 03:18:55 DEBUG { 03:18:55 DEBUG ----- output ----- 03:18:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:18:55 DEBUG { 03:18:55 DEBUG ----- output ----- 03:18:55 DEBUG { 03:18:55 DEBUG ----- output ----- 03:18:55 DEBUG ----- output ----- 03:18:55 DEBUG ----- output ----- 03:18:55 DEBUG { 03:18:55 DEBUG { 03:18:55 DEBUG "status": "success", 03:18:55 DEBUG "status": "success", 03:18:55 DEBUG ----- output ----- 03:18:55 DEBUG { 03:18:55 DEBUG "status": "success", 03:18:55 DEBUG { 03:18:55 DEBUG { 03:18:55 DEBUG { 03:18:55 DEBUG { 03:18:55 DEBUG "status": "success", 03:18:55 DEBUG "status": "success", 03:18:55 DEBUG "status": "success", 03:18:55 DEBUG { 03:18:55 DEBUG ----- output ----- 03:18:55 DEBUG { 03:18:55 DEBUG { 03:18:55 DEBUG "status": "success", 03:18:55 DEBUG { 03:18:55 DEBUG { 03:18:55 DEBUG { 03:18:55 DEBUG "status": "success", 03:18:55 DEBUG { 03:18:55 DEBUG "data": { 03:18:55 DEBUG "data": { 03:18:55 DEBUG { 03:18:55 DEBUG "status": "success", 03:18:55 DEBUG "data": { 03:18:55 DEBUG "status": "success", 03:18:55 DEBUG "status": "success", 03:18:55 DEBUG "status": "success", 03:18:55 DEBUG "status": "success", 03:18:55 DEBUG "data": { 03:18:55 DEBUG "data": { 03:18:55 DEBUG "data": { 03:18:55 DEBUG "status": "success", 03:18:55 DEBUG "status": "success", 03:18:55 DEBUG "status": "success", 03:18:55 DEBUG "status": "success", 03:18:55 DEBUG "data": { 03:18:55 DEBUG "resultType": "vector", 03:18:55 DEBUG "status": "success", 03:18:55 DEBUG "status": "success", 03:18:55 DEBUG "data": { 03:18:55 DEBUG "status": "success", 03:18:55 DEBUG "resultType": "vector", 03:18:55 DEBUG "resultType": "vector", 03:18:55 DEBUG "status": "success", 03:18:55 DEBUG "data": { 03:18:55 DEBUG "resultType": "vector", 03:18:55 DEBUG "data": { 03:18:55 DEBUG "data": { 03:18:55 DEBUG "data": { 03:18:55 DEBUG "data": { 03:18:55 DEBUG "resultType": "vector", 03:18:55 DEBUG "resultType": "vector", 03:18:55 DEBUG "resultType": "vector", 03:18:55 DEBUG "data": { 03:18:55 DEBUG "data": { 03:18:55 DEBUG "data": { 03:18:55 DEBUG "data": { 03:18:55 DEBUG "result": [ 03:18:55 DEBUG "status": "success", 03:18:55 DEBUG "data": { 03:18:55 DEBUG "data": { 03:18:55 DEBUG "resultType": "vector", 03:18:55 DEBUG "data": { 03:18:55 DEBUG "result": [ 03:18:55 DEBUG "result": [ 03:18:55 DEBUG "data": { 03:18:55 DEBUG "resultType": "vector", 03:18:55 DEBUG "resultType": "vector", 03:18:55 DEBUG "resultType": "vector", 03:18:55 DEBUG "resultType": "vector", 03:18:55 DEBUG "resultType": "vector", 03:18:55 DEBUG "resultType": "vector", 03:18:55 DEBUG "result": [ 03:18:55 DEBUG "result": [ 03:18:55 DEBUG "result": [ 03:18:55 DEBUG "resultType": "vector", 03:18:55 DEBUG "resultType": "vector", 03:18:55 DEBUG "resultType": "vector", 03:18:55 DEBUG "resultType": "vector", 03:18:55 DEBUG { 03:18:55 DEBUG "resultType": "vector", 03:18:55 DEBUG "resultType": "vector", 03:18:55 DEBUG "result": [ 03:18:55 DEBUG "resultType": "vector", 03:18:55 DEBUG "result": [ 03:18:55 DEBUG { 03:18:55 DEBUG { 03:18:55 DEBUG "result": [ 03:18:55 DEBUG "result": [ 03:18:55 DEBUG "result": [ 03:18:55 DEBUG "result": [ 03:18:55 DEBUG "result": [ 03:18:55 DEBUG "result": [ 03:18:55 DEBUG "result": [ 03:18:55 DEBUG { 03:18:55 DEBUG { 03:18:55 DEBUG { 03:18:55 DEBUG "result": [ 03:18:55 DEBUG "result": [ 03:18:55 DEBUG "result": [ 03:18:55 DEBUG "result": [ 03:18:55 DEBUG "metric": { 03:18:55 DEBUG "result": [ 03:18:55 DEBUG "result": [ 03:18:55 DEBUG { 03:18:55 DEBUG "metric": { 03:18:55 DEBUG { 03:18:55 DEBUG "data": { 03:18:55 DEBUG "metric": { 03:18:55 DEBUG { 03:18:55 DEBUG { 03:18:55 DEBUG { 03:18:55 DEBUG { 03:18:55 DEBUG { 03:18:55 DEBUG "metric": { 03:18:55 DEBUG { 03:18:55 DEBUG "metric": { 03:18:55 DEBUG "metric": { 03:18:55 DEBUG "metric": { 03:18:55 DEBUG { 03:18:55 DEBUG { 03:18:55 DEBUG { 03:18:55 DEBUG { 03:18:55 DEBUG "pod": "am-767f6b67b5-9zw22" 03:18:55 DEBUG { 03:18:55 DEBUG { 03:18:55 DEBUG "metric": { 03:18:55 DEBUG "instance": "10.142.0.107:9100" 03:18:55 DEBUG }, 03:18:55 DEBUG "resultType": "vector", 03:18:55 DEBUG "pod": "am-767f6b67b5-9zw22" 03:18:55 DEBUG }, 03:18:55 DEBUG "metric": { 03:18:55 DEBUG "metric": { 03:18:55 DEBUG "metric": { 03:18:55 DEBUG "instance": "10.142.0.107:9100" 03:18:55 DEBUG "metric": { 03:18:55 DEBUG "pod": "ds-idrepo-0" 03:18:55 DEBUG { 03:18:55 DEBUG "instance": "10.142.0.107:9100" 03:18:55 DEBUG "pod": "ds-idrepo-0" 03:18:55 DEBUG "metric": { 03:18:55 DEBUG "metric": { 03:18:55 DEBUG "metric": { 03:18:55 DEBUG "metric": { 03:18:55 DEBUG }, 03:18:55 DEBUG "metric": { 03:18:55 DEBUG "metric": { 03:18:55 DEBUG "metric": { 03:18:55 DEBUG "value": [ 03:18:55 DEBUG "result": [ 03:18:55 DEBUG "pod": "ds-idrepo-0" 03:18:55 DEBUG "value": [ 03:18:55 DEBUG "metric": { 03:18:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:18:55 DEBUG "pod": "am-767f6b67b5-9zw22" 03:18:55 DEBUG "pod": "am-767f6b67b5-9zw22" 03:18:55 DEBUG }, 03:18:55 DEBUG "value": [ 03:18:55 DEBUG }, 03:18:55 DEBUG "metric": { 03:18:55 DEBUG }, 03:18:55 DEBUG }, 03:18:55 DEBUG "pod": "am-767f6b67b5-9zw22" 03:18:55 DEBUG "pod": "ds-cts-0" 03:18:55 DEBUG "pod": "ds-idrepo-0" 03:18:55 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:18:55 DEBUG "value": [ 03:18:55 DEBUG "pod": "ds-cts-0" 03:18:55 DEBUG "pod": "am-767f6b67b5-9zw22" 03:18:55 DEBUG "instance": "10.142.0.107:9100" 03:18:55 DEBUG 03:18:55 DEBUG { 03:18:55 DEBUG "metric": { 03:18:55 DEBUG 03:18:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:18:55 DEBUG }, 03:18:55 DEBUG }, 03:18:55 DEBUG }, 03:18:55 DEBUG 03:18:55 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:18:55 DEBUG "value": [ 03:18:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:18:55 DEBUG "value": [ 03:18:55 DEBUG "value": [ 03:18:55 DEBUG }, 03:18:55 DEBUG }, 03:18:55 DEBUG }, 03:18:55 DEBUG }, 03:18:55 DEBUG 03:18:55 DEBUG }, 03:18:55 DEBUG }, 03:18:55 DEBUG }, 03:18:55 DEBUG 03:18:55 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:18:55 DEBUG 03:18:55 DEBUG }, 03:18:55 DEBUG }, 03:18:55 DEBUG "value": [ 03:18:55 DEBUG "value": [ 03:18:55 DEBUG "value": [ 03:18:55 DEBUG 03:18:55 DEBUG }, 03:18:55 DEBUG 03:18:55 DEBUG }, 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG "value": [ 03:18:55 DEBUG "value": [ 03:18:55 DEBUG "value": [ 03:18:55 DEBUG "value": [ 03:18:55 DEBUG 03:18:55 DEBUG "value": [ 03:18:55 DEBUG "value": [ 03:18:55 DEBUG "value": [ 03:18:55 DEBUG 03:18:55 DEBUG }, 03:18:55 DEBUG 03:18:55 DEBUG "value": [ 03:18:55 DEBUG "value": [ 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG "value": [ 03:18:55 DEBUG 03:18:55 DEBUG "value": [ 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG [...] 03:18:55 DEBUG "value": [ 03:18:55 DEBUG [...] 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG [...] 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG [...] 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG [...] 03:18:55 DEBUG 03:18:55 DEBUG [...] 03:18:55 DEBUG [...] 03:18:55 DEBUG 03:18:55 DEBUG [...] 03:18:55 DEBUG [...] 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG [...] 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG [...] 03:18:55 DEBUG [...] 03:18:55 DEBUG [...] 03:18:55 DEBUG [...] 03:18:55 DEBUG 03:18:55 DEBUG [...] 03:18:55 DEBUG [...] 03:18:55 DEBUG [...] 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG [...] 03:18:55 DEBUG [...] 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG "pod": "am-767f6b67b5-vbltz" 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG "instance": "10.142.0.43:9100" 03:18:55 DEBUG [...] 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG "instance": "10.142.0.43:9100" 03:18:55 DEBUG }, 03:18:55 DEBUG [...] 03:18:55 DEBUG 03:18:55 DEBUG [...] 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG "pod": "am-767f6b67b5-vbltz" 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG }, 03:18:55 DEBUG 03:18:55 DEBUG "pod": "am-767f6b67b5-vbltz" 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG }, 03:18:55 DEBUG 03:18:55 DEBUG "value": [ 03:18:55 DEBUG 03:18:55 DEBUG "pod": "ds-idrepo-2" 03:18:55 DEBUG 03:18:55 DEBUG "instance": "10.142.0.43:9100" 03:18:55 DEBUG "pod": "ds-cts-2" 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG }, 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG "value": [ 03:18:55 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:18:55 DEBUG }, 03:18:55 DEBUG 03:18:55 DEBUG 03:18:55 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:18:55 DEBUG }, 03:18:55 DEBUG 03:18:55 DEBUG "pod": "am-767f6b67b5-vbltz" 03:18:55 DEBUG 03:18:55 DEBUG }, 03:18:55 DEBUG 03:18:55 DEBUG }, 03:18:55 DEBUG }, 03:18:55 DEBUG "pod": "am-767f6b67b5-vbltz" 03:18:55 DEBUG "pod": "ds-cts-2" 03:18:55 DEBUG "pod": "ds-idrepo-2" 03:18:55 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:18:55 DEBUG "value": [ 03:18:55 DEBUG "pod": "ds-cts-2" 03:18:55 DEBUG }, 03:18:55 DEBUG "instance": "10.142.0.43:9100" 03:18:55 DEBUG 1713665935, 03:18:55 DEBUG "4391391714304" 03:18:55 DEBUG "value": [ 03:18:55 DEBUG 03:18:55 DEBUG "pod": "ds-idrepo-2" 03:18:55 DEBUG "value": [ 03:18:55 DEBUG "value": [ 03:18:55 DEBUG 1713665935, 03:18:55 DEBUG "600859" 03:18:55 DEBUG 03:18:55 DEBUG "value": [ 03:18:55 DEBUG 03:18:55 DEBUG "value": [ 03:18:55 DEBUG "value": [ 03:18:55 DEBUG }, 03:18:55 DEBUG }, 03:18:55 DEBUG }, 03:18:55 DEBUG }, 03:18:55 DEBUG 1713665935, 03:18:55 DEBUG "value": [ 03:18:55 DEBUG "pod": "am-767f6b67b5-vbltz" 03:18:55 DEBUG }, 03:18:55 DEBUG }, 03:18:55 DEBUG ] 03:18:55 DEBUG 1713665935, 03:18:55 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:18:55 DEBUG }, 03:18:55 DEBUG 1713665935, 03:18:55 DEBUG 1713665935, 03:18:55 DEBUG "0.07999999999999736" 03:18:55 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:18:55 DEBUG }, 03:18:55 DEBUG "value": [ 03:18:55 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:18:55 DEBUG 1713665935, 03:18:55 DEBUG 1713665935, 03:18:55 DEBUG "value": [ 03:18:55 DEBUG "value": [ 03:18:55 DEBUG "value": [ 03:18:55 DEBUG "value": [ 03:18:55 DEBUG "0" 03:18:55 DEBUG 1713665935, 03:18:55 DEBUG }, 03:18:55 DEBUG "value": [ 03:18:55 DEBUG "value": [ 03:18:55 DEBUG } 03:18:55 DEBUG "139795.22184300338" 03:18:55 DEBUG }, 03:18:55 DEBUG "value": [ 03:18:55 DEBUG "11.775612299520903" 03:18:55 DEBUG ] 03:18:55 DEBUG ] 03:18:55 DEBUG } 03:18:55 DEBUG 1713665935, 03:18:55 DEBUG 1713665935, 03:18:55 DEBUG }, 03:18:55 DEBUG "100146887.16666666" 03:18:55 DEBUG "0" 03:18:55 DEBUG 1713665935, 03:18:55 DEBUG 1713665935, 03:18:55 DEBUG 1713665935, 03:18:55 DEBUG 1713665935, 03:18:55 DEBUG ] 03:18:55 DEBUG "30.03333333333333" 03:18:55 DEBUG "value": [ 03:18:55 DEBUG 1713665935, 03:18:55 DEBUG 1713665935, 03:18:55 DEBUG ] 03:18:55 DEBUG ] 03:18:55 DEBUG "value": [ 03:18:55 DEBUG 1713665935, 03:18:55 DEBUG "0" 03:18:55 DEBUG } 03:18:55 DEBUG ] 03:18:55 DEBUG }, 03:18:55 DEBUG "0" 03:18:55 DEBUG "149.889" 03:18:55 DEBUG "value": [ 03:18:55 DEBUG ] 03:18:55 DEBUG } 03:18:55 DEBUG "3327.233333333333" 03:18:55 DEBUG "0" 03:18:55 DEBUG "0" 03:18:55 DEBUG "11.818757332600299" 03:18:55 DEBUG } 03:18:55 DEBUG ] 03:18:55 DEBUG 1713665935, 03:18:55 DEBUG "100143473.06666666" 03:18:55 DEBUG "3326.233333333333" 03:18:55 DEBUG } 03:18:55 DEBUG } 03:18:55 DEBUG 1713665935, 03:18:55 DEBUG ] 03:18:55 DEBUG ] 03:18:55 DEBUG } 03:18:55 DEBUG } 03:18:55 DEBUG "value": [ 03:18:55 DEBUG ] 03:18:55 DEBUG ] 03:18:55 DEBUG 1713665935, 03:18:55 DEBUG ] 03:18:55 DEBUG ] 03:18:55 DEBUG ] 03:18:55 DEBUG ] 03:18:55 DEBUG ] 03:18:55 DEBUG ] 03:18:55 DEBUG ] 03:18:55 DEBUG } 03:18:55 DEBUG "3340.4" 03:18:55 DEBUG ] 03:18:55 DEBUG ] 03:18:55 DEBUG } 03:18:55 DEBUG ] 03:18:55 DEBUG "0" 03:18:55 DEBUG } 03:18:55 DEBUG ] 03:18:55 DEBUG ] 03:18:55 DEBUG } 03:18:55 DEBUG 1713665935, 03:18:55 DEBUG } 03:18:55 DEBUG ] 03:18:55 DEBUG "1030721536" 03:18:55 DEBUG ] 03:18:55 DEBUG } 03:18:55 DEBUG } 03:18:55 DEBUG } 03:18:55 DEBUG } 03:18:55 DEBUG } 03:18:55 DEBUG } 03:18:55 DEBUG ] 03:18:55 DEBUG ] 03:18:55 DEBUG } 03:18:55 DEBUG } 03:18:55 DEBUG } 03:18:55 DEBUG ] 03:18:55 DEBUG ] 03:18:55 DEBUG } 03:18:55 DEBUG } 03:18:55 DEBUG "0" 03:18:55 DEBUG ] 03:18:55 DEBUG } 03:18:55 DEBUG } 03:18:55 DEBUG } 03:18:55 DEBUG } 03:18:55 DEBUG ] 03:18:55 DEBUG ] 03:18:55 DEBUG ] 03:18:55 DEBUG ] 03:18:55 DEBUG } 03:18:55 DEBUG } 03:18:55 DEBUG } 03:18:55 DEBUG ] 03:18:55 DEBUG ] 03:18:55 DEBUG } 03:18:55 DEBUG } 03:18:55 DEBUG } 03:18:55 DEBUG } 03:18:55 DEBUG } 03:18:55 DEBUG } 03:18:55 DEBUG } 03:18:55 DEBUG } 03:18:55 DEBUG ] 03:18:55 DEBUG ] 03:18:55 DEBUG } 03:18:55 DEBUG } 03:18:55 DEBUG } 03:18:55 DEBUG } 03:18:55 DEBUG } 03:18:55 DEBUG } 03:18:55 DEBUG } 03:18:55 DEBUG ] 03:18:55 DEBUG } 03:18:55 DEBUG ] 03:18:55 DEBUG ] 03:18:55 DEBUG } 03:18:55 DEBUG } 03:18:55 DEBUG } 03:18:55 DEBUG } 03:18:55 DEBUG } 03:18:55 DEBUG } 03:18:55 DEBUG ] 03:18:55 DEBUG } 03:18:55 DEBUG } 03:18:55 DEBUG } 03:18:55 DEBUG } 03:18:55 DEBUG } 03:18:55 DEBUG } 03:18:55 DEBUG } 03:18:55 DEBUG } 03:18:55 DEBUG } 03:18:55 DEBUG } 03:18:55 DEBUG } 03:18:55 DEBUG } 03:19:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665993" 03:19:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713665993" 03:19:53 ERROR [http_cmd]: ERROR 03:19:53 INFO [http_cmd]: http status code OK 03:19:53 ERROR --- status code --- 03:19:53 DEBUG --- status code --- 03:19:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:19:53 DEBUG --- http response --- 03:19:53 ERROR http status code is 400 (expected [200, 401, 403, 404]) 03:19:53 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 03:19:53 ERROR --- http response --- 03:19:53 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 03:19:53 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 03:19:54 INFO 03:19:54 INFO [loop_until]: kubectl --namespace=xlou top pods 03:19:54 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:19:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665994" 03:19:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665994" 03:19:54 INFO [loop_until]: OK (rc = 0) 03:19:54 DEBUG --- stdout --- 03:19:54 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 22219m 8795Mi am-767f6b67b5-qcqz2 21991m 9126Mi am-767f6b67b5-vbltz 23914m 8834Mi ds-cts-0 19332m 12453Mi ds-cts-1 19639m 12478Mi ds-cts-2 20605m 12540Mi ds-idrepo-0 9727m 3918Mi ds-idrepo-1 9567m 3946Mi ds-idrepo-2 9730m 3915Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1179Mi idm-b85f9d978-wnhxb 4m 1239Mi lodemon-54c87f8cbb-jzrjs 17m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-68c5d5b884-p97w5 11614m 893Mi 03:19:54 DEBUG --- stderr --- 03:19:54 DEBUG 03:19:54 INFO [http_cmd]: http status code OK 03:19:54 DEBUG --- status code --- 03:19:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:19:54 DEBUG --- http response --- 03:19:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:19:54 DEBUG ----- output ----- 03:19:54 DEBUG { 03:19:54 DEBUG "status": "success", 03:19:54 DEBUG "data": { 03:19:54 DEBUG "resultType": "vector", 03:19:54 DEBUG "result": [ 03:19:54 DEBUG { 03:19:54 DEBUG "metric": { 03:19:54 DEBUG "instance": "10.142.0.107:9100" 03:19:54 DEBUG }, 03:19:54 DEBUG "value": [ 03:19:54 DEBUG 03:19:54 DEBUG 03:19:54 DEBUG 03:19:54 DEBUG [...] 03:19:54 DEBUG 03:19:54 DEBUG 03:19:54 DEBUG 03:19:54 DEBUG "instance": "10.142.0.43:9100" 03:19:54 DEBUG }, 03:19:54 DEBUG "value": [ 03:19:54 DEBUG 1713665994, 03:19:54 DEBUG "0.38096666666664114" 03:19:54 DEBUG ] 03:19:54 DEBUG } 03:19:54 DEBUG ] 03:19:54 DEBUG } 03:19:54 DEBUG } 03:19:54 INFO [http_cmd]: http status code OK 03:19:54 DEBUG --- status code --- 03:19:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:19:54 DEBUG --- http response --- 03:19:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:19:54 DEBUG ----- output ----- 03:19:54 DEBUG { 03:19:54 DEBUG "status": "success", 03:19:54 DEBUG "data": { 03:19:54 DEBUG "resultType": "vector", 03:19:54 DEBUG "result": [ 03:19:54 DEBUG { 03:19:54 DEBUG "metric": { 03:19:54 DEBUG "pod": "ds-cts-0" 03:19:54 DEBUG }, 03:19:54 DEBUG "value": [ 03:19:54 DEBUG 03:19:54 DEBUG 03:19:54 DEBUG 03:19:54 DEBUG [...] 03:19:54 DEBUG 03:19:54 DEBUG 03:19:54 DEBUG 03:19:54 DEBUG "pod": "ds-cts-2" 03:19:54 DEBUG }, 03:19:54 DEBUG "value": [ 03:19:54 DEBUG 1713665994, 03:19:54 DEBUG "0.4869" 03:19:54 DEBUG ] 03:19:54 DEBUG } 03:19:54 DEBUG ] 03:19:54 DEBUG } 03:19:54 DEBUG } 03:19:55 INFO 03:19:55 INFO [loop_until]: kubectl --namespace=xlou top node 03:19:55 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:19:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665995" 03:19:55 INFO [http_cmd]: http status code OK 03:19:55 DEBUG --- status code --- 03:19:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:19:55 INFO [loop_until]: OK (rc = 0) 03:19:55 DEBUG --- stdout --- 03:19:55 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 66m 0% 6286Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 9926m 33% 7750Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 21964m 73% 11604Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 81m 0% 3341Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 19818m 66% 15774Mi 14% gke-xlou-cdm-default-pool-1b0769c8-wpwl 155m 0% 5263Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 22316m 74% 12062Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 19671m 65% 3177Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 24667m 82% 11080Mi 9% gke-xlou-cdm-ds-0f601751-j488 10367m 34% 7350Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 19027m 63% 17588Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 10170m 34% 8013Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20479m 68% 15258Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 11972m 75% 11227Mi 19% 03:19:55 DEBUG --- stderr --- 03:19:55 DEBUG 03:19:55 DEBUG --- http response --- 03:19:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:19:55 DEBUG ----- output ----- 03:19:55 DEBUG { 03:19:55 DEBUG "status": "success", 03:19:55 DEBUG "data": { 03:19:55 DEBUG "resultType": "vector", 03:19:55 DEBUG "result": [ 03:19:55 DEBUG { 03:19:55 DEBUG "metric": { 03:19:55 DEBUG "pod": "am-767f6b67b5-9zw22" 03:19:55 DEBUG }, 03:19:55 DEBUG "value": [ 03:19:55 DEBUG 03:19:55 DEBUG 03:19:55 DEBUG 03:19:55 DEBUG [...] 03:19:55 DEBUG 03:19:55 DEBUG 03:19:55 DEBUG 03:19:55 DEBUG "pod": "am-767f6b67b5-vbltz" 03:19:55 DEBUG }, 03:19:55 DEBUG "value": [ 03:19:55 DEBUG 1713665995, 03:19:55 DEBUG "3350.733333333333" 03:19:55 DEBUG ] 03:19:55 DEBUG } 03:19:55 DEBUG ] 03:19:55 DEBUG } 03:19:55 DEBUG } 03:19:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665995" 03:19:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665995" 03:19:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665995" 03:19:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665995" 03:19:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665995" 03:19:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665995" 03:19:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665995" 03:19:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665995" 03:19:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665995" 03:19:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665995" 03:19:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665995" 03:19:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665995" 03:19:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665995" 03:19:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713665995" 03:19:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665995" 03:19:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713665995" 03:19:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713665995" 03:19:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665995" 03:19:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665995" 03:19:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713665995" 03:19:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713665995" 03:19:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713665995" 03:19:55 INFO [http_cmd]: http status code OK 03:19:55 INFO [http_cmd]: http status code OK 03:19:55 INFO [http_cmd]: http status code OK 03:19:55 INFO [http_cmd]: http status code OK 03:19:55 INFO [http_cmd]: http status code OK 03:19:55 INFO [http_cmd]: http status code OK 03:19:55 INFO [http_cmd]: http status code OK 03:19:56 INFO [http_cmd]: http status code OK 03:19:56 INFO [http_cmd]: http status code OK 03:19:56 INFO [http_cmd]: http status code OK 03:19:56 INFO [http_cmd]: http status code OK 03:19:56 INFO [http_cmd]: http status code OK 03:19:56 INFO [http_cmd]: http status code OK 03:19:56 INFO [http_cmd]: http status code OK 03:19:56 INFO [http_cmd]: http status code OK 03:19:56 DEBUG --- status code --- 03:19:56 INFO [http_cmd]: http status code OK 03:19:56 INFO [http_cmd]: http status code OK 03:19:56 DEBUG --- status code --- 03:19:56 DEBUG --- status code --- 03:19:56 INFO [http_cmd]: http status code OK 03:19:56 INFO [http_cmd]: http status code OK 03:19:56 INFO [http_cmd]: http status code OK 03:19:56 DEBUG --- status code --- 03:19:56 DEBUG --- status code --- 03:19:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:19:56 DEBUG --- status code --- 03:19:56 DEBUG --- status code --- 03:19:56 DEBUG --- status code --- 03:19:56 DEBUG --- status code --- 03:19:56 DEBUG --- status code --- 03:19:56 INFO [http_cmd]: http status code OK 03:19:56 INFO [http_cmd]: http status code OK 03:19:56 DEBUG --- status code --- 03:19:56 DEBUG --- status code --- 03:19:56 DEBUG --- status code --- 03:19:56 DEBUG --- status code --- 03:19:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:19:56 DEBUG --- status code --- 03:19:56 DEBUG --- status code --- 03:19:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:19:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:19:56 DEBUG --- http response --- 03:19:56 DEBUG --- status code --- 03:19:56 DEBUG --- status code --- 03:19:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:19:56 DEBUG --- http response --- 03:19:56 DEBUG --- status code --- 03:19:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:19:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:19:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:19:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:19:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:19:56 DEBUG --- status code --- 03:19:56 DEBUG --- status code --- 03:19:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:19:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:19:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:19:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:19:56 DEBUG --- http response --- 03:19:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:19:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:19:56 DEBUG --- http response --- 03:19:56 DEBUG --- status code --- 03:19:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:19:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:19:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:19:56 DEBUG --- http response --- 03:19:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 03:19:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:19:56 DEBUG --- http response --- 03:19:56 DEBUG --- http response --- 03:19:56 DEBUG --- http response --- 03:19:56 DEBUG --- http response --- 03:19:56 DEBUG --- http response --- 03:19:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:19:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:19:56 DEBUG --- http response --- 03:19:56 DEBUG --- http response --- 03:19:56 DEBUG --- http response --- 03:19:56 DEBUG --- http response --- 03:19:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:19:56 DEBUG --- http response --- 03:19:56 DEBUG --- http response --- 03:19:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:19:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:19:56 DEBUG ----- output ----- 03:19:56 DEBUG --- http response --- 03:19:56 DEBUG --- http response --- 03:19:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:19:56 DEBUG ----- output ----- 03:19:56 DEBUG --- http response --- 03:19:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:19:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 03:19:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:19:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:19:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:19:56 DEBUG --- http response --- 03:19:56 DEBUG --- http response --- 03:19:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:19:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:19:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:19:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:19:56 DEBUG ----- output ----- 03:19:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:19:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:19:56 DEBUG --- http response --- 03:19:56 DEBUG ----- output ----- 03:19:56 DEBUG { 03:19:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:19:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:19:56 DEBUG ----- output ----- 03:19:56 DEBUG { 03:19:56 DEBUG ----- output ----- 03:19:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:19:56 DEBUG ----- output ----- 03:19:56 DEBUG ----- output ----- 03:19:56 DEBUG ----- output ----- 03:19:56 DEBUG ----- output ----- 03:19:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:19:56 DEBUG ----- output ----- 03:19:56 DEBUG ----- output ----- 03:19:56 DEBUG { 03:19:56 DEBUG ----- output ----- 03:19:56 DEBUG ----- output ----- 03:19:56 DEBUG { 03:19:56 DEBUG ----- output ----- 03:19:56 DEBUG ----- output ----- 03:19:56 DEBUG { 03:19:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:19:56 DEBUG "status": "success", 03:19:56 DEBUG ----- output ----- 03:19:56 DEBUG ----- output ----- 03:19:56 DEBUG { 03:19:56 DEBUG "status": "success", 03:19:56 DEBUG { 03:19:56 DEBUG ----- output ----- 03:19:56 DEBUG { 03:19:56 DEBUG { 03:19:56 DEBUG { 03:19:56 DEBUG { 03:19:56 DEBUG { 03:19:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:19:56 DEBUG ----- output ----- 03:19:56 DEBUG "status": "success", 03:19:56 DEBUG { 03:19:56 DEBUG { 03:19:56 DEBUG "status": "success", 03:19:56 DEBUG { 03:19:56 DEBUG { 03:19:56 DEBUG "status": "success", 03:19:56 DEBUG ----- output ----- 03:19:56 DEBUG "data": { 03:19:56 DEBUG { 03:19:56 DEBUG { 03:19:56 DEBUG "status": "success", 03:19:56 DEBUG "data": { 03:19:56 DEBUG "status": "success", 03:19:56 DEBUG { 03:19:56 DEBUG "status": "success", 03:19:56 DEBUG "status": "success", 03:19:56 DEBUG "status": "success", 03:19:56 DEBUG "status": "success", 03:19:56 DEBUG "data": { 03:19:56 DEBUG ----- output ----- 03:19:56 DEBUG { 03:19:56 DEBUG "data": { 03:19:56 DEBUG "status": "success", 03:19:56 DEBUG "status": "success", 03:19:56 DEBUG "data": { 03:19:56 DEBUG "status": "success", 03:19:56 DEBUG "status": "success", 03:19:56 DEBUG "data": { 03:19:56 DEBUG "resultType": "vector", 03:19:56 DEBUG "result": [ 03:19:56 DEBUG { 03:19:56 DEBUG "resultType": "vector", 03:19:56 DEBUG "data": { 03:19:56 DEBUG "resultType": "vector", 03:19:56 DEBUG "data": { 03:19:56 DEBUG "data": { 03:19:56 DEBUG "status": "success", 03:19:56 DEBUG "data": { 03:19:56 DEBUG "data": { 03:19:56 DEBUG "status": "success", 03:19:56 DEBUG "resultType": "vector", 03:19:56 DEBUG { 03:19:56 DEBUG "status": "success", 03:19:56 DEBUG "resultType": "vector", 03:19:56 DEBUG "data": { 03:19:56 DEBUG "data": { 03:19:56 DEBUG "resultType": "vector", 03:19:56 DEBUG "data": { 03:19:56 DEBUG "data": { 03:19:56 DEBUG "metric": { 03:19:56 DEBUG { 03:19:56 DEBUG "result": [ 03:19:56 DEBUG "status": "success", 03:19:56 DEBUG "status": "success", 03:19:56 DEBUG "data": { 03:19:56 DEBUG "resultType": "vector", 03:19:56 DEBUG "resultType": "vector", 03:19:56 DEBUG "resultType": "vector", 03:19:56 DEBUG "data": { 03:19:56 DEBUG "resultType": "vector", 03:19:56 DEBUG "resultType": "vector", 03:19:56 DEBUG "data": { 03:19:56 DEBUG "result": [ 03:19:56 DEBUG "status": "success", 03:19:56 DEBUG "data": { 03:19:56 DEBUG "result": [ 03:19:56 DEBUG "resultType": "vector", 03:19:56 DEBUG "resultType": "vector", 03:19:56 DEBUG "result": [ 03:19:56 DEBUG "resultType": "vector", 03:19:56 DEBUG "resultType": "vector", 03:19:56 DEBUG "pod": "am-767f6b67b5-9zw22" 03:19:56 DEBUG "status": "success", 03:19:56 DEBUG { 03:19:56 DEBUG "data": { 03:19:56 DEBUG "result": [ 03:19:56 DEBUG "resultType": "vector", 03:19:56 DEBUG "result": [ 03:19:56 DEBUG "result": [ 03:19:56 DEBUG "result": [ 03:19:56 DEBUG "resultType": "vector", 03:19:56 DEBUG "result": [ 03:19:56 DEBUG "result": [ 03:19:56 DEBUG "resultType": "vector", 03:19:56 DEBUG { 03:19:56 DEBUG "data": { 03:19:56 DEBUG "resultType": "vector", 03:19:56 DEBUG { 03:19:56 DEBUG "result": [ 03:19:56 DEBUG "result": [ 03:19:56 DEBUG { 03:19:56 DEBUG "result": [ 03:19:56 DEBUG { 03:19:56 DEBUG }, 03:19:56 DEBUG "data": { 03:19:56 DEBUG "metric": { 03:19:56 DEBUG "resultType": "vector", 03:19:56 DEBUG { 03:19:56 DEBUG "result": [ 03:19:56 DEBUG { 03:19:56 DEBUG { 03:19:56 DEBUG { 03:19:56 DEBUG "result": [ 03:19:56 DEBUG { 03:19:56 DEBUG { 03:19:56 DEBUG "result": [ 03:19:56 DEBUG "metric": { 03:19:56 DEBUG "resultType": "vector", 03:19:56 DEBUG "result": [ 03:19:56 DEBUG "metric": { 03:19:56 DEBUG { 03:19:56 DEBUG { 03:19:56 DEBUG "metric": { 03:19:56 DEBUG "metric": { 03:19:56 DEBUG "result": [ 03:19:56 DEBUG "value": [ 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG "result": [ 03:19:56 DEBUG "metric": { 03:19:56 DEBUG { 03:19:56 DEBUG "metric": { 03:19:56 DEBUG "metric": { 03:19:56 DEBUG "metric": { 03:19:56 DEBUG { 03:19:56 DEBUG "metric": { 03:19:56 DEBUG "metric": { 03:19:56 DEBUG { 03:19:56 DEBUG "pod": "ds-cts-0" 03:19:56 DEBUG "result": [ 03:19:56 DEBUG { 03:19:56 DEBUG "pod": "am-767f6b67b5-9zw22" 03:19:56 DEBUG "metric": { 03:19:56 DEBUG "metric": { 03:19:56 DEBUG "pod": "ds-idrepo-0" 03:19:56 DEBUG "instance": "10.142.0.107:9100" 03:19:56 DEBUG { 03:19:56 DEBUG 03:19:56 DEBUG "resultType": "vector", 03:19:56 DEBUG "pod": "am-767f6b67b5-9zw22" 03:19:56 DEBUG { 03:19:56 DEBUG "pod": "ds-idrepo-1" 03:19:56 DEBUG "metric": { 03:19:56 DEBUG "instance": "10.142.0.107:9100" 03:19:56 DEBUG "pod": "ds-cts-0" 03:19:56 DEBUG "instance": "10.142.0.107:9100" 03:19:56 DEBUG "metric": { 03:19:56 DEBUG "instance": "10.142.0.107:9100" 03:19:56 DEBUG "pod": "ds-idrepo-0" 03:19:56 DEBUG "metric": { 03:19:56 DEBUG }, 03:19:56 DEBUG { 03:19:56 DEBUG "metric": { 03:19:56 DEBUG }, 03:19:56 DEBUG "pod": "ds-idrepo-0" 03:19:56 DEBUG "pod": "ds-idrepo-0" 03:19:56 DEBUG }, 03:19:56 DEBUG }, 03:19:56 DEBUG "metric": { 03:19:56 DEBUG [...] 03:19:56 DEBUG "result": [ 03:19:56 DEBUG }, 03:19:56 DEBUG "metric": { 03:19:56 DEBUG }, 03:19:56 DEBUG "pod": "am-767f6b67b5-9zw22" 03:19:56 DEBUG }, 03:19:56 DEBUG }, 03:19:56 DEBUG }, 03:19:56 DEBUG "pod": "ds-idrepo-1" 03:19:56 DEBUG }, 03:19:56 DEBUG }, 03:19:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:19:56 DEBUG "value": [ 03:19:56 DEBUG "metric": { 03:19:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:19:56 DEBUG "value": [ 03:19:56 DEBUG }, 03:19:56 DEBUG }, 03:19:56 DEBUG "value": [ 03:19:56 DEBUG "value": [ 03:19:56 DEBUG "pod": "am-767f6b67b5-9zw22" 03:19:56 DEBUG 03:19:56 DEBUG { 03:19:56 DEBUG "value": [ 03:19:56 DEBUG "pod": "ds-idrepo-1" 03:19:56 DEBUG "value": [ 03:19:56 DEBUG }, 03:19:56 DEBUG "value": [ 03:19:56 DEBUG "value": [ 03:19:56 DEBUG "value": [ 03:19:56 DEBUG }, 03:19:56 DEBUG "value": [ 03:19:56 DEBUG "value": [ 03:19:56 DEBUG }, 03:19:56 DEBUG 03:19:56 DEBUG "pod": "am-767f6b67b5-9zw22" 03:19:56 DEBUG }, 03:19:56 DEBUG 03:19:56 DEBUG "value": [ 03:19:56 DEBUG "value": [ 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG }, 03:19:56 DEBUG 03:19:56 DEBUG "metric": { 03:19:56 DEBUG 03:19:56 DEBUG }, 03:19:56 DEBUG 03:19:56 DEBUG "value": [ 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG "value": [ 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG "value": [ 03:19:56 DEBUG 03:19:56 DEBUG }, 03:19:56 DEBUG "value": [ 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG "value": [ 03:19:56 DEBUG 03:19:56 DEBUG "pod": "ds-idrepo-1" 03:19:56 DEBUG 03:19:56 DEBUG "value": [ 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG "value": [ 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG "pod": "am-767f6b67b5-vbltz" 03:19:56 DEBUG }, 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG [...] 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG [...] 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG [...] 03:19:56 DEBUG [...] 03:19:56 DEBUG 03:19:56 DEBUG }, 03:19:56 DEBUG "value": [ 03:19:56 DEBUG [...] 03:19:56 DEBUG 03:19:56 DEBUG [...] 03:19:56 DEBUG 03:19:56 DEBUG [...] 03:19:56 DEBUG [...] 03:19:56 DEBUG [...] 03:19:56 DEBUG 03:19:56 DEBUG [...] 03:19:56 DEBUG [...] 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG [...] 03:19:56 DEBUG [...] 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG "value": [ 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG [...] 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG [...] 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG [...] 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG [...] 03:19:56 DEBUG 1713665995, 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG [...] 03:19:56 DEBUG [...] 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG "pod": "ds-cts-2" 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG "600803" 03:19:56 DEBUG ] 03:19:56 DEBUG 03:19:56 DEBUG "pod": "am-767f6b67b5-vbltz" 03:19:56 DEBUG 03:19:56 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:19:56 DEBUG 03:19:56 DEBUG "instance": "10.142.0.43:9100" 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG [...] 03:19:56 DEBUG }, 03:19:56 DEBUG "value": [ 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG "pod": "ds-idrepo-2" 03:19:56 DEBUG "instance": "10.142.0.43:9100" 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG } 03:19:56 DEBUG 03:19:56 DEBUG }, 03:19:56 DEBUG 03:19:56 DEBUG }, 03:19:56 DEBUG }, 03:19:56 DEBUG 03:19:56 DEBUG "instance": "10.142.0.43:9100" 03:19:56 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:19:56 DEBUG "pod": "ds-idrepo-2" 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG 03:19:56 DEBUG "pod": "am-767f6b67b5-vbltz" 03:19:56 DEBUG 1713665995, 03:19:56 DEBUG 03:19:56 DEBUG "pod": "ds-cts-2" 03:19:56 DEBUG }, 03:19:56 DEBUG "pod": "ds-idrepo-2" 03:19:56 DEBUG }, 03:19:56 DEBUG 03:19:56 DEBUG [...] 03:19:56 DEBUG ] 03:19:56 DEBUG 03:19:56 DEBUG "value": [ 03:19:56 DEBUG 03:19:56 DEBUG "value": [ 03:19:56 DEBUG "value": [ 03:19:56 DEBUG "pod": "ds-cts-2" 03:19:56 DEBUG }, 03:19:56 DEBUG }, 03:19:56 DEBUG }, 03:19:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:19:56 DEBUG 03:19:56 DEBUG "instance": "10.142.0.43:9100" 03:19:56 DEBUG }, 03:19:56 DEBUG "0" 03:19:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:19:56 DEBUG }, 03:19:56 DEBUG "value": [ 03:19:56 DEBUG }, 03:19:56 DEBUG "value": [ 03:19:56 DEBUG "pod": "am-767f6b67b5-vbltz" 03:19:56 DEBUG 03:19:56 DEBUG } 03:19:56 DEBUG 03:19:56 DEBUG 1713665995, 03:19:56 DEBUG "pod": "am-767f6b67b5-vbltz" 03:19:56 DEBUG 1713665995, 03:19:56 DEBUG 1713665995, 03:19:56 DEBUG }, 03:19:56 DEBUG "value": [ 03:19:56 DEBUG "value": [ 03:19:56 DEBUG "value": [ 03:19:56 DEBUG }, 03:19:56 DEBUG "value": [ 03:19:56 DEBUG }, 03:19:56 DEBUG "value": [ 03:19:56 DEBUG ] 03:19:56 DEBUG }, 03:19:56 DEBUG "value": [ 03:19:56 DEBUG 1713665995, 03:19:56 DEBUG "value": [ 03:19:56 DEBUG 1713665995, 03:19:56 DEBUG }, 03:19:56 DEBUG 03:19:56 DEBUG } 03:19:56 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:19:56 DEBUG "0" 03:19:56 DEBUG }, 03:19:56 DEBUG "66382.59684019683" 03:19:56 DEBUG "100687718.93333334" 03:19:56 DEBUG ] 03:19:56 DEBUG 1713665995, 03:19:56 DEBUG 1713665995, 03:19:56 DEBUG 1713665995, 03:19:56 DEBUG 1713665995, 03:19:56 DEBUG "value": [ 03:19:56 DEBUG 1713665995, 03:19:56 DEBUG "3347.766666666667" 03:19:56 DEBUG 03:19:56 DEBUG "value": [ 03:19:56 DEBUG 1713665995, 03:19:56 DEBUG "0" 03:19:56 DEBUG 1713665995, 03:19:56 DEBUG "100684331.53333333" 03:19:56 DEBUG "value": [ 03:19:56 DEBUG 03:19:56 DEBUG }, 03:19:56 DEBUG ] 03:19:56 DEBUG "value": [ 03:19:56 DEBUG ] 03:19:56 DEBUG "value": [ 03:19:56 DEBUG } 03:19:56 DEBUG "4396496655360" 03:19:56 DEBUG "5.45702415492883" 03:19:56 DEBUG "0" 03:19:56 DEBUG "1034174464" 03:19:56 DEBUG 1713665995, 03:19:56 DEBUG ] 03:19:56 DEBUG } 03:19:56 DEBUG "pod": "am-767f6b67b5-vbltz" 03:19:56 DEBUG 1713665995, 03:19:56 DEBUG "11.735817749000256" 03:19:56 DEBUG ] 03:19:56 DEBUG "0" 03:19:56 DEBUG ] 03:19:56 DEBUG 1713665995, 03:19:56 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:19:56 DEBUG "value": [ 03:19:56 DEBUG } 03:19:56 DEBUG 1713665995, 03:19:56 DEBUG } 03:19:56 DEBUG 1713665995, 03:19:56 DEBUG "33.96666666666667" 03:19:56 DEBUG ] 03:19:56 DEBUG ] 03:19:56 DEBUG } 03:19:56 DEBUG ] 03:19:56 DEBUG "0.07033333333332244" 03:19:56 DEBUG } 03:19:56 DEBUG ] 03:19:56 DEBUG }, 03:19:56 DEBUG "0" 03:19:56 DEBUG ] 03:19:56 DEBUG } 03:19:56 DEBUG ] 03:19:56 DEBUG } 03:19:56 DEBUG "3352.0333333333333" 03:19:56 DEBUG }, 03:19:56 DEBUG 1713665995, 03:19:56 DEBUG ] 03:19:56 DEBUG "3326.5" 03:19:56 DEBUG ] 03:19:56 DEBUG ] 03:19:56 DEBUG ] 03:19:56 DEBUG } 03:19:56 DEBUG ] 03:19:56 DEBUG ] 03:19:56 DEBUG } 03:19:56 DEBUG ] 03:19:56 DEBUG ] 03:19:56 DEBUG } 03:19:56 DEBUG "value": [ 03:19:56 DEBUG 1713665995, 03:19:56 DEBUG } 03:19:56 DEBUG ] 03:19:56 DEBUG } 03:19:56 DEBUG ] 03:19:56 DEBUG ] 03:19:56 DEBUG "value": [ 03:19:56 DEBUG "0" 03:19:56 DEBUG } 03:19:56 DEBUG ] 03:19:56 DEBUG } 03:19:56 DEBUG } 03:19:56 DEBUG } 03:19:56 DEBUG } 03:19:56 DEBUG } 03:19:56 DEBUG } 03:19:56 DEBUG ] 03:19:56 DEBUG } 03:19:56 DEBUG } 03:19:56 DEBUG } 03:19:56 DEBUG "149.889" 03:19:56 DEBUG ] 03:19:56 DEBUG ] 03:19:56 DEBUG } 03:19:56 DEBUG ] 03:19:56 DEBUG } 03:19:56 DEBUG } 03:19:56 DEBUG 1713665995, 03:19:56 DEBUG ] 03:19:56 DEBUG } 03:19:56 DEBUG } 03:19:56 DEBUG } 03:19:56 DEBUG ] 03:19:56 DEBUG ] 03:19:56 DEBUG ] 03:19:56 DEBUG } 03:19:56 DEBUG } 03:19:56 DEBUG ] 03:19:56 DEBUG } 03:19:56 DEBUG ] 03:19:56 DEBUG } 03:19:56 DEBUG } 03:19:56 DEBUG } 03:19:56 DEBUG } 03:19:56 DEBUG } 03:19:56 DEBUG ] 03:19:56 DEBUG "0" 03:19:56 DEBUG } 03:19:56 DEBUG ] 03:19:56 DEBUG } 03:19:56 DEBUG } 03:19:56 DEBUG } 03:19:56 DEBUG } 03:19:56 DEBUG } 03:19:56 DEBUG } 03:19:56 DEBUG ] 03:19:56 DEBUG } 03:19:56 DEBUG } 03:19:56 DEBUG } 03:19:56 DEBUG ] 03:19:56 DEBUG ] 03:19:56 DEBUG } 03:19:56 DEBUG } 03:19:56 DEBUG } 03:19:56 DEBUG } 03:19:56 DEBUG } 03:19:56 DEBUG ] 03:19:56 DEBUG } 03:19:56 DEBUG } 03:19:56 DEBUG } 03:19:56 DEBUG } 03:19:56 DEBUG } 03:19:56 DEBUG } 03:19:56 DEBUG } 03:19:56 DEBUG ] 03:19:56 DEBUG } 03:19:56 DEBUG } 03:19:56 DEBUG } 03:19:56 DEBUG } 03:20:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713666053" 03:20:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666053" 03:20:53 INFO [http_cmd]: http status code OK 03:20:53 ERROR [http_cmd]: ERROR 03:20:53 DEBUG --- status code --- 03:20:53 ERROR --- status code --- 03:20:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:20:53 ERROR http status code is 400 (expected [200, 401, 403, 404]) 03:20:53 DEBUG --- http response --- 03:20:53 ERROR --- http response --- 03:20:53 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 03:20:53 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 03:20:53 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 03:20:54 INFO 03:20:54 INFO [loop_until]: kubectl --namespace=xlou top pods 03:20:54 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:20:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666054" 03:20:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666054" 03:20:54 INFO [http_cmd]: http status code OK 03:20:54 DEBUG --- status code --- 03:20:54 INFO [http_cmd]: http status code OK 03:20:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:20:54 DEBUG --- status code --- 03:20:54 DEBUG --- http response --- 03:20:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:20:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:20:54 DEBUG ----- output ----- 03:20:54 DEBUG --- http response --- 03:20:54 DEBUG { 03:20:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:20:54 DEBUG "status": "success", 03:20:54 DEBUG ----- output ----- 03:20:54 DEBUG "data": { 03:20:54 DEBUG { 03:20:54 DEBUG "resultType": "vector", 03:20:54 DEBUG "status": "success", 03:20:54 DEBUG "result": [ 03:20:54 DEBUG "data": { 03:20:54 DEBUG { 03:20:54 DEBUG "resultType": "vector", 03:20:54 DEBUG "metric": { 03:20:54 DEBUG "result": [ 03:20:54 DEBUG "pod": "ds-cts-0" 03:20:54 DEBUG }, 03:20:54 DEBUG "value": [ 03:20:54 DEBUG 03:20:54 DEBUG 03:20:54 DEBUG 03:20:54 DEBUG [...] 03:20:54 DEBUG 03:20:54 DEBUG 03:20:54 DEBUG 03:20:54 DEBUG "pod": "ds-cts-2" 03:20:54 DEBUG }, 03:20:54 DEBUG "value": [ 03:20:54 DEBUG 1713666054, 03:20:54 DEBUG "0.5348333333333338" 03:20:54 DEBUG ] 03:20:54 DEBUG } 03:20:54 DEBUG ] 03:20:54 DEBUG } 03:20:54 DEBUG } 03:20:54 DEBUG { 03:20:54 DEBUG "metric": { 03:20:54 DEBUG "instance": "10.142.0.107:9100" 03:20:54 DEBUG }, 03:20:54 DEBUG "value": [ 03:20:54 DEBUG 03:20:54 DEBUG 03:20:54 DEBUG 03:20:54 DEBUG [...] 03:20:54 DEBUG 03:20:54 DEBUG 03:20:54 DEBUG 03:20:54 DEBUG "instance": "10.142.0.43:9100" 03:20:54 DEBUG }, 03:20:54 DEBUG "value": [ 03:20:54 DEBUG 1713666054, 03:20:54 DEBUG "0.39953333333332924" 03:20:54 DEBUG ] 03:20:54 DEBUG } 03:20:54 DEBUG ] 03:20:54 DEBUG } 03:20:54 DEBUG } 03:20:54 INFO [loop_until]: OK (rc = 0) 03:20:54 DEBUG --- stdout --- 03:20:54 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 22102m 8796Mi am-767f6b67b5-qcqz2 22412m 9129Mi am-767f6b67b5-vbltz 24115m 8823Mi ds-cts-0 19874m 12490Mi ds-cts-1 19472m 12502Mi ds-cts-2 20596m 12539Mi ds-idrepo-0 9425m 3918Mi ds-idrepo-1 9568m 3946Mi ds-idrepo-2 9580m 3915Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1179Mi idm-b85f9d978-wnhxb 3m 1239Mi lodemon-54c87f8cbb-jzrjs 16m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-68c5d5b884-p97w5 11695m 895Mi 03:20:54 DEBUG --- stderr --- 03:20:54 DEBUG 03:20:55 INFO 03:20:55 INFO [loop_until]: kubectl --namespace=xlou top node 03:20:55 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:20:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666055" 03:20:55 INFO [http_cmd]: http status code OK 03:20:55 DEBUG --- status code --- 03:20:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:20:55 DEBUG --- http response --- 03:20:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:20:55 DEBUG ----- output ----- 03:20:55 DEBUG { 03:20:55 DEBUG "status": "success", 03:20:55 DEBUG "data": { 03:20:55 DEBUG "resultType": "vector", 03:20:55 DEBUG "result": [ 03:20:55 DEBUG { 03:20:55 DEBUG "metric": { 03:20:55 DEBUG "pod": "am-767f6b67b5-9zw22" 03:20:55 DEBUG }, 03:20:55 DEBUG "value": [ 03:20:55 DEBUG 03:20:55 DEBUG 03:20:55 DEBUG 03:20:55 DEBUG [...] 03:20:55 DEBUG 03:20:55 DEBUG 03:20:55 DEBUG 03:20:55 DEBUG "pod": "am-767f6b67b5-vbltz" 03:20:55 DEBUG }, 03:20:55 DEBUG "value": [ 03:20:55 DEBUG 1713666055, 03:20:55 DEBUG "3333.1" 03:20:55 DEBUG ] 03:20:55 DEBUG } 03:20:55 DEBUG ] 03:20:55 DEBUG } 03:20:55 DEBUG } 03:20:55 INFO [loop_until]: OK (rc = 0) 03:20:55 DEBUG --- stdout --- 03:20:55 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 62m 0% 6286Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 9927m 33% 7762Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 21965m 73% 11613Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 85m 0% 3355Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 19989m 66% 15807Mi 14% gke-xlou-cdm-default-pool-1b0769c8-wpwl 155m 0% 5250Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 22421m 75% 12061Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 19753m 66% 3177Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 24762m 82% 11090Mi 9% gke-xlou-cdm-ds-0f601751-j488 10380m 34% 7350Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 19590m 65% 17641Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 10199m 34% 8020Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20522m 68% 15228Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 12000m 75% 11227Mi 19% 03:20:55 DEBUG --- stderr --- 03:20:55 DEBUG 03:20:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666056" 03:20:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666056" 03:20:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666056" 03:20:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666056" 03:20:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713666056" 03:20:56 INFO [http_cmd]: http status code OK 03:20:56 DEBUG --- status code --- 03:20:56 INFO [http_cmd]: http status code OK 03:20:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:20:56 DEBUG --- status code --- 03:20:56 DEBUG --- http response --- 03:20:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:20:56 DEBUG --- http response --- 03:20:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:20:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 03:20:56 DEBUG ----- output ----- 03:20:56 DEBUG ----- output ----- 03:20:56 DEBUG { 03:20:56 DEBUG { 03:20:56 DEBUG "status": "success", 03:20:56 DEBUG "status": "success", 03:20:56 DEBUG "data": { 03:20:56 DEBUG "data": { 03:20:56 DEBUG "resultType": "vector", 03:20:56 DEBUG "resultType": "vector", 03:20:56 DEBUG "result": [ 03:20:56 INFO [http_cmd]: http status code OK 03:20:56 DEBUG "result": [ 03:20:56 DEBUG { 03:20:56 DEBUG --- status code --- 03:20:56 DEBUG { 03:20:56 DEBUG "metric": { 03:20:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:20:56 DEBUG "metric": { 03:20:56 DEBUG "pod": "ds-idrepo-0" 03:20:56 DEBUG --- http response --- 03:20:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:20:56 DEBUG }, 03:20:56 DEBUG }, 03:20:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 03:20:56 INFO [http_cmd]: http status code OK 03:20:56 DEBUG "value": [ 03:20:56 DEBUG "value": [ 03:20:56 DEBUG ----- output ----- 03:20:56 DEBUG --- status code --- 03:20:56 DEBUG 03:20:56 INFO [http_cmd]: http status code OK 03:20:56 DEBUG 03:20:56 DEBUG { 03:20:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:20:56 DEBUG 03:20:56 DEBUG --- status code --- 03:20:56 DEBUG 03:20:56 DEBUG "status": "success", 03:20:56 DEBUG --- http response --- 03:20:56 DEBUG 03:20:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:20:56 DEBUG 03:20:56 DEBUG "data": { 03:20:56 DEBUG [...] 03:20:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:20:56 DEBUG --- http response --- 03:20:56 DEBUG [...] 03:20:56 DEBUG "resultType": "vector", 03:20:56 DEBUG 03:20:56 DEBUG ----- output ----- 03:20:56 DEBUG 03:20:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:20:56 DEBUG "result": [ 03:20:56 DEBUG 03:20:56 DEBUG { 03:20:56 DEBUG 03:20:56 DEBUG ----- output ----- 03:20:56 DEBUG { 03:20:56 DEBUG 03:20:56 DEBUG "status": "success", 03:20:56 DEBUG 03:20:56 DEBUG { 03:20:56 DEBUG "metric": { 03:20:56 DEBUG "pod": "ds-idrepo-2" 03:20:56 DEBUG "data": { 03:20:56 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:20:56 DEBUG "status": "success", 03:20:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:20:56 DEBUG }, 03:20:56 DEBUG "resultType": "vector", 03:20:56 DEBUG }, 03:20:56 DEBUG "data": { 03:20:56 DEBUG }, 03:20:56 DEBUG "value": [ 03:20:56 DEBUG "result": [ 03:20:56 DEBUG "value": [ 03:20:56 DEBUG "resultType": "vector", 03:20:56 DEBUG "value": [ 03:20:56 DEBUG 1713666056, 03:20:56 DEBUG { 03:20:56 DEBUG 1713666056, 03:20:56 DEBUG "result": [ 03:20:56 DEBUG 03:20:56 DEBUG "0" 03:20:56 DEBUG "metric": { 03:20:56 DEBUG "28499390.74959187" 03:20:56 DEBUG { 03:20:56 DEBUG 03:20:56 DEBUG ] 03:20:56 DEBUG "pod": "am-767f6b67b5-9zw22" 03:20:56 DEBUG ] 03:20:56 DEBUG "metric": { 03:20:56 DEBUG 03:20:56 DEBUG } 03:20:56 DEBUG }, 03:20:56 DEBUG } 03:20:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:20:56 DEBUG [...] 03:20:56 DEBUG ] 03:20:56 DEBUG "value": [ 03:20:56 DEBUG ] 03:20:56 DEBUG }, 03:20:56 DEBUG 03:20:56 DEBUG } 03:20:56 DEBUG 03:20:56 DEBUG } 03:20:56 DEBUG "value": [ 03:20:56 DEBUG 03:20:56 DEBUG } 03:20:56 DEBUG 03:20:56 DEBUG } 03:20:56 DEBUG 03:20:56 DEBUG 03:20:56 DEBUG 03:20:56 DEBUG 03:20:56 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:20:56 DEBUG [...] 03:20:56 DEBUG 03:20:56 DEBUG }, 03:20:56 DEBUG 03:20:56 DEBUG [...] 03:20:56 DEBUG "value": [ 03:20:56 DEBUG 03:20:56 DEBUG 03:20:56 DEBUG 1713666056, 03:20:56 DEBUG 03:20:56 DEBUG 03:20:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666056" 03:20:56 DEBUG "6.736138020711726" 03:20:56 DEBUG "pod": "am-767f6b67b5-vbltz" 03:20:56 DEBUG 03:20:56 DEBUG ] 03:20:56 DEBUG }, 03:20:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:20:56 DEBUG } 03:20:56 DEBUG "value": [ 03:20:56 DEBUG }, 03:20:56 DEBUG ] 03:20:56 DEBUG 1713666056, 03:20:56 DEBUG "value": [ 03:20:56 DEBUG } 03:20:56 DEBUG "0" 03:20:56 DEBUG ] 03:20:56 DEBUG 1713666056, 03:20:56 DEBUG } 03:20:56 DEBUG } 03:20:56 DEBUG "11.80462506922754" 03:20:56 DEBUG ] 03:20:56 DEBUG ] 03:20:56 DEBUG } 03:20:56 DEBUG } 03:20:56 DEBUG } 03:20:56 DEBUG ] 03:20:56 DEBUG } 03:20:56 DEBUG } 03:20:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666056" 03:20:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713666056" 03:20:56 INFO [http_cmd]: http status code OK 03:20:56 DEBUG --- status code --- 03:20:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:20:56 DEBUG --- http response --- 03:20:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:20:56 DEBUG ----- output ----- 03:20:56 DEBUG { 03:20:56 DEBUG "status": "success", 03:20:56 DEBUG "data": { 03:20:56 DEBUG "resultType": "vector", 03:20:56 DEBUG "result": [ 03:20:56 DEBUG { 03:20:56 DEBUG "metric": { 03:20:56 DEBUG "pod": "am-767f6b67b5-9zw22" 03:20:56 DEBUG }, 03:20:56 DEBUG "value": [ 03:20:56 DEBUG 03:20:56 DEBUG 03:20:56 DEBUG 03:20:56 DEBUG [...] 03:20:56 DEBUG 03:20:56 DEBUG 03:20:56 DEBUG 03:20:56 DEBUG "pod": "am-767f6b67b5-vbltz" 03:20:56 DEBUG }, 03:20:56 DEBUG "value": [ 03:20:56 DEBUG 1713666056, 03:20:56 DEBUG "3334.9666666666667" 03:20:56 DEBUG ] 03:20:56 DEBUG } 03:20:56 DEBUG ] 03:20:56 DEBUG } 03:20:56 DEBUG } 03:20:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666056" 03:20:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666056" 03:20:56 INFO [http_cmd]: http status code OK 03:20:56 INFO [http_cmd]: http status code OK 03:20:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666056" 03:20:56 DEBUG --- status code --- 03:20:56 DEBUG --- status code --- 03:20:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:20:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713666056" 03:20:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:20:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666056" 03:20:56 DEBUG --- http response --- 03:20:56 DEBUG --- http response --- 03:20:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:20:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666056" 03:20:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666056" 03:20:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666056" 03:20:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666056" 03:20:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713666056" 03:20:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666056" 03:20:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666056" 03:20:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:20:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666056" 03:20:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666056" 03:20:56 DEBUG ----- output ----- 03:20:56 INFO [http_cmd]: http status code OK 03:20:56 INFO [http_cmd]: http status code OK 03:20:56 INFO [http_cmd]: http status code OK 03:20:56 INFO [http_cmd]: http status code OK 03:20:56 INFO [http_cmd]: http status code OK 03:20:56 DEBUG ----- output ----- 03:20:56 INFO [http_cmd]: http status code OK 03:20:56 INFO [http_cmd]: http status code OK 03:20:56 INFO [http_cmd]: http status code OK 03:20:56 INFO [http_cmd]: http status code OK 03:20:56 DEBUG { 03:20:56 INFO [http_cmd]: http status code OK 03:20:56 DEBUG --- status code --- 03:20:56 DEBUG --- status code --- 03:20:56 DEBUG --- status code --- 03:20:56 DEBUG --- status code --- 03:20:56 DEBUG --- status code --- 03:20:56 INFO [http_cmd]: http status code OK 03:20:56 INFO [http_cmd]: http status code OK 03:20:56 DEBUG { 03:20:56 DEBUG --- status code --- 03:20:56 DEBUG --- status code --- 03:20:56 DEBUG --- status code --- 03:20:56 DEBUG --- status code --- 03:20:56 DEBUG "status": "success", 03:20:56 DEBUG "data": { 03:20:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:20:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:20:56 DEBUG --- status code --- 03:20:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:20:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:20:56 DEBUG --- status code --- 03:20:56 INFO [http_cmd]: http status code OK 03:20:56 DEBUG --- status code --- 03:20:56 DEBUG "status": "success", 03:20:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:20:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:20:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:20:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:20:56 DEBUG "resultType": "vector", 03:20:56 DEBUG --- http response --- 03:20:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:20:56 DEBUG --- http response --- 03:20:56 DEBUG --- http response --- 03:20:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:20:56 DEBUG --- http response --- 03:20:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:20:56 INFO [http_cmd]: http status code OK 03:20:56 DEBUG --- status code --- 03:20:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:20:56 DEBUG "data": { 03:20:56 DEBUG --- http response --- 03:20:56 DEBUG --- http response --- 03:20:56 DEBUG --- http response --- 03:20:56 DEBUG --- http response --- 03:20:56 DEBUG "result": [ 03:20:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:20:56 DEBUG --- http response --- 03:20:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:20:56 DEBUG --- http response --- 03:20:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:20:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:20:56 DEBUG --- http response --- 03:20:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:20:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:20:56 DEBUG --- http response --- 03:20:56 DEBUG "resultType": "vector", 03:20:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:20:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:20:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:20:56 DEBUG ----- output ----- 03:20:56 DEBUG { 03:20:56 DEBUG "metric": { 03:20:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:20:56 DEBUG ----- output ----- 03:20:56 DEBUG { 03:20:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:20:56 DEBUG ----- output ----- 03:20:56 DEBUG ----- output ----- 03:20:56 DEBUG --- status code --- 03:20:56 DEBUG --- http response --- 03:20:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 03:20:56 DEBUG ----- output ----- 03:20:56 DEBUG ----- output ----- 03:20:56 DEBUG ----- output ----- 03:20:56 DEBUG { 03:20:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:20:56 DEBUG "pod": "am-767f6b67b5-9zw22" 03:20:56 DEBUG ----- output ----- 03:20:56 DEBUG ----- output ----- 03:20:56 DEBUG "status": "success", 03:20:56 DEBUG ----- output ----- 03:20:56 DEBUG ----- output ----- 03:20:56 DEBUG { 03:20:56 DEBUG { 03:20:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:20:56 DEBUG { 03:20:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:20:56 DEBUG "result": [ 03:20:56 DEBUG { 03:20:56 DEBUG { 03:20:56 DEBUG "status": "success", 03:20:56 DEBUG ----- output ----- 03:20:56 DEBUG }, 03:20:56 DEBUG { 03:20:56 DEBUG { 03:20:56 DEBUG "data": { 03:20:56 DEBUG "resultType": "vector", 03:20:56 DEBUG "result": [ 03:20:56 DEBUG "status": "success", 03:20:56 DEBUG "status": "success", 03:20:56 DEBUG "data": { 03:20:56 DEBUG "status": "success", 03:20:56 DEBUG ----- output ----- 03:20:56 DEBUG { 03:20:56 DEBUG "status": "success", 03:20:56 DEBUG "status": "success", 03:20:56 DEBUG "data": { 03:20:56 DEBUG { 03:20:56 DEBUG "value": [ 03:20:56 DEBUG "status": "success", 03:20:56 DEBUG "status": "success", 03:20:56 DEBUG { 03:20:56 DEBUG { 03:20:56 DEBUG { 03:20:56 DEBUG "data": { 03:20:56 DEBUG "resultType": "vector", 03:20:56 DEBUG --- http response --- 03:20:56 DEBUG "data": { 03:20:56 DEBUG { 03:20:56 DEBUG "metric": { 03:20:56 DEBUG "pod": "am-767f6b67b5-9zw22" 03:20:56 DEBUG "data": { 03:20:56 DEBUG "resultType": "vector", 03:20:56 DEBUG "status": "success", 03:20:56 DEBUG 03:20:56 DEBUG "data": { 03:20:56 DEBUG "data": { 03:20:56 DEBUG "status": "success", 03:20:56 DEBUG "status": "success", 03:20:56 DEBUG "metric": { 03:20:56 DEBUG "resultType": "vector", 03:20:56 DEBUG "result": [ 03:20:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 03:20:56 DEBUG "resultType": "vector", 03:20:56 DEBUG "status": "success", 03:20:56 DEBUG }, 03:20:56 DEBUG "data": { 03:20:56 DEBUG "resultType": "vector", 03:20:56 DEBUG "result": [ 03:20:56 DEBUG "data": { 03:20:56 DEBUG 03:20:56 DEBUG "resultType": "vector", 03:20:56 DEBUG "resultType": "vector", 03:20:56 DEBUG "data": { 03:20:56 DEBUG "data": { 03:20:56 DEBUG "instance": "10.142.0.107:9100" 03:20:56 DEBUG "result": [ 03:20:56 DEBUG { 03:20:56 DEBUG ----- output ----- 03:20:56 DEBUG "result": [ 03:20:56 DEBUG "data": { 03:20:56 DEBUG "value": [ 03:20:56 DEBUG "resultType": "vector", 03:20:56 DEBUG "result": [ 03:20:56 DEBUG { 03:20:56 DEBUG "resultType": "vector", 03:20:56 DEBUG 03:20:56 DEBUG "result": [ 03:20:56 DEBUG "result": [ 03:20:56 DEBUG "resultType": "vector", 03:20:56 DEBUG "resultType": "vector", 03:20:56 DEBUG }, 03:20:56 DEBUG { 03:20:56 DEBUG "metric": { 03:20:56 DEBUG { 03:20:56 DEBUG { 03:20:56 DEBUG "resultType": "vector", 03:20:56 DEBUG 03:20:56 DEBUG "result": [ 03:20:56 DEBUG { 03:20:56 DEBUG "metric": { 03:20:56 DEBUG "result": [ 03:20:56 DEBUG [...] 03:20:56 DEBUG { 03:20:56 DEBUG { 03:20:56 DEBUG "result": [ 03:20:56 DEBUG "result": [ 03:20:56 DEBUG "value": [ 03:20:56 DEBUG "metric": { 03:20:56 DEBUG "pod": "am-767f6b67b5-9zw22" 03:20:56 DEBUG "status": "success", 03:20:56 DEBUG "metric": { 03:20:56 DEBUG "result": [ 03:20:56 DEBUG 03:20:56 DEBUG { 03:20:56 DEBUG "metric": { 03:20:56 DEBUG "pod": "ds-idrepo-0" 03:20:56 DEBUG { 03:20:56 DEBUG 03:20:56 DEBUG "metric": { 03:20:56 DEBUG "pod": "ds-cts-0" 03:20:56 DEBUG { 03:20:56 DEBUG { 03:20:56 DEBUG "metric": { 03:20:56 DEBUG "pod": "ds-idrepo-0" 03:20:56 DEBUG }, 03:20:56 DEBUG "data": { 03:20:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:20:56 DEBUG { 03:20:56 DEBUG 03:20:56 DEBUG "metric": { 03:20:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:20:56 DEBUG }, 03:20:56 DEBUG "metric": { 03:20:56 DEBUG 03:20:56 DEBUG }, 03:20:56 DEBUG "metric": { 03:20:56 DEBUG "metric": { 03:20:56 DEBUG "pod": "ds-idrepo-0" 03:20:56 DEBUG 03:20:56 DEBUG }, 03:20:56 DEBUG "value": [ 03:20:56 DEBUG "resultType": "vector", 03:20:56 DEBUG }, 03:20:56 DEBUG "metric": { 03:20:56 DEBUG [...] 03:20:56 DEBUG "pod": "am-767f6b67b5-9zw22" 03:20:56 DEBUG }, 03:20:56 DEBUG "value": [ 03:20:56 DEBUG "instance": "10.142.0.107:9100" 03:20:56 DEBUG 03:20:56 DEBUG "value": [ 03:20:56 DEBUG "instance": "10.142.0.107:9100" 03:20:56 DEBUG "instance": "10.142.0.107:9100" 03:20:56 DEBUG }, 03:20:56 DEBUG "value": [ 03:20:56 DEBUG 03:20:56 DEBUG 03:20:56 DEBUG 03:20:56 DEBUG "value": [ 03:20:56 DEBUG "pod": "ds-cts-0" 03:20:56 DEBUG 03:20:56 DEBUG }, 03:20:56 DEBUG "value": [ 03:20:56 DEBUG 03:20:56 DEBUG }, 03:20:56 DEBUG "value": [ 03:20:56 DEBUG 03:20:56 DEBUG 03:20:56 DEBUG }, 03:20:56 DEBUG "value": [ 03:20:56 DEBUG "result": [ 03:20:56 DEBUG 03:20:56 DEBUG 03:20:56 DEBUG 03:20:56 DEBUG 03:20:56 DEBUG }, 03:20:56 DEBUG 03:20:56 DEBUG "value": [ 03:20:56 DEBUG 03:20:56 DEBUG 03:20:56 DEBUG "pod": "am-767f6b67b5-vbltz" 03:20:56 DEBUG 03:20:56 DEBUG 03:20:56 DEBUG }, 03:20:56 DEBUG "value": [ 03:20:56 DEBUG 03:20:56 DEBUG { 03:20:56 DEBUG [...] 03:20:56 DEBUG 03:20:56 DEBUG 03:20:56 DEBUG 03:20:56 DEBUG "value": [ 03:20:56 DEBUG 03:20:56 DEBUG 03:20:56 DEBUG 03:20:56 DEBUG 03:20:56 DEBUG }, 03:20:56 DEBUG 03:20:56 DEBUG [...] 03:20:56 DEBUG "value": [ 03:20:56 DEBUG 03:20:56 DEBUG 03:20:56 DEBUG "metric": { 03:20:56 DEBUG 03:20:56 DEBUG [...] 03:20:56 DEBUG [...] 03:20:56 DEBUG 03:20:56 DEBUG 03:20:56 DEBUG "pod": "am-767f6b67b5-vbltz" 03:20:56 DEBUG 03:20:56 DEBUG 03:20:56 DEBUG [...] 03:20:56 DEBUG "value": [ 03:20:56 DEBUG 03:20:56 DEBUG 03:20:56 DEBUG 03:20:56 DEBUG 03:20:56 DEBUG 03:20:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:20:56 DEBUG 03:20:56 DEBUG 03:20:56 DEBUG 03:20:56 DEBUG 03:20:56 DEBUG 03:20:56 DEBUG }, 03:20:56 DEBUG 03:20:56 DEBUG [...] 03:20:56 DEBUG 03:20:56 DEBUG 1713666056, 03:20:56 DEBUG [...] 03:20:56 DEBUG 03:20:56 DEBUG 03:20:56 DEBUG 03:20:56 DEBUG [...] 03:20:56 DEBUG }, 03:20:56 DEBUG 03:20:56 DEBUG 03:20:56 DEBUG [...] 03:20:56 DEBUG 03:20:56 DEBUG 03:20:56 DEBUG "value": [ 03:20:56 DEBUG [...] 03:20:56 DEBUG 03:20:56 DEBUG 03:20:56 DEBUG "600703.5" 03:20:56 DEBUG 03:20:56 DEBUG 03:20:56 DEBUG 03:20:56 DEBUG "instance": "10.142.0.43:9100" 03:20:56 DEBUG 03:20:56 DEBUG "value": [ 03:20:56 DEBUG "pod": "ds-idrepo-2" 03:20:56 DEBUG 03:20:56 DEBUG 03:20:56 DEBUG 03:20:56 DEBUG [...] 03:20:56 DEBUG 1713666056, 03:20:56 DEBUG 03:20:56 DEBUG 03:20:56 DEBUG 03:20:56 DEBUG ] 03:20:56 DEBUG } 03:20:56 DEBUG 03:20:56 DEBUG }, 03:20:56 DEBUG [...] 03:20:56 DEBUG 03:20:56 DEBUG 03:20:56 DEBUG }, 03:20:56 DEBUG "instance": "10.142.0.43:9100" 03:20:56 DEBUG 03:20:56 DEBUG "pod": "am-767f6b67b5-vbltz" 03:20:56 DEBUG 03:20:56 DEBUG "149.889" 03:20:56 DEBUG 03:20:56 DEBUG 03:20:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:20:56 DEBUG 03:20:56 DEBUG ] 03:20:56 DEBUG [...] 03:20:56 DEBUG "value": [ 03:20:56 DEBUG 03:20:56 DEBUG 03:20:56 DEBUG 03:20:56 DEBUG "value": [ 03:20:56 DEBUG }, 03:20:56 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:20:56 DEBUG }, 03:20:56 DEBUG 03:20:56 DEBUG ] 03:20:56 DEBUG 03:20:56 DEBUG "pod": "ds-cts-2" 03:20:56 DEBUG }, 03:20:56 DEBUG "pod": "ds-cts-2" 03:20:56 DEBUG } 03:20:56 DEBUG 03:20:56 DEBUG 03:20:56 DEBUG 03:20:56 DEBUG 03:20:56 DEBUG 03:20:56 DEBUG 1713666056, 03:20:56 DEBUG "value": [ 03:20:56 DEBUG }, 03:20:56 DEBUG "value": [ 03:20:56 DEBUG 03:20:56 DEBUG } 03:20:56 DEBUG "pod": "am-767f6b67b5-vbltz" 03:20:56 DEBUG }, 03:20:56 DEBUG "value": [ 03:20:56 DEBUG }, 03:20:56 DEBUG } 03:20:56 DEBUG 1713666056, 03:20:56 DEBUG "instance": "10.142.0.43:9100" 03:20:56 DEBUG 03:20:56 DEBUG "instance": "10.142.0.43:9100" 03:20:56 DEBUG [...] 03:20:56 DEBUG "0" 03:20:56 DEBUG 1713666056, 03:20:56 DEBUG "value": [ 03:20:56 DEBUG 1713666056, 03:20:56 DEBUG "pod": "ds-cts-2" 03:20:56 DEBUG }, 03:20:56 DEBUG ] 03:20:56 DEBUG 1713666056, 03:20:56 DEBUG "value": [ 03:20:56 DEBUG "value": [ 03:20:56 DEBUG "4401638870016" 03:20:56 DEBUG }, 03:20:56 DEBUG "pod": "ds-idrepo-2" 03:20:56 DEBUG }, 03:20:56 DEBUG 03:20:56 DEBUG 03:20:56 DEBUG 03:20:56 DEBUG 1713666056, 03:20:56 DEBUG "3330.2999999999997" 03:20:56 DEBUG }, 03:20:56 DEBUG "value": [ 03:20:56 DEBUG } 03:20:56 DEBUG "0" 03:20:56 DEBUG 1713666056, 03:20:56 DEBUG 1713666056, 03:20:56 DEBUG ] 03:20:56 DEBUG "value": [ 03:20:56 DEBUG }, 03:20:56 DEBUG "value": [ 03:20:56 DEBUG ] 03:20:56 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:20:56 DEBUG "0" 03:20:56 DEBUG ] 03:20:56 DEBUG "value": [ 03:20:56 DEBUG 1713666056, 03:20:56 DEBUG } 03:20:56 DEBUG "99834988.4" 03:20:56 DEBUG ] 03:20:56 DEBUG "1034719232" 03:20:56 DEBUG "0" 03:20:56 DEBUG } 03:20:56 DEBUG 1713666056, 03:20:56 DEBUG "value": [ 03:20:56 DEBUG 1713666056, 03:20:56 DEBUG } 03:20:56 DEBUG }, 03:20:56 DEBUG ] 03:20:56 DEBUG } 03:20:56 DEBUG 1713666056, 03:20:56 DEBUG "30.333333333333332" 03:20:56 DEBUG ] 03:20:56 DEBUG } 03:20:56 DEBUG ] 03:20:56 DEBUG ] 03:20:56 DEBUG ] 03:20:56 DEBUG "0.0816666666666606" 03:20:56 DEBUG 1713666056, 03:20:56 DEBUG "99830552.53333333" 03:20:56 DEBUG ] 03:20:56 DEBUG "value": [ 03:20:56 DEBUG } 03:20:56 DEBUG ] 03:20:56 DEBUG "3333.733333333333" 03:20:56 DEBUG ] 03:20:56 DEBUG } 03:20:56 DEBUG ] 03:20:56 DEBUG } 03:20:56 DEBUG } 03:20:56 DEBUG } 03:20:56 DEBUG ] 03:20:56 DEBUG "0" 03:20:56 DEBUG ] 03:20:56 DEBUG } 03:20:56 DEBUG 1713666056, 03:20:56 DEBUG ] 03:20:56 DEBUG } 03:20:56 DEBUG ] 03:20:56 DEBUG } 03:20:56 DEBUG ] 03:20:56 DEBUG } 03:20:56 DEBUG ] 03:20:56 DEBUG ] 03:20:56 DEBUG } 03:20:56 DEBUG } 03:20:56 DEBUG ] 03:20:56 DEBUG } 03:20:56 DEBUG } 03:20:56 DEBUG "0" 03:20:56 DEBUG } 03:20:56 DEBUG } 03:20:56 DEBUG } 03:20:56 DEBUG ] 03:20:56 DEBUG } 03:20:56 DEBUG } 03:20:56 DEBUG } 03:20:56 DEBUG } 03:20:56 DEBUG ] 03:20:56 DEBUG } 03:20:56 DEBUG ] 03:20:56 DEBUG ] 03:20:56 DEBUG } 03:20:56 DEBUG ] 03:20:56 DEBUG } 03:20:56 DEBUG } 03:20:56 DEBUG } 03:20:56 DEBUG } 03:20:56 DEBUG } 03:20:56 DEBUG ] 03:20:56 DEBUG } 03:20:56 DEBUG } 03:20:56 DEBUG } 03:20:56 DEBUG } 03:20:56 DEBUG } 03:20:56 DEBUG } 03:20:56 DEBUG } 03:20:56 DEBUG ] 03:20:56 DEBUG } 03:20:56 DEBUG } 03:20:56 DEBUG } 03:20:56 DEBUG } 03:21:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713666113" 03:21:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666113" 03:21:53 ERROR [http_cmd]: ERROR 03:21:53 ERROR --- status code --- 03:21:53 ERROR http status code is 400 (expected [200, 401, 403, 404]) 03:21:53 ERROR --- http response --- 03:21:53 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 03:21:53 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 03:21:53 INFO [http_cmd]: http status code OK 03:21:53 DEBUG --- status code --- 03:21:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:21:53 DEBUG --- http response --- 03:21:53 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 03:21:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666114" 03:21:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666114" 03:21:54 INFO [http_cmd]: http status code OK 03:21:54 INFO [http_cmd]: http status code OK 03:21:54 DEBUG --- status code --- 03:21:54 DEBUG --- status code --- 03:21:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:21:54 DEBUG --- http response --- 03:21:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:21:54 DEBUG --- http response --- 03:21:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:21:54 DEBUG ----- output ----- 03:21:54 DEBUG { 03:21:54 DEBUG "status": "success", 03:21:54 DEBUG "data": { 03:21:54 DEBUG "resultType": "vector", 03:21:54 DEBUG "result": [ 03:21:54 DEBUG { 03:21:54 DEBUG "metric": { 03:21:54 DEBUG "pod": "ds-cts-0" 03:21:54 DEBUG }, 03:21:54 DEBUG "value": [ 03:21:54 DEBUG 03:21:54 DEBUG 03:21:54 DEBUG 03:21:54 DEBUG [...] 03:21:54 DEBUG 03:21:54 DEBUG 03:21:54 DEBUG 03:21:54 DEBUG "pod": "ds-cts-2" 03:21:54 DEBUG }, 03:21:54 DEBUG "value": [ 03:21:54 DEBUG 1713666114, 03:21:54 DEBUG "0.5041666666666667" 03:21:54 DEBUG ] 03:21:54 DEBUG } 03:21:54 DEBUG ] 03:21:54 DEBUG } 03:21:54 DEBUG } 03:21:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:21:54 DEBUG ----- output ----- 03:21:54 DEBUG { 03:21:54 DEBUG "status": "success", 03:21:54 DEBUG "data": { 03:21:54 DEBUG "resultType": "vector", 03:21:54 DEBUG "result": [ 03:21:54 DEBUG { 03:21:54 DEBUG "metric": { 03:21:54 DEBUG "instance": "10.142.0.107:9100" 03:21:54 DEBUG }, 03:21:54 DEBUG "value": [ 03:21:54 DEBUG 03:21:54 DEBUG 03:21:54 DEBUG 03:21:54 DEBUG [...] 03:21:54 DEBUG 03:21:54 DEBUG 03:21:54 DEBUG 03:21:54 DEBUG "instance": "10.142.0.43:9100" 03:21:54 DEBUG }, 03:21:54 DEBUG "value": [ 03:21:54 DEBUG 1713666114, 03:21:54 DEBUG "0.3923999999999466" 03:21:54 DEBUG ] 03:21:54 DEBUG } 03:21:54 DEBUG ] 03:21:54 DEBUG } 03:21:54 DEBUG } 03:21:54 INFO 03:21:54 INFO [loop_until]: kubectl --namespace=xlou top pods 03:21:54 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:21:54 INFO [loop_until]: OK (rc = 0) 03:21:54 DEBUG --- stdout --- 03:21:54 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 22140m 8796Mi am-767f6b67b5-qcqz2 22310m 9128Mi am-767f6b67b5-vbltz 24200m 8823Mi ds-cts-0 20289m 12492Mi ds-cts-1 19645m 12502Mi ds-cts-2 20821m 12562Mi ds-idrepo-0 9539m 3918Mi ds-idrepo-1 9410m 3947Mi ds-idrepo-2 9549m 3916Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1179Mi idm-b85f9d978-wnhxb 3m 1239Mi lodemon-54c87f8cbb-jzrjs 13m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-68c5d5b884-p97w5 11528m 897Mi 03:21:54 DEBUG --- stderr --- 03:21:54 DEBUG 03:21:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666115" 03:21:55 INFO [http_cmd]: http status code OK 03:21:55 DEBUG --- status code --- 03:21:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:21:55 DEBUG --- http response --- 03:21:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:21:55 DEBUG ----- output ----- 03:21:55 DEBUG { 03:21:55 DEBUG "status": "success", 03:21:55 DEBUG "data": { 03:21:55 DEBUG "resultType": "vector", 03:21:55 DEBUG "result": [ 03:21:55 DEBUG { 03:21:55 DEBUG "metric": { 03:21:55 DEBUG "pod": "am-767f6b67b5-9zw22" 03:21:55 DEBUG }, 03:21:55 DEBUG "value": [ 03:21:55 DEBUG 03:21:55 DEBUG 03:21:55 DEBUG 03:21:55 DEBUG [...] 03:21:55 DEBUG 03:21:55 DEBUG 03:21:55 DEBUG 03:21:55 DEBUG "pod": "am-767f6b67b5-vbltz" 03:21:55 DEBUG }, 03:21:55 DEBUG "value": [ 03:21:55 DEBUG 1713666115, 03:21:55 DEBUG "3330.6666666666665" 03:21:55 DEBUG ] 03:21:55 DEBUG } 03:21:55 DEBUG ] 03:21:55 DEBUG } 03:21:55 DEBUG } 03:21:55 INFO 03:21:55 INFO [loop_until]: kubectl --namespace=xlou top node 03:21:55 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:21:55 INFO [loop_until]: OK (rc = 0) 03:21:55 DEBUG --- stdout --- 03:21:55 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 62m 0% 6280Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 9912m 33% 7760Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 22072m 73% 11606Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 82m 0% 3344Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 20259m 67% 15796Mi 14% gke-xlou-cdm-default-pool-1b0769c8-wpwl 156m 0% 5268Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 22296m 74% 12050Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 19658m 65% 3178Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 23554m 78% 11086Mi 9% gke-xlou-cdm-ds-0f601751-j488 10352m 34% 7349Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 19802m 66% 17633Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 10173m 34% 8020Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20342m 68% 15281Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 11944m 75% 11227Mi 19% 03:21:55 DEBUG --- stderr --- 03:21:55 DEBUG 03:21:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666116" 03:21:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666116" 03:21:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666116" 03:21:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713666116" 03:21:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666116" 03:21:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666116" 03:21:56 INFO [http_cmd]: http status code OK 03:21:56 INFO [http_cmd]: http status code OK 03:21:56 DEBUG --- status code --- 03:21:56 DEBUG --- status code --- 03:21:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:21:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:21:56 DEBUG --- http response --- 03:21:56 DEBUG --- http response --- 03:21:56 INFO [http_cmd]: http status code OK 03:21:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:21:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:21:56 INFO [http_cmd]: http status code OK 03:21:56 DEBUG --- status code --- 03:21:56 INFO [http_cmd]: http status code OK 03:21:56 DEBUG ----- output ----- 03:21:56 DEBUG ----- output ----- 03:21:56 DEBUG --- status code --- 03:21:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:21:56 DEBUG --- status code --- 03:21:56 DEBUG { 03:21:56 DEBUG { 03:21:56 DEBUG "status": "success", 03:21:56 DEBUG --- http response --- 03:21:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:21:56 DEBUG "status": "success", 03:21:56 DEBUG "data": { 03:21:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:21:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:21:56 DEBUG --- http response --- 03:21:56 DEBUG "data": { 03:21:56 DEBUG "resultType": "vector", 03:21:56 DEBUG "result": [ 03:21:56 DEBUG ----- output ----- 03:21:56 DEBUG --- http response --- 03:21:56 DEBUG "resultType": "vector", 03:21:56 DEBUG { 03:21:56 DEBUG { 03:21:56 DEBUG "status": "success", 03:21:56 DEBUG "result": [ 03:21:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:21:56 DEBUG "metric": { 03:21:56 DEBUG "data": { 03:21:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:21:56 DEBUG { 03:21:56 DEBUG "metric": { 03:21:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:21:56 DEBUG "resultType": "vector", 03:21:56 DEBUG ----- output ----- 03:21:56 INFO [http_cmd]: http status code OK 03:21:56 DEBUG "pod": "ds-idrepo-0" 03:21:56 DEBUG }, 03:21:56 DEBUG "value": [ 03:21:56 DEBUG { 03:21:56 DEBUG ----- output ----- 03:21:56 DEBUG --- status code --- 03:21:56 DEBUG }, 03:21:56 DEBUG "result": [ 03:21:56 DEBUG 03:21:56 DEBUG "status": "success", 03:21:56 DEBUG { 03:21:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:21:56 DEBUG --- http response --- 03:21:56 DEBUG { 03:21:56 DEBUG 03:21:56 DEBUG "data": { 03:21:56 DEBUG "status": "success", 03:21:56 DEBUG "value": [ 03:21:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:21:56 DEBUG "metric": { 03:21:56 DEBUG "pod": "am-767f6b67b5-9zw22" 03:21:56 DEBUG "resultType": "vector", 03:21:56 DEBUG "data": { 03:21:56 DEBUG 03:21:56 DEBUG ----- output ----- 03:21:56 DEBUG }, 03:21:56 DEBUG 03:21:56 DEBUG "result": [ 03:21:56 DEBUG "resultType": "vector", 03:21:56 DEBUG 03:21:56 DEBUG 03:21:56 DEBUG "value": [ 03:21:56 DEBUG [...] 03:21:56 DEBUG { 03:21:56 DEBUG "result": [ 03:21:56 DEBUG { 03:21:56 DEBUG [...] 03:21:56 DEBUG 03:21:56 DEBUG 03:21:56 DEBUG "metric": { 03:21:56 DEBUG { 03:21:56 DEBUG "status": "success", 03:21:56 DEBUG 03:21:56 DEBUG 03:21:56 DEBUG 03:21:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:21:56 DEBUG "metric": { 03:21:56 DEBUG "data": { 03:21:56 DEBUG 03:21:56 DEBUG 03:21:56 DEBUG 03:21:56 DEBUG }, 03:21:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:21:56 DEBUG "resultType": "vector", 03:21:56 DEBUG 03:21:56 DEBUG [...] 03:21:56 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:21:56 DEBUG "value": [ 03:21:56 DEBUG }, 03:21:56 DEBUG "result": [ 03:21:56 DEBUG "pod": "ds-idrepo-2" 03:21:56 DEBUG }, 03:21:56 DEBUG }, 03:21:56 DEBUG 03:21:56 DEBUG 03:21:56 DEBUG { 03:21:56 DEBUG 03:21:56 DEBUG "value": [ 03:21:56 DEBUG "value": [ 03:21:56 DEBUG "value": [ 03:21:56 DEBUG 03:21:56 DEBUG "metric": { 03:21:56 DEBUG 03:21:56 DEBUG 1713666116, 03:21:56 DEBUG 1713666116, 03:21:56 DEBUG 03:21:56 DEBUG [...] 03:21:56 DEBUG "pod": "am-767f6b67b5-9zw22" 03:21:56 DEBUG 03:21:56 DEBUG "0" 03:21:56 DEBUG "23532.6223562785" 03:21:56 DEBUG 03:21:56 DEBUG 03:21:56 DEBUG }, 03:21:56 DEBUG "pod": "am-767f6b67b5-vbltz" 03:21:56 DEBUG ] 03:21:56 DEBUG ] 03:21:56 DEBUG 03:21:56 DEBUG 03:21:56 DEBUG "value": [ 03:21:56 DEBUG }, 03:21:56 DEBUG } 03:21:56 DEBUG } 03:21:56 DEBUG [...] 03:21:56 DEBUG 03:21:56 DEBUG 03:21:56 DEBUG "value": [ 03:21:56 DEBUG ] 03:21:56 DEBUG ] 03:21:56 DEBUG 03:21:56 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:21:56 DEBUG 03:21:56 DEBUG 1713666116, 03:21:56 DEBUG } 03:21:56 DEBUG } 03:21:56 DEBUG 03:21:56 DEBUG }, 03:21:56 DEBUG 03:21:56 DEBUG "0" 03:21:56 DEBUG } 03:21:56 DEBUG } 03:21:56 DEBUG 03:21:56 DEBUG "value": [ 03:21:56 DEBUG [...] 03:21:56 DEBUG ] 03:21:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:21:56 DEBUG 1713666116, 03:21:56 DEBUG 03:21:56 DEBUG } 03:21:56 DEBUG }, 03:21:56 DEBUG "11.696215490286905" 03:21:56 DEBUG 03:21:56 DEBUG ] 03:21:56 DEBUG "value": [ 03:21:56 DEBUG ] 03:21:56 DEBUG 03:21:56 DEBUG } 03:21:56 DEBUG 1713666116, 03:21:56 DEBUG } 03:21:56 DEBUG "pod": "am-767f6b67b5-vbltz" 03:21:56 DEBUG } 03:21:56 DEBUG "11.74263243189657" 03:21:56 DEBUG ] 03:21:56 DEBUG }, 03:21:56 DEBUG ] 03:21:56 DEBUG } 03:21:56 DEBUG "value": [ 03:21:56 DEBUG } 03:21:56 DEBUG } 03:21:56 DEBUG 1713666116, 03:21:56 DEBUG ] 03:21:56 DEBUG "3332.9333333333334" 03:21:56 DEBUG } 03:21:56 DEBUG ] 03:21:56 DEBUG } 03:21:56 DEBUG } 03:21:56 DEBUG ] 03:21:56 DEBUG } 03:21:56 DEBUG } 03:21:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666116" 03:21:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713666116" 03:21:56 INFO [http_cmd]: http status code OK 03:21:56 DEBUG --- status code --- 03:21:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:21:56 DEBUG --- http response --- 03:21:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:21:56 DEBUG ----- output ----- 03:21:56 DEBUG { 03:21:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666116" 03:21:56 DEBUG "status": "success", 03:21:56 DEBUG "data": { 03:21:56 DEBUG "resultType": "vector", 03:21:56 DEBUG "result": [ 03:21:56 DEBUG { 03:21:56 DEBUG "metric": { 03:21:56 DEBUG "pod": "am-767f6b67b5-9zw22" 03:21:56 DEBUG }, 03:21:56 DEBUG "value": [ 03:21:56 DEBUG 03:21:56 DEBUG 03:21:56 DEBUG 03:21:56 DEBUG [...] 03:21:56 DEBUG 03:21:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666116" 03:21:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666116" 03:21:56 DEBUG 03:21:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713666116" 03:21:56 DEBUG 03:21:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666116" 03:21:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666116" 03:21:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713666116" 03:21:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666116" 03:21:56 INFO [http_cmd]: http status code OK 03:21:56 DEBUG "pod": "am-767f6b67b5-vbltz" 03:21:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666116" 03:21:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666116" 03:21:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666116" 03:21:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666116" 03:21:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666116" 03:21:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666116" 03:21:56 INFO [http_cmd]: http status code OK 03:21:56 INFO [http_cmd]: http status code OK 03:21:56 INFO [http_cmd]: http status code OK 03:21:56 DEBUG --- status code --- 03:21:56 DEBUG }, 03:21:56 INFO [http_cmd]: http status code OK 03:21:56 INFO [http_cmd]: http status code OK 03:21:56 INFO [http_cmd]: http status code OK 03:21:56 INFO [http_cmd]: http status code OK 03:21:56 INFO [http_cmd]: http status code OK 03:21:56 DEBUG --- status code --- 03:21:56 DEBUG --- status code --- 03:21:56 DEBUG --- status code --- 03:21:56 INFO [http_cmd]: http status code OK 03:21:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:21:56 DEBUG "value": [ 03:21:56 INFO [http_cmd]: http status code OK 03:21:56 INFO [http_cmd]: http status code OK 03:21:56 DEBUG --- status code --- 03:21:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:21:56 INFO [http_cmd]: http status code OK 03:21:56 DEBUG --- status code --- 03:21:56 DEBUG --- status code --- 03:21:56 DEBUG --- status code --- 03:21:56 DEBUG --- status code --- 03:21:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:21:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:21:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:21:56 DEBUG --- status code --- 03:21:56 DEBUG --- http response --- 03:21:56 INFO [http_cmd]: http status code OK 03:21:56 DEBUG 1713666116, 03:21:56 DEBUG --- status code --- 03:21:56 DEBUG --- status code --- 03:21:56 INFO [http_cmd]: http status code OK 03:21:56 DEBUG --- http response --- 03:21:56 DEBUG --- status code --- 03:21:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:21:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:21:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:21:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:21:56 DEBUG --- http response --- 03:21:56 DEBUG --- http response --- 03:21:56 DEBUG --- http response --- 03:21:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:21:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:21:56 DEBUG --- status code --- 03:21:56 DEBUG "600318" 03:21:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:21:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:21:56 DEBUG --- status code --- 03:21:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:21:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:21:56 DEBUG --- http response --- 03:21:56 DEBUG --- http response --- 03:21:56 DEBUG --- http response --- 03:21:56 DEBUG --- http response --- 03:21:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:21:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:21:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:21:56 DEBUG --- http response --- 03:21:56 DEBUG ----- output ----- 03:21:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:21:56 DEBUG ] 03:21:56 DEBUG --- http response --- 03:21:56 DEBUG --- http response --- 03:21:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:21:56 DEBUG --- http response --- 03:21:56 DEBUG ----- output ----- 03:21:56 DEBUG { 03:21:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:21:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:21:56 DEBUG ----- output ----- 03:21:56 DEBUG ----- output ----- 03:21:56 DEBUG ----- output ----- 03:21:56 DEBUG ----- output ----- 03:21:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:21:56 DEBUG { 03:21:56 DEBUG --- http response --- 03:21:56 DEBUG } 03:21:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:21:56 DEBUG --- http response --- 03:21:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:21:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:21:56 DEBUG "status": "success", 03:21:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:21:56 DEBUG ----- output ----- 03:21:56 DEBUG { 03:21:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:21:56 DEBUG { 03:21:56 DEBUG { 03:21:56 DEBUG { 03:21:56 DEBUG "status": "success", 03:21:56 DEBUG "status": "success", 03:21:56 DEBUG "data": { 03:21:56 DEBUG ] 03:21:56 DEBUG ----- output ----- 03:21:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:21:56 DEBUG ----- output ----- 03:21:56 DEBUG ----- output ----- 03:21:56 DEBUG "data": { 03:21:56 DEBUG ----- output ----- 03:21:56 DEBUG { 03:21:56 DEBUG "status": "success", 03:21:56 DEBUG ----- output ----- 03:21:56 DEBUG "status": "success", 03:21:56 DEBUG "status": "success", 03:21:56 DEBUG ----- output ----- 03:21:56 DEBUG "data": { 03:21:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:21:56 DEBUG "resultType": "vector", 03:21:56 DEBUG } 03:21:56 DEBUG { 03:21:56 DEBUG ----- output ----- 03:21:56 DEBUG { 03:21:56 DEBUG { 03:21:56 DEBUG "resultType": "vector", 03:21:56 DEBUG "status": "success", 03:21:56 DEBUG { 03:21:56 DEBUG "status": "success", 03:21:56 DEBUG { 03:21:56 DEBUG "data": { 03:21:56 DEBUG "data": { 03:21:56 DEBUG { 03:21:56 DEBUG "resultType": "vector", 03:21:56 DEBUG ----- output ----- 03:21:56 DEBUG "result": [ 03:21:56 DEBUG } 03:21:56 DEBUG "status": "success", 03:21:56 DEBUG { 03:21:56 DEBUG "status": "success", 03:21:56 DEBUG "status": "success", 03:21:56 DEBUG "result": [ 03:21:56 DEBUG "data": { 03:21:56 DEBUG "data": { 03:21:56 DEBUG "data": { 03:21:56 DEBUG "status": "success", 03:21:56 DEBUG "resultType": "vector", 03:21:56 DEBUG "resultType": "vector", 03:21:56 DEBUG "status": "success", 03:21:56 DEBUG "result": [ 03:21:56 DEBUG { 03:21:56 DEBUG "status": "success", 03:21:56 DEBUG "data": { 03:21:56 DEBUG "status": "success", 03:21:56 DEBUG "data": { 03:21:56 DEBUG "data": { 03:21:56 DEBUG { 03:21:56 DEBUG "resultType": "vector", 03:21:56 DEBUG "resultType": "vector", 03:21:56 DEBUG "resultType": "vector", 03:21:56 DEBUG "data": { 03:21:56 DEBUG "result": [ 03:21:56 DEBUG "result": [ 03:21:56 DEBUG "data": { 03:21:56 DEBUG { 03:21:56 DEBUG "data": { 03:21:56 DEBUG { 03:21:56 DEBUG "resultType": "vector", 03:21:56 DEBUG "data": { 03:21:56 DEBUG "resultType": "vector", 03:21:56 DEBUG "resultType": "vector", 03:21:56 DEBUG "metric": { 03:21:56 DEBUG "result": [ 03:21:56 DEBUG "result": [ 03:21:56 DEBUG "result": [ 03:21:56 DEBUG "resultType": "vector", 03:21:56 DEBUG { 03:21:56 DEBUG { 03:21:56 DEBUG "resultType": "vector", 03:21:56 DEBUG "metric": { 03:21:56 DEBUG "resultType": "vector", 03:21:56 DEBUG "metric": { 03:21:56 DEBUG "result": [ 03:21:56 DEBUG "resultType": "vector", 03:21:56 DEBUG "result": [ 03:21:56 DEBUG { 03:21:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:21:56 DEBUG { 03:21:56 DEBUG { 03:21:56 DEBUG { 03:21:56 DEBUG "result": [ 03:21:56 DEBUG "metric": { 03:21:56 DEBUG "pod": "am-767f6b67b5-9zw22" 03:21:56 DEBUG "result": [ 03:21:56 DEBUG "pod": "ds-idrepo-0" 03:21:56 DEBUG "result": [ 03:21:56 DEBUG "pod": "am-767f6b67b5-9zw22" 03:21:56 DEBUG { 03:21:56 DEBUG "result": [ 03:21:56 DEBUG { 03:21:56 DEBUG "metric": { 03:21:56 DEBUG }, 03:21:56 DEBUG "metric": { 03:21:56 DEBUG "metric": { 03:21:56 DEBUG "metric": { 03:21:56 DEBUG { 03:21:56 DEBUG }, 03:21:56 DEBUG "metric": { 03:21:56 DEBUG { 03:21:56 DEBUG }, 03:21:56 DEBUG { 03:21:56 DEBUG }, 03:21:56 DEBUG "metric": { 03:21:56 DEBUG "result": [ 03:21:56 DEBUG "metric": { 03:21:56 DEBUG "instance": "10.142.0.107:9100" 03:21:56 DEBUG "value": [ 03:21:56 DEBUG "pod": "ds-idrepo-0" 03:21:56 DEBUG "instance": "10.142.0.107:9100" 03:21:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:21:56 DEBUG "metric": { 03:21:56 DEBUG "value": [ 03:21:56 DEBUG "instance": "10.142.0.107:9100" 03:21:56 DEBUG "metric": { 03:21:56 DEBUG "value": [ 03:21:56 DEBUG "metric": { 03:21:56 DEBUG "value": [ 03:21:56 DEBUG "instance": "10.142.0.107:9100" 03:21:56 DEBUG { 03:21:56 DEBUG "pod": "am-767f6b67b5-9zw22" 03:21:56 DEBUG }, 03:21:56 DEBUG 03:21:56 DEBUG }, 03:21:56 DEBUG }, 03:21:56 DEBUG }, 03:21:56 DEBUG "pod": "ds-cts-0" 03:21:56 DEBUG 03:21:56 DEBUG 03:21:56 DEBUG "pod": "ds-cts-0" 03:21:56 DEBUG 03:21:56 DEBUG "pod": "ds-idrepo-0" 03:21:56 DEBUG 03:21:56 DEBUG }, 03:21:56 DEBUG "metric": { 03:21:56 DEBUG }, 03:21:56 DEBUG "value": [ 03:21:56 DEBUG 03:21:56 DEBUG "value": [ 03:21:56 DEBUG "value": [ 03:21:56 DEBUG "value": [ 03:21:56 DEBUG }, 03:21:56 DEBUG "value": [ 03:21:56 DEBUG }, 03:21:56 DEBUG }, 03:21:56 DEBUG 03:21:56 DEBUG }, 03:21:56 DEBUG "value": [ 03:21:56 DEBUG "value": [ 03:21:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:21:56 DEBUG "value": [ 03:21:56 DEBUG 03:21:56 DEBUG 03:21:56 DEBUG 03:21:56 DEBUG 03:21:56 DEBUG 03:21:56 DEBUG 03:21:56 DEBUG 03:21:56 DEBUG "value": [ 03:21:56 DEBUG "value": [ 03:21:56 DEBUG 03:21:56 DEBUG 03:21:56 DEBUG 03:21:56 DEBUG 03:21:56 DEBUG }, 03:21:56 DEBUG 03:21:56 DEBUG 03:21:56 DEBUG [...] 03:21:56 DEBUG 03:21:56 DEBUG 03:21:56 DEBUG 03:21:56 DEBUG 03:21:56 DEBUG [...] 03:21:56 DEBUG 03:21:56 DEBUG 03:21:56 DEBUG [...] 03:21:56 DEBUG 03:21:56 DEBUG 03:21:56 DEBUG 03:21:56 DEBUG "value": [ 03:21:56 DEBUG 03:21:56 DEBUG 03:21:56 DEBUG 03:21:56 DEBUG 03:21:56 DEBUG [...] 03:21:56 DEBUG 03:21:56 DEBUG 03:21:56 DEBUG [...] 03:21:56 DEBUG 03:21:56 DEBUG 03:21:56 DEBUG 03:21:56 DEBUG 03:21:56 DEBUG [...] 03:21:56 DEBUG 03:21:56 DEBUG 03:21:56 DEBUG 03:21:56 DEBUG [...] 03:21:56 DEBUG 03:21:56 DEBUG 03:21:56 DEBUG 03:21:56 DEBUG [...] 03:21:56 DEBUG 03:21:56 DEBUG 03:21:56 DEBUG 03:21:56 DEBUG 03:21:56 DEBUG 03:21:56 DEBUG [...] 03:21:56 DEBUG 03:21:56 DEBUG [...] 03:21:56 DEBUG 03:21:56 DEBUG [...] 03:21:56 DEBUG 03:21:56 DEBUG 03:21:56 DEBUG [...] 03:21:56 DEBUG 03:21:56 DEBUG 03:21:56 DEBUG 03:21:56 DEBUG [...] 03:21:56 DEBUG [...] 03:21:56 DEBUG 03:21:56 DEBUG 03:21:56 DEBUG 03:21:56 DEBUG 03:21:56 DEBUG 03:21:56 DEBUG 03:21:56 DEBUG 03:21:56 DEBUG 03:21:56 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:21:56 DEBUG 03:21:56 DEBUG 03:21:56 DEBUG 03:21:56 DEBUG 03:21:56 DEBUG 03:21:56 DEBUG 03:21:56 DEBUG 03:21:56 DEBUG "pod": "ds-idrepo-2" 03:21:56 DEBUG 03:21:56 DEBUG 03:21:56 DEBUG 03:21:56 DEBUG [...] 03:21:56 DEBUG 03:21:56 DEBUG 03:21:56 DEBUG }, 03:21:56 DEBUG 03:21:56 DEBUG "pod": "ds-cts-2" 03:21:56 DEBUG 03:21:56 DEBUG "pod": "ds-cts-2" 03:21:56 DEBUG 03:21:56 DEBUG 03:21:56 DEBUG "pod": "am-767f6b67b5-vbltz" 03:21:56 DEBUG }, 03:21:56 DEBUG 03:21:56 DEBUG "pod": "am-767f6b67b5-vbltz" 03:21:56 DEBUG 03:21:56 DEBUG 03:21:56 DEBUG 03:21:56 DEBUG "instance": "10.142.0.43:9100" 03:21:56 DEBUG "value": [ 03:21:56 DEBUG 03:21:56 DEBUG }, 03:21:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:21:56 DEBUG }, 03:21:56 DEBUG 03:21:56 DEBUG 03:21:56 DEBUG "value": [ 03:21:56 DEBUG }, 03:21:56 DEBUG "pod": "ds-idrepo-2" 03:21:56 DEBUG }, 03:21:56 DEBUG "instance": "10.142.0.43:9100" 03:21:56 DEBUG 03:21:56 DEBUG "pod": "am-767f6b67b5-vbltz" 03:21:56 DEBUG }, 03:21:56 DEBUG 1713666116, 03:21:56 DEBUG "instance": "10.142.0.43:9100" 03:21:56 DEBUG "value": [ 03:21:56 DEBUG }, 03:21:56 DEBUG "value": [ 03:21:56 DEBUG "instance": "10.142.0.43:9100" 03:21:56 DEBUG "pod": "ds-cts-2" 03:21:56 DEBUG 1713666116, 03:21:56 DEBUG "value": [ 03:21:56 DEBUG }, 03:21:56 DEBUG "value": [ 03:21:56 DEBUG }, 03:21:56 DEBUG 03:21:56 DEBUG }, 03:21:56 DEBUG "value": [ 03:21:56 DEBUG "0" 03:21:56 DEBUG }, 03:21:56 DEBUG 1713666116, 03:21:56 DEBUG "value": [ 03:21:56 DEBUG 1713666116, 03:21:56 DEBUG "0" 03:21:56 DEBUG }, 03:21:56 DEBUG "3315" 03:21:56 DEBUG 1713666116, 03:21:56 DEBUG }, 03:21:56 DEBUG 1713666116, 03:21:56 DEBUG "value": [ 03:21:56 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:21:56 DEBUG "value": [ 03:21:56 DEBUG 1713666116, 03:21:56 DEBUG ] 03:21:56 DEBUG "value": [ 03:21:56 DEBUG "0" 03:21:56 DEBUG 1713666116, 03:21:56 DEBUG ] 03:21:56 DEBUG "value": [ 03:21:56 DEBUG ] 03:21:56 DEBUG "0" 03:21:56 DEBUG "value": [ 03:21:56 DEBUG "value": [ 03:21:56 DEBUG 1713666116, 03:21:56 DEBUG 1713666116, 03:21:56 DEBUG }, 03:21:56 DEBUG 1713666116, 03:21:56 DEBUG "0.07933333333334265" 03:21:56 DEBUG } 03:21:56 DEBUG 1713666116, 03:21:56 DEBUG ] 03:21:56 DEBUG "1038594048" 03:21:56 DEBUG } 03:21:56 DEBUG 1713666116, 03:21:56 DEBUG } 03:21:56 DEBUG ] 03:21:56 DEBUG 1713666116, 03:21:56 DEBUG "0" 03:21:56 DEBUG "149.889" 03:21:56 DEBUG "100430644.03333333" 03:21:56 DEBUG "value": [ 03:21:56 DEBUG "3328.633333333333" 03:21:56 DEBUG ] 03:21:56 DEBUG ] 03:21:56 DEBUG "100432705.1" 03:21:56 DEBUG } 03:21:56 DEBUG ] 03:21:56 DEBUG ] 03:21:56 DEBUG "31.066666666666666" 03:21:56 DEBUG ] 03:21:56 DEBUG } 03:21:56 DEBUG "4406797171712" 03:21:56 DEBUG ] 03:21:56 DEBUG ] 03:21:56 DEBUG ] 03:21:56 DEBUG 1713666116, 03:21:56 DEBUG ] 03:21:56 DEBUG } 03:21:56 DEBUG } 03:21:56 DEBUG ] 03:21:56 DEBUG ] 03:21:56 DEBUG } 03:21:56 DEBUG } 03:21:56 DEBUG ] 03:21:56 DEBUG } 03:21:56 DEBUG ] 03:21:56 DEBUG ] 03:21:56 DEBUG } 03:21:56 DEBUG } 03:21:56 DEBUG } 03:21:56 DEBUG "0" 03:21:56 DEBUG } 03:21:56 DEBUG ] 03:21:56 DEBUG } 03:21:56 DEBUG } 03:21:56 DEBUG } 03:21:56 DEBUG ] 03:21:56 DEBUG } 03:21:56 DEBUG } 03:21:56 DEBUG } 03:21:56 DEBUG } 03:21:56 DEBUG } 03:21:56 DEBUG ] 03:21:56 DEBUG ] 03:21:56 DEBUG ] 03:21:56 DEBUG ] 03:21:56 DEBUG ] 03:21:56 DEBUG } 03:21:56 DEBUG ] 03:21:56 DEBUG } 03:21:56 DEBUG } 03:21:56 DEBUG ] 03:21:56 DEBUG } 03:21:56 DEBUG ] 03:21:56 DEBUG } 03:21:56 DEBUG } 03:21:56 DEBUG } 03:21:56 DEBUG } 03:21:56 DEBUG } 03:21:56 DEBUG } 03:21:56 DEBUG } 03:21:56 DEBUG } 03:21:56 DEBUG } 03:21:56 DEBUG } 03:21:56 DEBUG } 03:21:56 DEBUG } 03:21:56 DEBUG } 03:21:56 DEBUG ] 03:21:56 DEBUG } 03:21:56 DEBUG } 03:21:56 DEBUG } 03:21:56 DEBUG } 03:21:56 DEBUG } 03:21:56 DEBUG } 03:22:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713666173" 03:22:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666173" 03:22:53 INFO [http_cmd]: http status code OK 03:22:53 DEBUG --- status code --- 03:22:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:22:53 DEBUG --- http response --- 03:22:53 ERROR [http_cmd]: ERROR 03:22:53 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 03:22:53 ERROR --- status code --- 03:22:53 ERROR http status code is 400 (expected [200, 401, 403, 404]) 03:22:53 ERROR --- http response --- 03:22:53 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 03:22:53 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 03:22:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666174" 03:22:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666174" 03:22:54 INFO [http_cmd]: http status code OK 03:22:54 INFO [http_cmd]: http status code OK 03:22:54 DEBUG --- status code --- 03:22:54 DEBUG --- status code --- 03:22:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:22:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:22:54 DEBUG --- http response --- 03:22:54 DEBUG --- http response --- 03:22:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:22:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:22:54 DEBUG ----- output ----- 03:22:54 DEBUG { 03:22:54 DEBUG "status": "success", 03:22:54 DEBUG "data": { 03:22:54 DEBUG ----- output ----- 03:22:54 DEBUG "resultType": "vector", 03:22:54 DEBUG { 03:22:54 DEBUG "result": [ 03:22:54 DEBUG "status": "success", 03:22:54 DEBUG { 03:22:54 DEBUG "metric": { 03:22:54 DEBUG "pod": "ds-cts-0" 03:22:54 DEBUG "data": { 03:22:54 DEBUG }, 03:22:54 DEBUG "resultType": "vector", 03:22:54 DEBUG "value": [ 03:22:54 DEBUG "result": [ 03:22:54 DEBUG 03:22:54 DEBUG { 03:22:54 DEBUG 03:22:54 DEBUG "metric": { 03:22:54 DEBUG 03:22:54 DEBUG "instance": "10.142.0.107:9100" 03:22:54 DEBUG [...] 03:22:54 DEBUG 03:22:54 DEBUG }, 03:22:54 DEBUG 03:22:54 DEBUG "value": [ 03:22:54 DEBUG 03:22:54 DEBUG 03:22:54 DEBUG 03:22:54 DEBUG 03:22:54 DEBUG "pod": "ds-cts-2" 03:22:54 DEBUG [...] 03:22:54 DEBUG }, 03:22:54 DEBUG 03:22:54 DEBUG "value": [ 03:22:54 DEBUG 03:22:54 DEBUG 1713666174, 03:22:54 DEBUG 03:22:54 DEBUG "0.5604333333333348" 03:22:54 DEBUG "instance": "10.142.0.43:9100" 03:22:54 DEBUG ] 03:22:54 DEBUG }, 03:22:54 DEBUG } 03:22:54 DEBUG "value": [ 03:22:54 DEBUG ] 03:22:54 DEBUG 1713666174, 03:22:54 DEBUG } 03:22:54 DEBUG "0.4114000000000639" 03:22:54 DEBUG } 03:22:54 DEBUG ] 03:22:54 DEBUG } 03:22:54 DEBUG ] 03:22:54 DEBUG } 03:22:54 DEBUG } 03:22:54 INFO 03:22:54 INFO [loop_until]: kubectl --namespace=xlou top pods 03:22:54 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:22:55 INFO [loop_until]: OK (rc = 0) 03:22:55 DEBUG --- stdout --- 03:22:55 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 21814m 8796Mi am-767f6b67b5-qcqz2 22162m 9128Mi am-767f6b67b5-vbltz 23487m 8815Mi ds-cts-0 20326m 12490Mi ds-cts-1 19608m 12499Mi ds-cts-2 20383m 12563Mi ds-idrepo-0 9507m 3918Mi ds-idrepo-1 9398m 3946Mi ds-idrepo-2 9509m 3915Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 3m 1179Mi idm-b85f9d978-wnhxb 3m 1239Mi lodemon-54c87f8cbb-jzrjs 19m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-68c5d5b884-p97w5 11604m 901Mi 03:22:55 DEBUG --- stderr --- 03:22:55 DEBUG 03:22:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666175" 03:22:55 INFO [http_cmd]: http status code OK 03:22:55 DEBUG --- status code --- 03:22:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:22:55 DEBUG --- http response --- 03:22:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:22:55 DEBUG ----- output ----- 03:22:55 DEBUG { 03:22:55 DEBUG "status": "success", 03:22:55 DEBUG "data": { 03:22:55 DEBUG "resultType": "vector", 03:22:55 DEBUG "result": [ 03:22:55 DEBUG { 03:22:55 DEBUG "metric": { 03:22:55 DEBUG "pod": "am-767f6b67b5-9zw22" 03:22:55 DEBUG }, 03:22:55 DEBUG "value": [ 03:22:55 DEBUG 03:22:55 DEBUG 03:22:55 DEBUG 03:22:55 DEBUG [...] 03:22:55 DEBUG 03:22:55 DEBUG 03:22:55 DEBUG 03:22:55 DEBUG "pod": "am-767f6b67b5-vbltz" 03:22:55 DEBUG }, 03:22:55 DEBUG "value": [ 03:22:55 DEBUG 1713666175, 03:22:55 DEBUG "3331.233333333333" 03:22:55 DEBUG ] 03:22:55 DEBUG } 03:22:55 DEBUG ] 03:22:55 DEBUG } 03:22:55 DEBUG } 03:22:55 INFO 03:22:55 INFO [loop_until]: kubectl --namespace=xlou top node 03:22:55 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:22:55 INFO [loop_until]: OK (rc = 0) 03:22:55 DEBUG --- stdout --- 03:22:55 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 6286Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 9547m 31% 7756Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 21919m 73% 11609Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 79m 0% 3351Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 20083m 67% 15802Mi 14% gke-xlou-cdm-default-pool-1b0769c8-wpwl 156m 0% 5258Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 22421m 75% 12056Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 19658m 65% 3170Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 24598m 82% 11088Mi 9% gke-xlou-cdm-ds-0f601751-j488 10338m 34% 7359Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 19588m 65% 17642Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 9632m 32% 8015Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20648m 69% 15283Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 11551m 72% 11226Mi 19% 03:22:55 DEBUG --- stderr --- 03:22:55 DEBUG 03:22:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666176" 03:22:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666176" 03:22:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666176" 03:22:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713666176" 03:22:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666176" 03:22:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666176" 03:22:56 INFO [http_cmd]: http status code OK 03:22:56 DEBUG --- status code --- 03:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:22:56 DEBUG --- http response --- 03:22:56 INFO [http_cmd]: http status code OK 03:22:56 INFO [http_cmd]: http status code OK 03:22:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:22:56 DEBUG --- status code --- 03:22:56 DEBUG --- status code --- 03:22:56 DEBUG ----- output ----- 03:22:56 INFO [http_cmd]: http status code OK 03:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:22:56 DEBUG { 03:22:56 DEBUG --- status code --- 03:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:22:56 DEBUG --- http response --- 03:22:56 DEBUG --- http response --- 03:22:56 DEBUG "status": "success", 03:22:56 DEBUG --- http response --- 03:22:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:22:56 INFO [http_cmd]: http status code OK 03:22:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:22:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:22:56 DEBUG "data": { 03:22:56 DEBUG ----- output ----- 03:22:56 INFO [http_cmd]: http status code OK 03:22:56 DEBUG --- status code --- 03:22:56 DEBUG ----- output ----- 03:22:56 DEBUG ----- output ----- 03:22:56 DEBUG "resultType": "vector", 03:22:56 DEBUG { 03:22:56 DEBUG --- status code --- 03:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:22:56 DEBUG { 03:22:56 DEBUG { 03:22:56 DEBUG "result": [ 03:22:56 DEBUG "status": "success", 03:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:22:56 DEBUG --- http response --- 03:22:56 DEBUG "status": "success", 03:22:56 DEBUG "status": "success", 03:22:56 DEBUG { 03:22:56 DEBUG "data": { 03:22:56 DEBUG --- http response --- 03:22:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:22:56 DEBUG "data": { 03:22:56 DEBUG "data": { 03:22:56 DEBUG "metric": { 03:22:56 DEBUG "pod": "ds-idrepo-0" 03:22:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:22:56 DEBUG ----- output ----- 03:22:56 DEBUG "resultType": "vector", 03:22:56 DEBUG "resultType": "vector", 03:22:56 DEBUG "resultType": "vector", 03:22:56 DEBUG }, 03:22:56 DEBUG ----- output ----- 03:22:56 DEBUG { 03:22:56 DEBUG "result": [ 03:22:56 DEBUG "result": [ 03:22:56 DEBUG "result": [ 03:22:56 DEBUG "value": [ 03:22:56 DEBUG { 03:22:56 DEBUG "status": "success", 03:22:56 DEBUG { 03:22:56 DEBUG { 03:22:56 DEBUG { 03:22:56 DEBUG 03:22:56 DEBUG "status": "success", 03:22:56 DEBUG "data": { 03:22:56 DEBUG "metric": { 03:22:56 DEBUG "metric": { 03:22:56 DEBUG "metric": { 03:22:56 DEBUG 03:22:56 DEBUG "data": { 03:22:56 DEBUG "resultType": "vector", 03:22:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:22:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:22:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:22:56 DEBUG 03:22:56 DEBUG "resultType": "vector", 03:22:56 DEBUG "result": [ 03:22:56 DEBUG }, 03:22:56 DEBUG }, 03:22:56 DEBUG }, 03:22:56 DEBUG [...] 03:22:56 DEBUG "result": [ 03:22:56 DEBUG { 03:22:56 DEBUG "value": [ 03:22:56 DEBUG 03:22:56 DEBUG "value": [ 03:22:56 DEBUG 03:22:56 DEBUG { 03:22:56 DEBUG "metric": { 03:22:56 DEBUG 03:22:56 DEBUG 03:22:56 DEBUG 03:22:56 DEBUG "metric": { 03:22:56 DEBUG "pod": "am-767f6b67b5-9zw22" 03:22:56 DEBUG "value": [ 03:22:56 DEBUG 03:22:56 DEBUG [...] 03:22:56 DEBUG 03:22:56 DEBUG 03:22:56 DEBUG }, 03:22:56 DEBUG 03:22:56 DEBUG 03:22:56 DEBUG 03:22:56 DEBUG "pod": "am-767f6b67b5-9zw22" 03:22:56 DEBUG 03:22:56 DEBUG "value": [ 03:22:56 DEBUG 03:22:56 DEBUG 03:22:56 DEBUG "pod": "ds-idrepo-2" 03:22:56 DEBUG }, 03:22:56 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 03:22:56 DEBUG 03:22:56 DEBUG 03:22:56 DEBUG [...] 03:22:56 DEBUG }, 03:22:56 DEBUG "value": [ 03:22:56 DEBUG }, 03:22:56 DEBUG 03:22:56 DEBUG [...] 03:22:56 DEBUG 03:22:56 DEBUG "value": [ 03:22:56 DEBUG 03:22:56 DEBUG 03:22:56 DEBUG 03:22:56 DEBUG 03:22:56 DEBUG 03:22:56 DEBUG 1713666176, 03:22:56 DEBUG 03:22:56 DEBUG "value": [ 03:22:56 DEBUG [...] 03:22:56 DEBUG 03:22:56 DEBUG 03:22:56 DEBUG "0" 03:22:56 DEBUG [...] 03:22:56 DEBUG 1713666176, 03:22:56 DEBUG 03:22:56 DEBUG 03:22:56 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 03:22:56 DEBUG ] 03:22:56 DEBUG } 03:22:56 DEBUG "0" 03:22:56 DEBUG 03:22:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:22:56 DEBUG }, 03:22:56 DEBUG 03:22:56 DEBUG ] 03:22:56 DEBUG ] 03:22:56 DEBUG 03:22:56 DEBUG "value": [ 03:22:56 DEBUG }, 03:22:56 DEBUG 03:22:56 DEBUG } 03:22:56 DEBUG } 03:22:56 DEBUG "pod": "am-767f6b67b5-vbltz" 03:22:56 DEBUG 1713666176, 03:22:56 DEBUG "value": [ 03:22:56 DEBUG 03:22:56 DEBUG } 03:22:56 DEBUG }, 03:22:56 DEBUG "11.715457294512065" 03:22:56 DEBUG 1713666176, 03:22:56 DEBUG "pod": "am-767f6b67b5-vbltz" 03:22:56 DEBUG ] 03:22:56 DEBUG "value": [ 03:22:56 DEBUG ] 03:22:56 DEBUG "0" 03:22:56 DEBUG ] 03:22:56 DEBUG } 03:22:56 DEBUG 1713666176, 03:22:56 DEBUG } 03:22:56 DEBUG }, 03:22:56 DEBUG } 03:22:56 DEBUG } 03:22:56 DEBUG "3329" 03:22:56 DEBUG ] 03:22:56 DEBUG "value": [ 03:22:56 DEBUG ] 03:22:56 DEBUG ] 03:22:56 DEBUG } 03:22:56 DEBUG 1713666176, 03:22:56 DEBUG } 03:22:56 DEBUG } 03:22:56 DEBUG } 03:22:56 DEBUG "0" 03:22:56 DEBUG ] 03:22:56 DEBUG ] 03:22:56 DEBUG } 03:22:56 DEBUG } 03:22:56 DEBUG } 03:22:56 DEBUG ] 03:22:56 DEBUG } 03:22:56 DEBUG } 03:22:56 DEBUG } 03:22:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666176" 03:22:56 INFO [http_cmd]: http status code OK 03:22:56 DEBUG --- status code --- 03:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:22:56 DEBUG --- http response --- 03:22:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:22:56 DEBUG ----- output ----- 03:22:56 DEBUG { 03:22:56 DEBUG "status": "success", 03:22:56 DEBUG "data": { 03:22:56 DEBUG "resultType": "vector", 03:22:56 DEBUG "result": [ 03:22:56 DEBUG { 03:22:56 DEBUG "metric": { 03:22:56 DEBUG "pod": "am-767f6b67b5-9zw22" 03:22:56 DEBUG }, 03:22:56 DEBUG "value": [ 03:22:56 DEBUG 03:22:56 DEBUG 03:22:56 DEBUG 03:22:56 DEBUG [...] 03:22:56 DEBUG 03:22:56 DEBUG 03:22:56 DEBUG 03:22:56 DEBUG "pod": "am-767f6b67b5-vbltz" 03:22:56 DEBUG }, 03:22:56 DEBUG "value": [ 03:22:56 DEBUG 1713666176, 03:22:56 DEBUG "600017.5" 03:22:56 DEBUG ] 03:22:56 DEBUG } 03:22:56 DEBUG ] 03:22:56 DEBUG } 03:22:56 DEBUG } 03:22:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666176" 03:22:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666176" 03:22:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713666176" 03:22:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666176" 03:22:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666176" 03:22:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666176" 03:22:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713666176" 03:22:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713666176" 03:22:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666176" 03:22:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666176" 03:22:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666176" 03:22:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666176" 03:22:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666176" 03:22:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666176" 03:22:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666176" 03:22:56 INFO [http_cmd]: http status code OK 03:22:56 INFO [http_cmd]: http status code OK 03:22:56 INFO [http_cmd]: http status code OK 03:22:56 DEBUG --- status code --- 03:22:56 INFO [http_cmd]: http status code OK 03:22:56 INFO [http_cmd]: http status code OK 03:22:56 INFO [http_cmd]: http status code OK 03:22:56 INFO [http_cmd]: http status code OK 03:22:56 INFO [http_cmd]: http status code OK 03:22:56 DEBUG --- status code --- 03:22:56 INFO [http_cmd]: http status code OK 03:22:56 INFO [http_cmd]: http status code OK 03:22:56 INFO [http_cmd]: http status code OK 03:22:56 DEBUG --- status code --- 03:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:22:56 INFO [http_cmd]: http status code OK 03:22:56 DEBUG --- status code --- 03:22:56 DEBUG --- status code --- 03:22:56 DEBUG --- status code --- 03:22:56 INFO [http_cmd]: http status code OK 03:22:56 DEBUG --- status code --- 03:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:22:56 DEBUG --- status code --- 03:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:22:56 DEBUG --- status code --- 03:22:56 DEBUG --- status code --- 03:22:56 DEBUG --- status code --- 03:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:22:56 DEBUG --- http response --- 03:22:56 DEBUG --- status code --- 03:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:22:56 INFO [http_cmd]: http status code OK 03:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:22:56 DEBUG --- status code --- 03:22:56 INFO [http_cmd]: http status code OK 03:22:56 DEBUG --- http response --- 03:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:22:56 DEBUG --- http response --- 03:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:22:56 DEBUG --- http response --- 03:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:22:56 DEBUG --- http response --- 03:22:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:22:56 DEBUG ----- output ----- 03:22:56 DEBUG --- http response --- 03:22:56 DEBUG --- http response --- 03:22:56 DEBUG --- status code --- 03:22:56 DEBUG --- http response --- 03:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:22:56 DEBUG --- status code --- 03:22:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:22:56 DEBUG --- http response --- 03:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:22:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:22:56 DEBUG --- http response --- 03:22:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:22:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:22:56 DEBUG { 03:22:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:22:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:22:56 DEBUG --- http response --- 03:22:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:22:56 DEBUG ----- output ----- 03:22:56 DEBUG --- http response --- 03:22:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:22:56 DEBUG ----- output ----- 03:22:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 03:22:56 DEBUG ----- output ----- 03:22:56 DEBUG ----- output ----- 03:22:56 DEBUG --- http response --- 03:22:56 DEBUG "status": "success", 03:22:56 DEBUG ----- output ----- 03:22:56 DEBUG { 03:22:56 DEBUG --- http response --- 03:22:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:22:56 DEBUG ----- output ----- 03:22:56 DEBUG --- http response --- 03:22:56 DEBUG { 03:22:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:22:56 DEBUG ----- output ----- 03:22:56 DEBUG { 03:22:56 DEBUG "status": "success", 03:22:56 DEBUG { 03:22:56 DEBUG "status": "success", 03:22:56 DEBUG "data": { 03:22:56 DEBUG "data": { 03:22:56 DEBUG ----- output ----- 03:22:56 DEBUG "status": "success", 03:22:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 03:22:56 DEBUG ----- output ----- 03:22:56 DEBUG { 03:22:56 DEBUG "status": "success", 03:22:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:22:56 DEBUG ----- output ----- 03:22:56 DEBUG { 03:22:56 DEBUG ----- output ----- 03:22:56 DEBUG "resultType": "vector", 03:22:56 DEBUG ----- output ----- 03:22:56 DEBUG { 03:22:56 DEBUG "resultType": "vector", 03:22:56 DEBUG { 03:22:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:22:56 DEBUG "data": { 03:22:56 DEBUG "resultType": "vector", 03:22:56 DEBUG { 03:22:56 DEBUG "data": { 03:22:56 DEBUG "status": "success", 03:22:56 DEBUG { 03:22:56 DEBUG "data": { 03:22:56 DEBUG "status": "success", 03:22:56 DEBUG { 03:22:56 DEBUG "result": [ 03:22:56 DEBUG { 03:22:56 DEBUG "status": "success", 03:22:56 DEBUG "result": [ 03:22:56 DEBUG "status": "success", 03:22:56 DEBUG ----- output ----- 03:22:56 DEBUG ----- output ----- 03:22:56 DEBUG "result": [ 03:22:56 DEBUG "status": "success", 03:22:56 DEBUG "resultType": "vector", 03:22:56 DEBUG "result": [ 03:22:56 DEBUG "status": "success", 03:22:56 DEBUG "resultType": "vector", 03:22:56 DEBUG "data": { 03:22:56 DEBUG "status": "success", 03:22:56 DEBUG { 03:22:56 DEBUG "status": "success", 03:22:56 DEBUG "data": { 03:22:56 DEBUG { 03:22:56 DEBUG "data": { 03:22:56 DEBUG { 03:22:56 DEBUG { 03:22:56 DEBUG { 03:22:56 DEBUG "data": { 03:22:56 DEBUG { 03:22:56 DEBUG "data": { 03:22:56 DEBUG "result": [ 03:22:56 DEBUG "data": { 03:22:56 DEBUG "resultType": "vector", 03:22:56 DEBUG "data": { 03:22:56 DEBUG "metric": { 03:22:56 DEBUG "data": { 03:22:56 DEBUG "resultType": "vector", 03:22:56 DEBUG "metric": { 03:22:56 DEBUG "resultType": "vector", 03:22:56 DEBUG "status": "success", 03:22:56 DEBUG "status": "success", 03:22:56 DEBUG "metric": { 03:22:56 DEBUG "resultType": "vector", 03:22:56 DEBUG "metric": { 03:22:56 DEBUG "resultType": "vector", 03:22:56 DEBUG { 03:22:56 DEBUG "resultType": "vector", 03:22:56 DEBUG "result": [ 03:22:56 DEBUG "resultType": "vector", 03:22:56 DEBUG "instance": "10.142.0.107:9100" 03:22:56 DEBUG "resultType": "vector", 03:22:56 DEBUG "result": [ 03:22:56 DEBUG "pod": "am-767f6b67b5-9zw22" 03:22:56 DEBUG "result": [ 03:22:56 DEBUG "data": { 03:22:56 DEBUG "data": { 03:22:56 DEBUG "pod": "ds-idrepo-0" 03:22:56 DEBUG "result": [ 03:22:56 DEBUG "instance": "10.142.0.107:9100" 03:22:56 DEBUG "result": [ 03:22:56 DEBUG "metric": { 03:22:56 DEBUG "result": [ 03:22:56 DEBUG { 03:22:56 DEBUG "result": [ 03:22:56 DEBUG }, 03:22:56 DEBUG "result": [ 03:22:56 DEBUG { 03:22:56 DEBUG }, 03:22:56 DEBUG { 03:22:56 DEBUG "resultType": "vector", 03:22:56 DEBUG "resultType": "vector", 03:22:56 DEBUG }, 03:22:56 DEBUG { 03:22:56 DEBUG }, 03:22:56 DEBUG { 03:22:56 DEBUG "pod": "ds-cts-0" 03:22:56 DEBUG { 03:22:56 DEBUG "metric": { 03:22:56 DEBUG { 03:22:56 DEBUG "value": [ 03:22:56 DEBUG { 03:22:56 DEBUG "metric": { 03:22:56 DEBUG "value": [ 03:22:56 DEBUG "metric": { 03:22:56 DEBUG "result": [ 03:22:56 DEBUG "result": [ 03:22:56 DEBUG "value": [ 03:22:56 DEBUG "metric": { 03:22:56 DEBUG "value": [ 03:22:56 DEBUG "metric": { 03:22:56 DEBUG }, 03:22:56 DEBUG "metric": { 03:22:56 DEBUG "pod": "am-767f6b67b5-9zw22" 03:22:56 DEBUG "metric": { 03:22:56 DEBUG 03:22:56 DEBUG "metric": { 03:22:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:22:56 DEBUG 03:22:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:22:56 DEBUG { 03:22:56 DEBUG { 03:22:56 DEBUG 03:22:56 DEBUG "pod": "am-767f6b67b5-9zw22" 03:22:56 DEBUG 03:22:56 DEBUG 03:22:56 DEBUG "value": [ 03:22:56 DEBUG }, 03:22:56 DEBUG "instance": "10.142.0.107:9100" 03:22:56 DEBUG "pod": "ds-idrepo-0" 03:22:56 DEBUG 03:22:56 DEBUG "instance": "10.142.0.107:9100" 03:22:56 DEBUG }, 03:22:56 DEBUG 03:22:56 DEBUG }, 03:22:56 DEBUG "metric": { 03:22:56 DEBUG "metric": { 03:22:56 DEBUG 03:22:56 DEBUG }, 03:22:56 DEBUG 03:22:56 DEBUG "pod": "ds-cts-0" 03:22:56 DEBUG 03:22:56 DEBUG "value": [ 03:22:56 DEBUG }, 03:22:56 DEBUG }, 03:22:56 DEBUG 03:22:56 DEBUG }, 03:22:56 DEBUG "value": [ 03:22:56 DEBUG 03:22:56 DEBUG "value": [ 03:22:56 DEBUG "pod": "ds-idrepo-0" 03:22:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:22:56 DEBUG 03:22:56 DEBUG "value": [ 03:22:56 DEBUG [...] 03:22:56 DEBUG }, 03:22:56 DEBUG 03:22:56 DEBUG 03:22:56 DEBUG "value": [ 03:22:56 DEBUG "value": [ 03:22:56 DEBUG [...] 03:22:56 DEBUG "value": [ 03:22:56 DEBUG 03:22:56 DEBUG [...] 03:22:56 DEBUG 03:22:56 DEBUG 03:22:56 DEBUG }, 03:22:56 DEBUG [...] 03:22:56 DEBUG 03:22:56 DEBUG 03:22:56 DEBUG 03:22:56 DEBUG 03:22:56 DEBUG 03:22:56 DEBUG 03:22:56 DEBUG 03:22:56 DEBUG 03:22:56 DEBUG 03:22:56 DEBUG 03:22:56 DEBUG 03:22:56 DEBUG 03:22:56 DEBUG }, 03:22:56 DEBUG "value": [ 03:22:56 DEBUG 03:22:56 DEBUG 03:22:56 DEBUG 03:22:56 DEBUG "value": [ 03:22:56 DEBUG [...] 03:22:56 DEBUG 03:22:56 DEBUG 03:22:56 DEBUG 03:22:56 DEBUG 03:22:56 DEBUG 03:22:56 DEBUG 03:22:56 DEBUG 03:22:56 DEBUG [...] 03:22:56 DEBUG "value": [ 03:22:56 DEBUG 03:22:56 DEBUG 03:22:56 DEBUG 03:22:56 DEBUG "instance": "10.142.0.43:9100" 03:22:56 DEBUG 03:22:56 DEBUG 03:22:56 DEBUG [...] 03:22:56 DEBUG 03:22:56 DEBUG 03:22:56 DEBUG 03:22:56 DEBUG 03:22:56 DEBUG [...] 03:22:56 DEBUG 03:22:56 DEBUG 03:22:56 DEBUG 03:22:56 DEBUG 03:22:56 DEBUG 03:22:56 DEBUG "pod": "ds-cts-2" 03:22:56 DEBUG }, 03:22:56 DEBUG 03:22:56 DEBUG 03:22:56 DEBUG 03:22:56 DEBUG 03:22:56 DEBUG [...] 03:22:56 DEBUG "instance": "10.142.0.43:9100" 03:22:56 DEBUG 03:22:56 DEBUG [...] 03:22:56 DEBUG "pod": "am-767f6b67b5-vbltz" 03:22:56 DEBUG 03:22:56 DEBUG 03:22:56 DEBUG 03:22:56 DEBUG "value": [ 03:22:56 DEBUG [...] 03:22:56 DEBUG }, 03:22:56 DEBUG [...] 03:22:56 DEBUG 03:22:56 DEBUG 03:22:56 DEBUG 03:22:56 DEBUG 03:22:56 DEBUG }, 03:22:56 DEBUG 03:22:56 DEBUG 03:22:56 DEBUG }, 03:22:56 DEBUG 03:22:56 DEBUG [...] 03:22:56 DEBUG 03:22:56 DEBUG 1713666176, 03:22:56 DEBUG 03:22:56 DEBUG "value": [ 03:22:56 DEBUG 03:22:56 DEBUG 03:22:56 DEBUG "pod": "am-767f6b67b5-vbltz" 03:22:56 DEBUG 03:22:56 DEBUG "value": [ 03:22:56 DEBUG [...] 03:22:56 DEBUG 03:22:56 DEBUG 03:22:56 DEBUG 03:22:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:22:56 DEBUG 03:22:56 DEBUG [...] 03:22:56 DEBUG "0" 03:22:56 DEBUG 03:22:56 DEBUG 1713666176, 03:22:56 DEBUG 03:22:56 DEBUG "pod": "ds-cts-2" 03:22:56 DEBUG }, 03:22:56 DEBUG "pod": "ds-idrepo-2" 03:22:56 DEBUG 1713666176, 03:22:56 DEBUG 03:22:56 DEBUG "instance": "10.142.0.43:9100" 03:22:56 DEBUG }, 03:22:56 DEBUG "pod": "idm-b85f9d978-txpwv" 03:22:56 DEBUG "value": [ 03:22:56 DEBUG 03:22:56 DEBUG 03:22:56 DEBUG ] 03:22:56 DEBUG 03:22:56 DEBUG "0.08733333333333158" 03:22:56 DEBUG 03:22:56 DEBUG }, 03:22:56 DEBUG "value": [ 03:22:56 DEBUG }, 03:22:56 DEBUG "4412080163840" 03:22:56 DEBUG 03:22:56 DEBUG "value": [ 03:22:56 DEBUG }, 03:22:56 DEBUG }, 03:22:56 DEBUG "value": [ 03:22:56 DEBUG 03:22:56 DEBUG 03:22:56 DEBUG } 03:22:56 DEBUG "pod": "am-767f6b67b5-vbltz" 03:22:56 DEBUG ] 03:22:56 DEBUG "pod": "ds-cts-2" 03:22:56 DEBUG "value": [ 03:22:56 DEBUG 1713666176, 03:22:56 DEBUG "value": [ 03:22:56 DEBUG ] 03:22:56 DEBUG 03:22:56 DEBUG 1713666176, 03:22:56 DEBUG "value": [ 03:22:56 DEBUG 1713666176, 03:22:56 DEBUG 1713666176, 03:22:56 DEBUG "pod": "idm-b85f9d978-txpwv" 03:22:56 DEBUG 03:22:56 DEBUG ] 03:22:56 DEBUG }, 03:22:56 DEBUG } 03:22:56 DEBUG }, 03:22:56 DEBUG 1713666176, 03:22:56 DEBUG "149.889" 03:22:56 DEBUG 1713666176, 03:22:56 DEBUG } 03:22:56 DEBUG "instance": "10.142.0.43:9100" 03:22:56 DEBUG "99726352.93333334" 03:22:56 DEBUG 1713666176, 03:22:56 DEBUG "0" 03:22:56 DEBUG "3357.0666666666666" 03:22:56 DEBUG }, 03:22:56 DEBUG "pod": "ds-idrepo-2" 03:22:56 DEBUG } 03:22:56 DEBUG "value": [ 03:22:56 DEBUG ] 03:22:56 DEBUG "value": [ 03:22:56 DEBUG "0" 03:22:56 DEBUG ] 03:22:56 DEBUG "0" 03:22:56 DEBUG ] 03:22:56 DEBUG }, 03:22:56 DEBUG ] 03:22:56 DEBUG "1042251776" 03:22:56 DEBUG ] 03:22:56 DEBUG ] 03:22:56 DEBUG "value": [ 03:22:56 DEBUG }, 03:22:56 DEBUG } 03:22:56 DEBUG 1713666176, 03:22:56 DEBUG } 03:22:56 DEBUG 1713666176, 03:22:56 DEBUG ] 03:22:56 DEBUG } 03:22:56 DEBUG ] 03:22:56 DEBUG } 03:22:56 DEBUG "value": [ 03:22:56 DEBUG } 03:22:56 DEBUG ] 03:22:56 DEBUG } 03:22:56 DEBUG } 03:22:56 DEBUG 1713666176, 03:22:56 DEBUG "value": [ 03:22:56 DEBUG "3332.2666666666664" 03:22:56 DEBUG } 03:22:56 DEBUG "30.099999999999998" 03:22:56 DEBUG } 03:22:56 DEBUG ] 03:22:56 DEBUG } 03:22:56 DEBUG } 03:22:56 DEBUG 1713666176, 03:22:56 DEBUG ] 03:22:56 DEBUG } 03:22:56 DEBUG ] 03:22:56 DEBUG ] 03:22:56 DEBUG "320.7554455079641" 03:22:56 DEBUG 1713666176, 03:22:56 DEBUG ] 03:22:56 DEBUG ] 03:22:56 DEBUG ] 03:22:56 DEBUG } 03:22:56 DEBUG ] 03:22:56 DEBUG "99722401.86666667" 03:22:56 DEBUG } 03:22:56 DEBUG ] 03:22:56 DEBUG } 03:22:56 DEBUG } 03:22:56 DEBUG ] 03:22:56 DEBUG "0" 03:22:56 DEBUG } 03:22:56 DEBUG } 03:22:56 DEBUG } 03:22:56 DEBUG } 03:22:56 DEBUG } 03:22:56 DEBUG ] 03:22:56 DEBUG } 03:22:56 DEBUG } 03:22:56 DEBUG } 03:22:56 DEBUG } 03:22:56 DEBUG } 03:22:56 DEBUG ] 03:22:56 DEBUG ] 03:22:56 DEBUG ] 03:22:56 DEBUG } 03:22:56 DEBUG } 03:22:56 DEBUG } 03:22:56 DEBUG } 03:22:56 DEBUG ] 03:22:56 DEBUG } 03:22:56 DEBUG } 03:22:56 DEBUG } 03:22:56 DEBUG ] 03:22:56 DEBUG } 03:22:56 DEBUG ] 03:22:56 DEBUG } 03:22:56 DEBUG } 03:22:56 DEBUG } 03:22:56 DEBUG } 03:22:56 DEBUG } 03:22:56 DEBUG } 03:22:56 DEBUG } 03:23:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713666233" 03:23:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666233" 03:23:53 ERROR [http_cmd]: ERROR 03:23:53 ERROR --- status code --- 03:23:53 INFO [http_cmd]: http status code OK 03:23:53 ERROR http status code is 400 (expected [200, 401, 403, 404]) 03:23:53 DEBUG --- status code --- 03:23:53 ERROR --- http response --- 03:23:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:23:53 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 03:23:53 DEBUG --- http response --- 03:23:53 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 03:23:53 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 03:23:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666234" 03:23:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666234" 03:23:54 INFO [http_cmd]: http status code OK 03:23:54 DEBUG --- status code --- 03:23:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:23:54 DEBUG --- http response --- 03:23:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:23:54 DEBUG ----- output ----- 03:23:54 DEBUG { 03:23:54 DEBUG "status": "success", 03:23:54 DEBUG "data": { 03:23:54 DEBUG "resultType": "vector", 03:23:54 DEBUG "result": [ 03:23:54 DEBUG { 03:23:54 DEBUG "metric": { 03:23:54 DEBUG "pod": "ds-cts-0" 03:23:54 INFO [http_cmd]: http status code OK 03:23:54 DEBUG }, 03:23:54 DEBUG --- status code --- 03:23:54 DEBUG "value": [ 03:23:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:23:54 DEBUG 03:23:54 DEBUG --- http response --- 03:23:54 DEBUG 03:23:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:23:54 DEBUG 03:23:54 DEBUG ----- output ----- 03:23:54 DEBUG [...] 03:23:54 DEBUG { 03:23:54 DEBUG 03:23:54 DEBUG "status": "success", 03:23:54 DEBUG 03:23:54 DEBUG "data": { 03:23:54 DEBUG 03:23:54 DEBUG "resultType": "vector", 03:23:54 DEBUG "pod": "ds-cts-2" 03:23:54 DEBUG "result": [ 03:23:54 DEBUG }, 03:23:54 DEBUG { 03:23:54 DEBUG "value": [ 03:23:54 DEBUG "metric": { 03:23:54 DEBUG 1713666234, 03:23:54 DEBUG "instance": "10.142.0.107:9100" 03:23:54 DEBUG "0.5189000000000002" 03:23:54 DEBUG }, 03:23:54 DEBUG ] 03:23:54 DEBUG "value": [ 03:23:54 DEBUG } 03:23:54 DEBUG 03:23:54 DEBUG ] 03:23:54 DEBUG 03:23:54 DEBUG } 03:23:54 DEBUG 03:23:54 DEBUG } 03:23:54 DEBUG [...] 03:23:54 DEBUG 03:23:54 DEBUG 03:23:54 DEBUG 03:23:54 DEBUG "instance": "10.142.0.43:9100" 03:23:54 DEBUG }, 03:23:54 DEBUG "value": [ 03:23:54 DEBUG 1713666234, 03:23:54 DEBUG "0.34393333333334414" 03:23:54 DEBUG ] 03:23:54 DEBUG } 03:23:54 DEBUG ] 03:23:54 DEBUG } 03:23:54 DEBUG } 03:23:55 INFO 03:23:55 INFO [loop_until]: kubectl --namespace=xlou top pods 03:23:55 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:23:55 INFO [loop_until]: OK (rc = 0) 03:23:55 DEBUG --- stdout --- 03:23:55 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 22214m 8795Mi am-767f6b67b5-qcqz2 22009m 9129Mi am-767f6b67b5-vbltz 24323m 8821Mi ds-cts-0 19853m 12481Mi ds-cts-1 19529m 12501Mi ds-cts-2 20468m 12556Mi ds-idrepo-0 9485m 3918Mi ds-idrepo-1 9558m 3946Mi ds-idrepo-2 9858m 3916Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1179Mi idm-b85f9d978-wnhxb 3m 1239Mi lodemon-54c87f8cbb-jzrjs 15m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-68c5d5b884-p97w5 11729m 904Mi 03:23:55 DEBUG --- stderr --- 03:23:55 DEBUG 03:23:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666235" 03:23:55 INFO [http_cmd]: http status code OK 03:23:55 DEBUG --- status code --- 03:23:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:23:55 DEBUG --- http response --- 03:23:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:23:55 DEBUG ----- output ----- 03:23:55 DEBUG { 03:23:55 DEBUG "status": "success", 03:23:55 DEBUG "data": { 03:23:55 DEBUG "resultType": "vector", 03:23:55 DEBUG "result": [ 03:23:55 DEBUG { 03:23:55 DEBUG "metric": { 03:23:55 DEBUG "pod": "am-767f6b67b5-9zw22" 03:23:55 DEBUG }, 03:23:55 DEBUG "value": [ 03:23:55 DEBUG 03:23:55 DEBUG 03:23:55 DEBUG 03:23:55 DEBUG [...] 03:23:55 DEBUG 03:23:55 DEBUG 03:23:55 DEBUG 03:23:55 DEBUG "pod": "am-767f6b67b5-vbltz" 03:23:55 DEBUG }, 03:23:55 DEBUG "value": [ 03:23:55 DEBUG 1713666235, 03:23:55 DEBUG "3330.366666666667" 03:23:55 DEBUG ] 03:23:55 DEBUG } 03:23:55 DEBUG ] 03:23:55 DEBUG } 03:23:55 DEBUG } 03:23:55 INFO 03:23:55 INFO [loop_until]: kubectl --namespace=xlou top node 03:23:55 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:23:55 INFO [loop_until]: OK (rc = 0) 03:23:55 DEBUG --- stdout --- 03:23:55 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 6262Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 9949m 33% 7762Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 21981m 73% 11607Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 78m 0% 3356Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 19966m 66% 15796Mi 14% gke-xlou-cdm-default-pool-1b0769c8-wpwl 148m 0% 5278Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 22313m 74% 12069Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 19128m 64% 3175Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 24644m 82% 11092Mi 9% gke-xlou-cdm-ds-0f601751-j488 10427m 34% 7363Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 19979m 66% 17630Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 10145m 33% 8003Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 19680m 65% 15284Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 12018m 75% 11229Mi 19% 03:23:55 DEBUG --- stderr --- 03:23:55 DEBUG 03:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666236" 03:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666236" 03:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713666236" 03:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666236" 03:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666236" 03:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666236" 03:23:56 INFO [http_cmd]: http status code OK 03:23:56 DEBUG --- status code --- 03:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:23:56 INFO [http_cmd]: http status code OK 03:23:56 INFO [http_cmd]: http status code OK 03:23:56 DEBUG --- http response --- 03:23:56 DEBUG --- status code --- 03:23:56 DEBUG --- status code --- 03:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:23:56 INFO [http_cmd]: http status code OK 03:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:23:56 DEBUG ----- output ----- 03:23:56 DEBUG --- http response --- 03:23:56 DEBUG --- status code --- 03:23:56 DEBUG --- http response --- 03:23:56 DEBUG { 03:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:23:56 DEBUG "status": "success", 03:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 03:23:56 DEBUG ----- output ----- 03:23:56 DEBUG --- http response --- 03:23:56 DEBUG "data": { 03:23:56 DEBUG ----- output ----- 03:23:56 DEBUG { 03:23:56 DEBUG "resultType": "vector", 03:23:56 INFO [http_cmd]: http status code OK 03:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 03:23:56 INFO [http_cmd]: http status code OK 03:23:56 DEBUG { 03:23:56 DEBUG "status": "success", 03:23:56 DEBUG "data": { 03:23:56 DEBUG --- status code --- 03:23:56 DEBUG ----- output ----- 03:23:56 DEBUG --- status code --- 03:23:56 DEBUG "status": "success", 03:23:56 DEBUG "result": [ 03:23:56 DEBUG "resultType": "vector", 03:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:23:56 DEBUG { 03:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:23:56 DEBUG "data": { 03:23:56 DEBUG { 03:23:56 DEBUG "result": [ 03:23:56 DEBUG --- http response --- 03:23:56 DEBUG "status": "success", 03:23:56 DEBUG --- http response --- 03:23:56 DEBUG "resultType": "vector", 03:23:56 DEBUG "metric": { 03:23:56 DEBUG { 03:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:23:56 DEBUG "data": { 03:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:23:56 DEBUG "result": [ 03:23:56 DEBUG "pod": "ds-idrepo-0" 03:23:56 DEBUG "metric": { 03:23:56 DEBUG ----- output ----- 03:23:56 DEBUG "resultType": "vector", 03:23:56 DEBUG ----- output ----- 03:23:56 DEBUG { 03:23:56 DEBUG "metric": { 03:23:56 DEBUG "pod": "ds-idrepo-1" 03:23:56 DEBUG { 03:23:56 DEBUG "result": [ 03:23:56 DEBUG { 03:23:56 DEBUG }, 03:23:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:23:56 DEBUG }, 03:23:56 DEBUG "status": "success", 03:23:56 DEBUG { 03:23:56 DEBUG "status": "success", 03:23:56 DEBUG "value": [ 03:23:56 DEBUG }, 03:23:56 DEBUG "value": [ 03:23:56 DEBUG "data": { 03:23:56 DEBUG "metric": { 03:23:56 DEBUG "data": { 03:23:56 DEBUG 03:23:56 DEBUG "value": [ 03:23:56 DEBUG 03:23:56 DEBUG "resultType": "vector", 03:23:56 DEBUG "pod": "ds-idrepo-1" 03:23:56 DEBUG "resultType": "vector", 03:23:56 DEBUG "result": [ 03:23:56 DEBUG { 03:23:56 DEBUG 03:23:56 DEBUG "result": [ 03:23:56 DEBUG }, 03:23:56 DEBUG 03:23:56 DEBUG "metric": { 03:23:56 DEBUG 03:23:56 DEBUG 03:23:56 DEBUG { 03:23:56 DEBUG "value": [ 03:23:56 DEBUG 03:23:56 DEBUG "pod": "am-767f6b67b5-9zw22" 03:23:56 DEBUG 03:23:56 DEBUG [...] 03:23:56 DEBUG "metric": { 03:23:56 DEBUG 03:23:56 DEBUG [...] 03:23:56 DEBUG }, 03:23:56 DEBUG 03:23:56 DEBUG 03:23:56 DEBUG "pod": "am-767f6b67b5-9zw22" 03:23:56 DEBUG 03:23:56 DEBUG 03:23:56 DEBUG "value": [ 03:23:56 DEBUG [...] 03:23:56 DEBUG 03:23:56 DEBUG }, 03:23:56 DEBUG 03:23:56 DEBUG 03:23:56 DEBUG 03:23:56 DEBUG 03:23:56 DEBUG 03:23:56 DEBUG "value": [ 03:23:56 DEBUG [...] 03:23:56 DEBUG 03:23:56 DEBUG "pod": "ds-idrepo-2" 03:23:56 DEBUG }, 03:23:56 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:23:56 DEBUG 03:23:56 DEBUG 03:23:56 DEBUG 03:23:56 DEBUG "value": [ 03:23:56 DEBUG }, 03:23:56 DEBUG 03:23:56 DEBUG 03:23:56 DEBUG 03:23:56 DEBUG 03:23:56 DEBUG 1713666236, 03:23:56 DEBUG "value": [ 03:23:56 DEBUG 03:23:56 DEBUG [...] 03:23:56 DEBUG 03:23:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:23:56 DEBUG "0" 03:23:56 DEBUG 1713666236, 03:23:56 DEBUG "0" 03:23:56 DEBUG 03:23:56 DEBUG 03:23:56 DEBUG 03:23:56 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:23:56 DEBUG ] 03:23:56 DEBUG [...] 03:23:56 DEBUG }, 03:23:56 DEBUG 03:23:56 DEBUG }, 03:23:56 DEBUG ] 03:23:56 DEBUG } 03:23:56 DEBUG 03:23:56 DEBUG "value": [ 03:23:56 DEBUG "pod": "am-767f6b67b5-vbltz" 03:23:56 DEBUG "value": [ 03:23:56 DEBUG 1713666236, 03:23:56 DEBUG ] 03:23:56 DEBUG 03:23:56 DEBUG 1713666236, 03:23:56 DEBUG }, 03:23:56 DEBUG "0" 03:23:56 DEBUG } 03:23:56 DEBUG } 03:23:56 DEBUG 03:23:56 DEBUG "11.727406643012838" 03:23:56 DEBUG "value": [ 03:23:56 DEBUG ] 03:23:56 DEBUG ] 03:23:56 DEBUG } 03:23:56 DEBUG "pod": "am-767f6b67b5-vbltz" 03:23:56 DEBUG ] 03:23:56 DEBUG 1713666236, 03:23:56 DEBUG } 03:23:56 DEBUG } 03:23:56 DEBUG }, 03:23:56 DEBUG } 03:23:56 DEBUG "0" 03:23:56 DEBUG ] 03:23:56 DEBUG } 03:23:56 DEBUG "value": [ 03:23:56 DEBUG ] 03:23:56 DEBUG ] 03:23:56 DEBUG } 03:23:56 DEBUG 1713666236, 03:23:56 DEBUG "3330.3" 03:23:56 DEBUG } 03:23:56 DEBUG } 03:23:56 DEBUG } 03:23:56 DEBUG ] 03:23:56 DEBUG ] 03:23:56 DEBUG } 03:23:56 DEBUG } 03:23:56 DEBUG } 03:23:56 DEBUG ] 03:23:56 DEBUG } 03:23:56 DEBUG } 03:23:56 DEBUG } 03:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666236" 03:23:56 INFO [http_cmd]: http status code OK 03:23:56 DEBUG --- status code --- 03:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:23:56 DEBUG --- http response --- 03:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:23:56 DEBUG ----- output ----- 03:23:56 DEBUG { 03:23:56 DEBUG "status": "success", 03:23:56 DEBUG "data": { 03:23:56 DEBUG "resultType": "vector", 03:23:56 DEBUG "result": [ 03:23:56 DEBUG { 03:23:56 DEBUG "metric": { 03:23:56 DEBUG "pod": "am-767f6b67b5-9zw22" 03:23:56 DEBUG }, 03:23:56 DEBUG "value": [ 03:23:56 DEBUG 03:23:56 DEBUG 03:23:56 DEBUG 03:23:56 DEBUG [...] 03:23:56 DEBUG 03:23:56 DEBUG 03:23:56 DEBUG 03:23:56 DEBUG "pod": "am-767f6b67b5-vbltz" 03:23:56 DEBUG }, 03:23:56 DEBUG "value": [ 03:23:56 DEBUG 1713666236, 03:23:56 DEBUG "599552.5" 03:23:56 DEBUG ] 03:23:56 DEBUG } 03:23:56 DEBUG ] 03:23:56 DEBUG } 03:23:56 DEBUG } 03:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713666236" 03:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666236" 03:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666236" 03:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666236" 03:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713666236" 03:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666236" 03:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666236" 03:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666236" 03:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666236" 03:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713666236" 03:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666236" 03:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666236" 03:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666236" 03:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666236" 03:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666236" 03:23:56 INFO [http_cmd]: http status code OK 03:23:56 INFO [http_cmd]: http status code OK 03:23:56 INFO [http_cmd]: http status code OK 03:23:56 INFO [http_cmd]: http status code OK 03:23:56 INFO [http_cmd]: http status code OK 03:23:56 INFO [http_cmd]: http status code OK 03:23:56 DEBUG --- status code --- 03:23:56 DEBUG --- status code --- 03:23:56 INFO [http_cmd]: http status code OK 03:23:56 INFO [http_cmd]: http status code OK 03:23:56 INFO [http_cmd]: http status code OK 03:23:56 DEBUG --- status code --- 03:23:56 DEBUG --- status code --- 03:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:23:56 DEBUG --- status code --- 03:23:56 INFO [http_cmd]: http status code OK 03:23:56 DEBUG --- status code --- 03:23:56 INFO [http_cmd]: http status code OK 03:23:56 INFO [http_cmd]: http status code OK 03:23:56 DEBUG --- status code --- 03:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:23:56 DEBUG --- status code --- 03:23:56 DEBUG --- status code --- 03:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:23:56 DEBUG --- status code --- 03:23:56 INFO [http_cmd]: http status code OK 03:23:56 DEBUG --- status code --- 03:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:23:56 DEBUG --- status code --- 03:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:23:56 DEBUG --- status code --- 03:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:23:56 DEBUG --- http response --- 03:23:56 INFO [http_cmd]: http status code OK 03:23:56 DEBUG --- http response --- 03:23:56 INFO [http_cmd]: http status code OK 03:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:23:56 DEBUG --- http response --- 03:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:23:56 DEBUG --- http response --- 03:23:56 DEBUG --- http response --- 03:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:23:56 DEBUG --- http response --- 03:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:23:56 DEBUG --- http response --- 03:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:23:56 DEBUG --- status code --- 03:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:23:56 DEBUG --- status code --- 03:23:56 DEBUG --- http response --- 03:23:56 DEBUG --- http response --- 03:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:23:56 DEBUG --- http response --- 03:23:56 DEBUG --- http response --- 03:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:23:56 DEBUG --- http response --- 03:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:23:56 DEBUG --- http response --- 03:23:56 DEBUG ----- output ----- 03:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:23:56 DEBUG ----- output ----- 03:23:56 DEBUG ----- output ----- 03:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:23:56 DEBUG ----- output ----- 03:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:23:56 DEBUG ----- output ----- 03:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:23:56 DEBUG ----- output ----- 03:23:56 DEBUG ----- output ----- 03:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:23:56 DEBUG { 03:23:56 DEBUG { 03:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:23:56 DEBUG { 03:23:56 DEBUG --- http response --- 03:23:56 DEBUG ----- output ----- 03:23:56 DEBUG ----- output ----- 03:23:56 DEBUG { 03:23:56 DEBUG ----- output ----- 03:23:56 DEBUG { 03:23:56 DEBUG ----- output ----- 03:23:56 DEBUG ----- output ----- 03:23:56 DEBUG { 03:23:56 DEBUG "status": "success", 03:23:56 DEBUG ----- output ----- 03:23:56 DEBUG "status": "success", 03:23:56 DEBUG "data": { 03:23:56 DEBUG "resultType": "vector", 03:23:56 DEBUG "result": [ 03:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:23:56 DEBUG "status": "success", 03:23:56 DEBUG "status": "success", 03:23:56 DEBUG { 03:23:56 DEBUG { 03:23:56 DEBUG "status": "success", 03:23:56 DEBUG { 03:23:56 DEBUG { 03:23:56 DEBUG "status": "success", 03:23:56 DEBUG "data": { 03:23:56 DEBUG { 03:23:56 DEBUG { 03:23:56 DEBUG "status": "success", 03:23:56 DEBUG --- http response --- 03:23:56 DEBUG { 03:23:56 DEBUG ----- output ----- 03:23:56 DEBUG "data": { 03:23:56 DEBUG "data": { 03:23:56 DEBUG "status": "success", 03:23:56 DEBUG "status": "success", 03:23:56 DEBUG "data": { 03:23:56 DEBUG "status": "success", 03:23:56 DEBUG { 03:23:56 DEBUG "resultType": "vector", 03:23:56 DEBUG "data": { 03:23:56 DEBUG "resultType": "vector", 03:23:56 DEBUG "metric": { 03:23:56 DEBUG "pod": "am-767f6b67b5-9zw22" 03:23:56 DEBUG "status": "success", 03:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:23:56 DEBUG { 03:23:56 DEBUG "resultType": "vector", 03:23:56 DEBUG "resultType": "vector", 03:23:56 DEBUG "data": { 03:23:56 DEBUG "data": { 03:23:56 DEBUG "resultType": "vector", 03:23:56 DEBUG "data": { 03:23:56 DEBUG "status": "success", 03:23:56 DEBUG "data": { 03:23:56 DEBUG "status": "success", 03:23:56 DEBUG "result": [ 03:23:56 DEBUG "data": { 03:23:56 DEBUG }, 03:23:56 DEBUG "data": { 03:23:56 DEBUG ----- output ----- 03:23:56 DEBUG "status": "success", 03:23:56 DEBUG "result": [ 03:23:56 DEBUG "result": [ 03:23:56 DEBUG "resultType": "vector", 03:23:56 DEBUG "resultType": "vector", 03:23:56 DEBUG "result": [ 03:23:56 DEBUG { 03:23:56 DEBUG "resultType": "vector", 03:23:56 DEBUG "data": { 03:23:56 DEBUG "resultType": "vector", 03:23:56 DEBUG "result": [ 03:23:56 DEBUG "resultType": "vector", 03:23:56 DEBUG "value": [ 03:23:56 DEBUG "resultType": "vector", 03:23:56 DEBUG { 03:23:56 DEBUG "data": { 03:23:56 DEBUG { 03:23:56 DEBUG { 03:23:56 DEBUG "result": [ 03:23:56 DEBUG "resultType": "vector", 03:23:56 DEBUG "result": [ 03:23:56 DEBUG "metric": { 03:23:56 DEBUG "result": [ 03:23:56 DEBUG "result": [ 03:23:56 DEBUG { 03:23:56 DEBUG { 03:23:56 DEBUG "result": [ 03:23:56 DEBUG 03:23:56 DEBUG "result": [ 03:23:56 DEBUG "status": "success", 03:23:56 DEBUG "resultType": "vector", 03:23:56 DEBUG "metric": { 03:23:56 DEBUG "metric": { 03:23:56 DEBUG { 03:23:56 DEBUG "result": [ 03:23:56 DEBUG { 03:23:56 DEBUG "instance": "10.142.0.107:9100" 03:23:56 DEBUG { 03:23:56 DEBUG { 03:23:56 DEBUG "metric": { 03:23:56 DEBUG "instance": "10.142.0.107:9100" 03:23:56 DEBUG }, 03:23:56 DEBUG "value": [ 03:23:56 DEBUG { 03:23:56 DEBUG "data": { 03:23:56 DEBUG "result": [ 03:23:56 DEBUG "pod": "ds-idrepo-0" 03:23:56 DEBUG "pod": "ds-cts-0" 03:23:56 DEBUG "metric": { 03:23:56 DEBUG { 03:23:56 DEBUG "metric": { 03:23:56 DEBUG }, 03:23:56 DEBUG "metric": { 03:23:56 DEBUG "metric": { 03:23:56 DEBUG { 03:23:56 DEBUG 03:23:56 DEBUG 03:23:56 DEBUG "metric": { 03:23:56 DEBUG "metric": { 03:23:56 DEBUG "resultType": "vector", 03:23:56 DEBUG { 03:23:56 DEBUG }, 03:23:56 DEBUG }, 03:23:56 DEBUG "pod": "ds-cts-0" 03:23:56 DEBUG "metric": { 03:23:56 DEBUG "pod": "ds-idrepo-0" 03:23:56 DEBUG "value": [ 03:23:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:23:56 DEBUG "instance": "10.142.0.107:9100" 03:23:56 DEBUG "metric": { 03:23:56 DEBUG 03:23:56 DEBUG 03:23:56 DEBUG "pod": "ds-idrepo-1" 03:23:56 DEBUG "pod": "am-767f6b67b5-9zw22" 03:23:56 DEBUG "result": [ 03:23:56 DEBUG "metric": { 03:23:56 DEBUG "value": [ 03:23:56 DEBUG "value": [ 03:23:56 DEBUG }, 03:23:56 DEBUG "instance": "10.142.0.107:9100" 03:23:56 DEBUG }, 03:23:56 DEBUG 03:23:56 DEBUG }, 03:23:56 DEBUG }, 03:23:56 DEBUG "pod": "ds-idrepo-1" 03:23:56 DEBUG [...] 03:23:56 DEBUG 03:23:56 DEBUG }, 03:23:56 DEBUG }, 03:23:56 DEBUG { 03:23:56 DEBUG "pod": "am-767f6b67b5-9zw22" 03:23:56 DEBUG 03:23:56 DEBUG 03:23:56 DEBUG "value": [ 03:23:56 DEBUG }, 03:23:56 DEBUG "value": [ 03:23:56 DEBUG 03:23:56 DEBUG "value": [ 03:23:56 DEBUG "value": [ 03:23:56 DEBUG }, 03:23:56 DEBUG 03:23:56 DEBUG [...] 03:23:56 DEBUG "value": [ 03:23:56 DEBUG "value": [ 03:23:56 DEBUG "metric": { 03:23:56 DEBUG }, 03:23:56 DEBUG 03:23:56 DEBUG 03:23:56 DEBUG 03:23:56 DEBUG "value": [ 03:23:56 DEBUG 03:23:56 DEBUG 03:23:56 DEBUG 03:23:56 DEBUG 03:23:56 DEBUG "value": [ 03:23:56 DEBUG 03:23:56 DEBUG 03:23:56 DEBUG 03:23:56 DEBUG 03:23:56 DEBUG "pod": "ds-idrepo-0" 03:23:56 DEBUG "value": [ 03:23:56 DEBUG 03:23:56 DEBUG [...] 03:23:56 DEBUG 03:23:56 DEBUG 03:23:56 DEBUG 03:23:56 DEBUG [...] 03:23:56 DEBUG 03:23:56 DEBUG 03:23:56 DEBUG 03:23:56 DEBUG 03:23:56 DEBUG 03:23:56 DEBUG 03:23:56 DEBUG 03:23:56 DEBUG }, 03:23:56 DEBUG 03:23:56 DEBUG [...] 03:23:56 DEBUG 03:23:56 DEBUG 03:23:56 DEBUG 03:23:56 DEBUG 03:23:56 DEBUG 03:23:56 DEBUG 03:23:56 DEBUG 03:23:56 DEBUG 03:23:56 DEBUG "pod": "am-767f6b67b5-vbltz" 03:23:56 DEBUG 03:23:56 DEBUG 03:23:56 DEBUG 03:23:56 DEBUG "value": [ 03:23:56 DEBUG 03:23:56 DEBUG 03:23:56 DEBUG 03:23:56 DEBUG 03:23:56 DEBUG 03:23:56 DEBUG [...] 03:23:56 DEBUG 03:23:56 DEBUG [...] 03:23:56 DEBUG [...] 03:23:56 DEBUG 03:23:56 DEBUG }, 03:23:56 DEBUG "instance": "10.142.0.43:9100" 03:23:56 DEBUG [...] 03:23:56 DEBUG [...] 03:23:56 DEBUG 03:23:56 DEBUG 03:23:56 DEBUG 03:23:56 DEBUG 03:23:56 DEBUG [...] 03:23:56 DEBUG [...] 03:23:56 DEBUG 03:23:56 DEBUG 03:23:56 DEBUG 03:23:56 DEBUG 03:23:56 DEBUG [...] 03:23:56 DEBUG "value": [ 03:23:56 DEBUG }, 03:23:56 DEBUG 03:23:56 DEBUG 03:23:56 DEBUG 03:23:56 DEBUG [...] 03:23:56 DEBUG 03:23:56 DEBUG "pod": "ds-cts-2" 03:23:56 DEBUG 03:23:56 DEBUG 03:23:56 DEBUG 03:23:56 DEBUG "instance": "10.142.0.43:9100" 03:23:56 DEBUG 03:23:56 DEBUG 03:23:56 DEBUG 03:23:56 DEBUG 1713666236, 03:23:56 DEBUG "value": [ 03:23:56 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 03:23:56 DEBUG 03:23:56 DEBUG 03:23:56 DEBUG 03:23:56 DEBUG "pod": "ds-cts-2" 03:23:56 DEBUG }, 03:23:56 DEBUG 03:23:56 DEBUG 03:23:56 DEBUG 03:23:56 DEBUG }, 03:23:56 DEBUG 03:23:56 DEBUG 03:23:56 DEBUG 03:23:56 DEBUG "149.889" 03:23:56 DEBUG 1713666236, 03:23:56 DEBUG }, 03:23:56 DEBUG 03:23:56 DEBUG 03:23:56 DEBUG 03:23:56 DEBUG }, 03:23:56 DEBUG "value": [ 03:23:56 DEBUG 03:23:56 DEBUG 03:23:56 DEBUG "pod": "ds-cts-2" 03:23:56 DEBUG "value": [ 03:23:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:23:56 DEBUG "instance": "10.142.0.43:9100" 03:23:56 DEBUG }, 03:23:56 DEBUG ] 03:23:56 DEBUG "100357026.76666667" 03:23:56 DEBUG "value": [ 03:23:56 DEBUG 03:23:56 DEBUG [...] 03:23:56 DEBUG 03:23:56 DEBUG "value": [ 03:23:56 DEBUG 1713666236, 03:23:56 DEBUG "instance": "10.142.0.43:9100" 03:23:56 DEBUG "pod": "ds-idrepo-2" 03:23:56 DEBUG }, 03:23:56 DEBUG 1713666236, 03:23:56 DEBUG }, 03:23:56 DEBUG 03:23:56 DEBUG "value": [ 03:23:56 DEBUG } 03:23:56 DEBUG ] 03:23:56 DEBUG 1713666236, 03:23:56 DEBUG "pod": "am-767f6b67b5-vbltz" 03:23:56 DEBUG 03:23:56 DEBUG "pod": "am-767f6b67b5-vbltz" 03:23:56 DEBUG 1713666236, 03:23:56 DEBUG "29.966666666666665" 03:23:56 DEBUG }, 03:23:56 DEBUG }, 03:23:56 DEBUG "value": [ 03:23:56 DEBUG "4417003637760" 03:23:56 DEBUG "value": [ 03:23:56 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 03:23:56 DEBUG 1713666236, 03:23:56 DEBUG "100353835.6" 03:23:56 DEBUG } 03:23:56 DEBUG "0" 03:23:56 DEBUG }, 03:23:56 DEBUG 03:23:56 DEBUG }, 03:23:56 DEBUG "0" 03:23:56 DEBUG ] 03:23:56 DEBUG "value": [ 03:23:56 DEBUG "value": [ 03:23:56 DEBUG 1713666236, 03:23:56 DEBUG ] 03:23:56 DEBUG 1713666236, 03:23:56 DEBUG "1046470656" 03:23:56 DEBUG }, 03:23:56 DEBUG ] 03:23:56 DEBUG ] 03:23:56 DEBUG ] 03:23:56 DEBUG "value": [ 03:23:56 DEBUG 03:23:56 DEBUG "value": [ 03:23:56 DEBUG 1713666236, 03:23:56 DEBUG } 03:23:56 DEBUG 1713666236, 03:23:56 DEBUG 1713666236, 03:23:56 DEBUG "0" 03:23:56 DEBUG } 03:23:56 DEBUG ] 03:23:56 DEBUG ] 03:23:56 DEBUG "value": [ 03:23:56 DEBUG } 03:23:56 DEBUG } 03:23:56 DEBUG } 03:23:56 DEBUG 1713666236, 03:23:56 DEBUG "pod": "ds-idrepo-2" 03:23:56 DEBUG "3330.1666666666665" 03:23:56 DEBUG ] 03:23:56 DEBUG ] 03:23:56 DEBUG "0.0496666666666594" 03:23:56 DEBUG "0" 03:23:56 DEBUG ] 03:23:56 DEBUG ] 03:23:56 DEBUG } 03:23:56 DEBUG } 03:23:56 DEBUG 1713666236, 03:23:56 DEBUG ] 03:23:56 DEBUG } 03:23:56 DEBUG ] 03:23:56 DEBUG "3326.633333333333" 03:23:56 DEBUG }, 03:23:56 DEBUG ] 03:23:56 DEBUG } 03:23:56 DEBUG } 03:23:56 DEBUG ] 03:23:56 DEBUG ] 03:23:56 DEBUG } 03:23:56 DEBUG } 03:23:56 DEBUG } 03:23:56 DEBUG ] 03:23:56 DEBUG } 03:23:56 DEBUG } 03:23:56 DEBUG } 03:23:56 DEBUG ] 03:23:56 DEBUG } 03:23:56 DEBUG } 03:23:56 DEBUG ] 03:23:56 DEBUG } 03:23:56 DEBUG } 03:23:56 DEBUG } 03:23:56 DEBUG ] 03:23:56 DEBUG } 03:23:56 DEBUG } 03:23:56 DEBUG "0" 03:23:56 DEBUG } 03:23:56 DEBUG } 03:23:56 DEBUG "value": [ 03:23:56 DEBUG ] 03:23:56 DEBUG ] 03:23:56 DEBUG } 03:23:56 DEBUG ] 03:23:56 DEBUG ] 03:23:56 DEBUG } 03:23:56 DEBUG ] 03:23:56 DEBUG 1713666236, 03:23:56 DEBUG } 03:23:56 DEBUG } 03:23:56 DEBUG } 03:23:56 DEBUG } 03:23:56 DEBUG } 03:23:56 DEBUG } 03:23:56 DEBUG } 03:23:56 DEBUG "0" 03:23:56 DEBUG } 03:23:56 DEBUG } 03:23:56 DEBUG } 03:23:56 DEBUG ] 03:23:56 DEBUG ] 03:23:56 DEBUG } 03:23:56 DEBUG } 03:23:56 DEBUG } 03:23:56 DEBUG } 03:23:56 DEBUG ] 03:23:56 DEBUG } 03:23:56 DEBUG } 03:24:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713666293" 03:24:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666293" 03:24:53 ERROR [http_cmd]: ERROR 03:24:53 INFO [http_cmd]: http status code OK 03:24:53 ERROR --- status code --- 03:24:53 DEBUG --- status code --- 03:24:53 ERROR http status code is 400 (expected [200, 401, 403, 404]) 03:24:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:24:53 ERROR --- http response --- 03:24:53 DEBUG --- http response --- 03:24:53 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 03:24:53 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 03:24:53 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 03:24:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666295" 03:24:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666295" 03:24:55 INFO [http_cmd]: http status code OK 03:24:55 DEBUG --- status code --- 03:24:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:24:55 DEBUG --- http response --- 03:24:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:24:55 DEBUG ----- output ----- 03:24:55 DEBUG { 03:24:55 DEBUG "status": "success", 03:24:55 DEBUG "data": { 03:24:55 DEBUG "resultType": "vector", 03:24:55 DEBUG "result": [ 03:24:55 DEBUG { 03:24:55 DEBUG "metric": { 03:24:55 DEBUG "pod": "ds-cts-0" 03:24:55 DEBUG }, 03:24:55 DEBUG "value": [ 03:24:55 DEBUG 03:24:55 DEBUG 03:24:55 DEBUG 03:24:55 DEBUG [...] 03:24:55 DEBUG 03:24:55 DEBUG 03:24:55 DEBUG 03:24:55 DEBUG "pod": "ds-cts-2" 03:24:55 DEBUG }, 03:24:55 DEBUG "value": [ 03:24:55 DEBUG 1713666295, 03:24:55 DEBUG "0.533633333333332" 03:24:55 INFO [http_cmd]: http status code OK 03:24:55 DEBUG ] 03:24:55 DEBUG --- status code --- 03:24:55 DEBUG } 03:24:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:24:55 DEBUG ] 03:24:55 DEBUG --- http response --- 03:24:55 DEBUG } 03:24:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:24:55 DEBUG } 03:24:55 DEBUG ----- output ----- 03:24:55 DEBUG { 03:24:55 DEBUG "status": "success", 03:24:55 DEBUG "data": { 03:24:55 DEBUG "resultType": "vector", 03:24:55 DEBUG "result": [ 03:24:55 DEBUG { 03:24:55 DEBUG "metric": { 03:24:55 DEBUG "instance": "10.142.0.107:9100" 03:24:55 DEBUG }, 03:24:55 DEBUG "value": [ 03:24:55 DEBUG 03:24:55 DEBUG 03:24:55 DEBUG 03:24:55 DEBUG [...] 03:24:55 DEBUG 03:24:55 DEBUG 03:24:55 DEBUG 03:24:55 DEBUG "instance": "10.142.0.43:9100" 03:24:55 DEBUG }, 03:24:55 DEBUG "value": [ 03:24:55 DEBUG 1713666295, 03:24:55 DEBUG "0.402333333333335" 03:24:55 DEBUG ] 03:24:55 DEBUG } 03:24:55 DEBUG ] 03:24:55 DEBUG } 03:24:55 DEBUG } 03:24:55 INFO 03:24:55 INFO [loop_until]: kubectl --namespace=xlou top pods 03:24:55 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:24:55 INFO [loop_until]: OK (rc = 0) 03:24:55 DEBUG --- stdout --- 03:24:55 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 22329m 8796Mi am-767f6b67b5-qcqz2 22146m 9130Mi am-767f6b67b5-vbltz 24613m 8815Mi ds-cts-0 20393m 12480Mi ds-cts-1 19689m 12495Mi ds-cts-2 20106m 12555Mi ds-idrepo-0 9457m 3918Mi ds-idrepo-1 9462m 3946Mi ds-idrepo-2 9762m 3915Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1179Mi idm-b85f9d978-wnhxb 3m 1239Mi lodemon-54c87f8cbb-jzrjs 13m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-68c5d5b884-p97w5 12005m 906Mi 03:24:55 DEBUG --- stderr --- 03:24:55 DEBUG 03:24:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666295" 03:24:55 INFO [http_cmd]: http status code OK 03:24:55 DEBUG --- status code --- 03:24:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:24:55 DEBUG --- http response --- 03:24:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:24:55 DEBUG ----- output ----- 03:24:55 DEBUG { 03:24:55 DEBUG "status": "success", 03:24:55 DEBUG "data": { 03:24:55 DEBUG "resultType": "vector", 03:24:55 DEBUG "result": [ 03:24:55 DEBUG { 03:24:55 DEBUG "metric": { 03:24:55 DEBUG "pod": "am-767f6b67b5-9zw22" 03:24:55 DEBUG }, 03:24:55 DEBUG "value": [ 03:24:55 DEBUG 03:24:55 DEBUG 03:24:55 DEBUG 03:24:55 DEBUG [...] 03:24:55 DEBUG 03:24:55 DEBUG 03:24:55 DEBUG 03:24:55 DEBUG "pod": "am-767f6b67b5-vbltz" 03:24:55 DEBUG }, 03:24:55 DEBUG "value": [ 03:24:55 DEBUG 1713666295, 03:24:55 DEBUG "3332.733333333333" 03:24:55 DEBUG ] 03:24:55 DEBUG } 03:24:55 DEBUG ] 03:24:55 DEBUG } 03:24:55 DEBUG } 03:24:56 INFO 03:24:56 INFO [loop_until]: kubectl --namespace=xlou top node 03:24:56 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:24:56 INFO [loop_until]: OK (rc = 0) 03:24:56 DEBUG --- stdout --- 03:24:56 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 62m 0% 6277Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 10053m 33% 7768Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 22128m 74% 11602Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 89m 0% 3347Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 19242m 64% 15796Mi 14% gke-xlou-cdm-default-pool-1b0769c8-wpwl 140m 0% 5262Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 22463m 75% 12065Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 19645m 65% 3190Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 24668m 82% 11082Mi 9% gke-xlou-cdm-ds-0f601751-j488 10434m 34% 7332Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 19650m 65% 17622Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 10172m 34% 8015Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20426m 68% 15261Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 11997m 75% 11227Mi 19% 03:24:56 DEBUG --- stderr --- 03:24:56 DEBUG 03:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666296" 03:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666296" 03:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666296" 03:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666296" 03:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713666296" 03:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666296" 03:24:56 INFO [http_cmd]: http status code OK 03:24:56 INFO [http_cmd]: http status code OK 03:24:56 DEBUG --- status code --- 03:24:56 DEBUG --- status code --- 03:24:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:24:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:24:56 INFO [http_cmd]: http status code OK 03:24:56 DEBUG --- http response --- 03:24:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 03:24:56 DEBUG --- http response --- 03:24:56 DEBUG --- status code --- 03:24:56 INFO [http_cmd]: http status code OK 03:24:56 DEBUG ----- output ----- 03:24:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:24:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 03:24:56 DEBUG --- status code --- 03:24:56 DEBUG { 03:24:56 INFO [http_cmd]: http status code OK 03:24:56 INFO [http_cmd]: http status code OK 03:24:56 DEBUG --- http response --- 03:24:56 DEBUG ----- output ----- 03:24:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:24:56 DEBUG "status": "success", 03:24:56 DEBUG "data": { 03:24:56 DEBUG "resultType": "vector", 03:24:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:24:56 DEBUG { 03:24:56 DEBUG --- http response --- 03:24:56 DEBUG --- status code --- 03:24:56 DEBUG "result": [ 03:24:56 DEBUG --- status code --- 03:24:56 DEBUG ----- output ----- 03:24:56 DEBUG "status": "success", 03:24:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:24:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:24:56 DEBUG { 03:24:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:24:56 DEBUG { 03:24:56 DEBUG "data": { 03:24:56 DEBUG --- http response --- 03:24:56 DEBUG ----- output ----- 03:24:56 DEBUG "metric": { 03:24:56 DEBUG --- http response --- 03:24:56 DEBUG "status": "success", 03:24:56 DEBUG "resultType": "vector", 03:24:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:24:56 DEBUG { 03:24:56 DEBUG "pod": "ds-idrepo-1" 03:24:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:24:56 DEBUG "data": { 03:24:56 DEBUG "result": [ 03:24:56 DEBUG ----- output ----- 03:24:56 DEBUG "status": "success", 03:24:56 DEBUG }, 03:24:56 DEBUG ----- output ----- 03:24:56 DEBUG "resultType": "vector", 03:24:56 DEBUG { 03:24:56 DEBUG { 03:24:56 DEBUG "data": { 03:24:56 DEBUG "value": [ 03:24:56 DEBUG { 03:24:56 DEBUG "result": [ 03:24:56 DEBUG "metric": { 03:24:56 DEBUG "status": "success", 03:24:56 DEBUG "resultType": "vector", 03:24:56 DEBUG 03:24:56 DEBUG "status": "success", 03:24:56 DEBUG { 03:24:56 DEBUG "pod": "ds-idrepo-1" 03:24:56 DEBUG "data": { 03:24:56 DEBUG "result": [ 03:24:56 DEBUG { 03:24:56 DEBUG "data": { 03:24:56 DEBUG "metric": { 03:24:56 DEBUG }, 03:24:56 DEBUG "resultType": "vector", 03:24:56 DEBUG 03:24:56 DEBUG "metric": { 03:24:56 DEBUG "resultType": "vector", 03:24:56 DEBUG "pod": "am-767f6b67b5-9zw22" 03:24:56 DEBUG "value": [ 03:24:56 DEBUG "result": [ 03:24:56 DEBUG 03:24:56 DEBUG "pod": "ds-idrepo-0" 03:24:56 DEBUG "result": [ 03:24:56 DEBUG }, 03:24:56 DEBUG 03:24:56 DEBUG { 03:24:56 DEBUG [...] 03:24:56 DEBUG }, 03:24:56 DEBUG { 03:24:56 DEBUG "value": [ 03:24:56 DEBUG 03:24:56 DEBUG "metric": { 03:24:56 DEBUG 03:24:56 DEBUG "value": [ 03:24:56 DEBUG "metric": { 03:24:56 DEBUG 03:24:56 DEBUG 03:24:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:24:56 DEBUG 03:24:56 DEBUG 03:24:56 DEBUG "pod": "am-767f6b67b5-9zw22" 03:24:56 DEBUG 03:24:56 DEBUG [...] 03:24:56 DEBUG }, 03:24:56 DEBUG 03:24:56 DEBUG 03:24:56 DEBUG }, 03:24:56 DEBUG 03:24:56 DEBUG 03:24:56 DEBUG 03:24:56 DEBUG "pod": "ds-cts-0" 03:24:56 DEBUG 03:24:56 DEBUG "value": [ 03:24:56 DEBUG [...] 03:24:56 DEBUG 03:24:56 DEBUG }, 03:24:56 DEBUG [...] 03:24:56 DEBUG 03:24:56 DEBUG 03:24:56 DEBUG "pod": "ds-cts-0" 03:24:56 DEBUG "value": [ 03:24:56 DEBUG "value": [ 03:24:56 DEBUG 03:24:56 DEBUG 03:24:56 DEBUG 03:24:56 DEBUG }, 03:24:56 DEBUG 1713666296, 03:24:56 DEBUG "409194526.500219" 03:24:56 DEBUG 03:24:56 DEBUG 03:24:56 DEBUG 03:24:56 DEBUG "value": [ 03:24:56 DEBUG 03:24:56 DEBUG ] 03:24:56 DEBUG "pod": "ds-idrepo-2" 03:24:56 DEBUG 03:24:56 DEBUG "pod": "am-767f6b67b5-vbltz" 03:24:56 DEBUG 1713666296, 03:24:56 DEBUG 03:24:56 DEBUG } 03:24:56 DEBUG }, 03:24:56 DEBUG [...] 03:24:56 DEBUG }, 03:24:56 DEBUG "19.879153213187724" 03:24:56 DEBUG 03:24:56 DEBUG ] 03:24:56 DEBUG "value": [ 03:24:56 DEBUG 03:24:56 DEBUG "value": [ 03:24:56 DEBUG ] 03:24:56 DEBUG [...] 03:24:56 DEBUG } 03:24:56 DEBUG } 03:24:56 DEBUG 03:24:56 DEBUG 1713666296, 03:24:56 DEBUG } 03:24:56 DEBUG 03:24:56 DEBUG 1713666296, 03:24:56 DEBUG 03:24:56 DEBUG "0" 03:24:56 DEBUG ] 03:24:56 DEBUG 03:24:56 DEBUG "0" 03:24:56 DEBUG "pod": "am-767f6b67b5-vbltz" 03:24:56 DEBUG ] 03:24:56 DEBUG } 03:24:56 DEBUG 03:24:56 DEBUG ] 03:24:56 DEBUG }, 03:24:56 DEBUG } 03:24:56 DEBUG } 03:24:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:24:56 DEBUG } 03:24:56 DEBUG "value": [ 03:24:56 DEBUG ] 03:24:56 DEBUG }, 03:24:56 DEBUG ] 03:24:56 DEBUG 1713666296, 03:24:56 DEBUG } 03:24:56 DEBUG "value": [ 03:24:56 DEBUG } 03:24:56 DEBUG "3336.9333333333334" 03:24:56 DEBUG } 03:24:56 DEBUG 1713666296, 03:24:56 DEBUG } 03:24:56 DEBUG ] 03:24:56 DEBUG } 03:24:56 DEBUG ] 03:24:56 DEBUG } 03:24:56 DEBUG "11.747856250598463" 03:24:56 DEBUG } 03:24:56 DEBUG ] 03:24:56 DEBUG } 03:24:56 DEBUG ] 03:24:56 DEBUG } 03:24:56 DEBUG } 03:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666296" 03:24:56 INFO [http_cmd]: http status code OK 03:24:56 DEBUG --- status code --- 03:24:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:24:56 DEBUG --- http response --- 03:24:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:24:56 DEBUG ----- output ----- 03:24:56 DEBUG { 03:24:56 DEBUG "status": "success", 03:24:56 DEBUG "data": { 03:24:56 DEBUG "resultType": "vector", 03:24:56 DEBUG "result": [ 03:24:56 DEBUG { 03:24:56 DEBUG "metric": { 03:24:56 DEBUG "pod": "am-767f6b67b5-9zw22" 03:24:56 DEBUG }, 03:24:56 DEBUG "value": [ 03:24:56 DEBUG 03:24:56 DEBUG 03:24:56 DEBUG 03:24:56 DEBUG [...] 03:24:56 DEBUG 03:24:56 DEBUG 03:24:56 DEBUG 03:24:56 DEBUG "pod": "am-767f6b67b5-vbltz" 03:24:56 DEBUG }, 03:24:56 DEBUG "value": [ 03:24:56 DEBUG 1713666296, 03:24:56 DEBUG "599637" 03:24:56 DEBUG ] 03:24:56 DEBUG } 03:24:56 DEBUG ] 03:24:56 DEBUG } 03:24:56 DEBUG } 03:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666296" 03:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713666296" 03:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666296" 03:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666296" 03:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666296" 03:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666296" 03:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713666296" 03:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666296" 03:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666296" 03:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666296" 03:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666296" 03:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666296" 03:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666296" 03:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713666296" 03:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666296" 03:24:56 INFO [http_cmd]: http status code OK 03:24:56 INFO [http_cmd]: http status code OK 03:24:56 INFO [http_cmd]: http status code OK 03:24:56 INFO [http_cmd]: http status code OK 03:24:56 INFO [http_cmd]: http status code OK 03:24:56 INFO [http_cmd]: http status code OK 03:24:56 INFO [http_cmd]: http status code OK 03:24:56 INFO [http_cmd]: http status code OK 03:24:56 DEBUG --- status code --- 03:24:56 INFO [http_cmd]: http status code OK 03:24:56 INFO [http_cmd]: http status code OK 03:24:56 INFO [http_cmd]: http status code OK 03:24:56 DEBUG --- status code --- 03:24:56 INFO [http_cmd]: http status code OK 03:24:56 DEBUG --- status code --- 03:24:56 DEBUG --- status code --- 03:24:56 DEBUG --- status code --- 03:24:56 DEBUG --- status code --- 03:24:56 DEBUG --- status code --- 03:24:56 INFO [http_cmd]: http status code OK 03:24:56 INFO [http_cmd]: http status code OK 03:24:56 DEBUG --- status code --- 03:24:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:24:56 DEBUG --- status code --- 03:24:56 DEBUG --- status code --- 03:24:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:24:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:24:56 DEBUG --- status code --- 03:24:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:24:56 DEBUG --- http response --- 03:24:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:24:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:24:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:24:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:24:56 DEBUG --- http response --- 03:24:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 03:24:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:24:56 DEBUG --- http response --- 03:24:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:24:56 DEBUG --- http response --- 03:24:56 DEBUG --- status code --- 03:24:56 DEBUG --- http response --- 03:24:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:24:56 INFO [http_cmd]: http status code OK 03:24:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:24:56 DEBUG --- http response --- 03:24:56 DEBUG --- http response --- 03:24:56 DEBUG --- http response --- 03:24:56 DEBUG ----- output ----- 03:24:56 DEBUG --- status code --- 03:24:56 DEBUG --- status code --- 03:24:56 DEBUG --- http response --- 03:24:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:24:56 DEBUG --- http response --- 03:24:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:24:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:24:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:24:56 DEBUG --- http response --- 03:24:56 DEBUG --- status code --- 03:24:56 DEBUG ----- output ----- 03:24:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:24:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:24:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:24:56 DEBUG { 03:24:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:24:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:24:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:24:56 DEBUG ----- output ----- 03:24:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:24:56 DEBUG ----- output ----- 03:24:56 DEBUG --- http response --- 03:24:56 DEBUG ----- output ----- 03:24:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 03:24:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:24:56 DEBUG { 03:24:56 DEBUG ----- output ----- 03:24:56 DEBUG ----- output ----- 03:24:56 DEBUG ----- output ----- 03:24:56 DEBUG "status": "success", 03:24:56 DEBUG --- http response --- 03:24:56 DEBUG --- http response --- 03:24:56 DEBUG ----- output ----- 03:24:56 DEBUG { 03:24:56 DEBUG ----- output ----- 03:24:56 DEBUG { 03:24:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:24:56 DEBUG { 03:24:56 DEBUG ----- output ----- 03:24:56 DEBUG --- http response --- 03:24:56 DEBUG "status": "success", 03:24:56 DEBUG { 03:24:56 DEBUG { 03:24:56 DEBUG { 03:24:56 DEBUG "data": { 03:24:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:24:56 DEBUG { 03:24:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:24:56 DEBUG "status": "success", 03:24:56 DEBUG { 03:24:56 DEBUG "status": "success", 03:24:56 DEBUG ----- output ----- 03:24:56 DEBUG "status": "success", 03:24:56 DEBUG { 03:24:56 DEBUG "status": "success", 03:24:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:24:56 DEBUG "status": "success", 03:24:56 DEBUG "status": "success", 03:24:56 DEBUG "status": "success", 03:24:56 DEBUG "resultType": "vector", 03:24:56 DEBUG ----- output ----- 03:24:56 DEBUG "status": "success", 03:24:56 DEBUG ----- output ----- 03:24:56 DEBUG "data": { 03:24:56 DEBUG "data": { 03:24:56 DEBUG "status": "success", 03:24:56 DEBUG { 03:24:56 DEBUG "data": { 03:24:56 DEBUG "data": { 03:24:56 DEBUG "data": { 03:24:56 DEBUG ----- output ----- 03:24:56 DEBUG "data": { 03:24:56 DEBUG "data": { 03:24:56 DEBUG "data": { 03:24:56 DEBUG "result": [ 03:24:56 DEBUG { 03:24:56 DEBUG "data": { 03:24:56 DEBUG { 03:24:56 DEBUG "resultType": "vector", 03:24:56 DEBUG "resultType": "vector", 03:24:56 DEBUG "data": { 03:24:56 DEBUG "status": "success", 03:24:56 DEBUG "resultType": "vector", 03:24:56 DEBUG "resultType": "vector", 03:24:56 DEBUG "resultType": "vector", 03:24:56 DEBUG { 03:24:56 DEBUG "resultType": "vector", 03:24:56 DEBUG "resultType": "vector", 03:24:56 DEBUG "resultType": "vector", 03:24:56 DEBUG { 03:24:56 DEBUG "status": "success", 03:24:56 DEBUG "resultType": "vector", 03:24:56 DEBUG "status": "success", 03:24:56 DEBUG "result": [ 03:24:56 DEBUG "result": [ 03:24:56 DEBUG "resultType": "vector", 03:24:56 DEBUG "data": { 03:24:56 DEBUG "result": [ 03:24:56 DEBUG "result": [ 03:24:56 DEBUG "result": [ 03:24:56 DEBUG "status": "success", 03:24:56 DEBUG "result": [ 03:24:56 DEBUG "result": [ 03:24:56 DEBUG { 03:24:56 DEBUG "result": [ 03:24:56 DEBUG "data": { 03:24:56 DEBUG "result": [ 03:24:56 DEBUG "data": { 03:24:56 DEBUG "resultType": "vector", 03:24:56 DEBUG { 03:24:56 DEBUG "result": [ 03:24:56 DEBUG "resultType": "vector", 03:24:56 DEBUG { 03:24:56 DEBUG { 03:24:56 DEBUG { 03:24:56 DEBUG "data": { 03:24:56 DEBUG { 03:24:56 DEBUG "metric": { 03:24:56 DEBUG "metric": { 03:24:56 DEBUG { 03:24:56 DEBUG "metric": { 03:24:56 DEBUG "resultType": "vector", 03:24:56 DEBUG "result": [ 03:24:56 DEBUG { 03:24:56 DEBUG "metric": { 03:24:56 DEBUG { 03:24:56 DEBUG "result": [ 03:24:56 DEBUG "metric": { 03:24:56 DEBUG "metric": { 03:24:56 DEBUG "metric": { 03:24:56 DEBUG "resultType": "vector", 03:24:56 DEBUG "metric": { 03:24:56 DEBUG "pod": "ds-cts-0" 03:24:56 DEBUG "instance": "10.142.0.107:9100" 03:24:56 DEBUG { 03:24:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:24:56 DEBUG "result": [ 03:24:56 DEBUG { 03:24:56 DEBUG "metric": { 03:24:56 DEBUG "instance": "10.142.0.107:9100" 03:24:56 DEBUG "metric": { 03:24:56 DEBUG { 03:24:56 DEBUG "metric": { 03:24:56 DEBUG "pod": "ds-idrepo-1" 03:24:56 DEBUG }, 03:24:56 DEBUG "result": [ 03:24:56 DEBUG { 03:24:56 DEBUG "metric": { 03:24:56 DEBUG }, 03:24:56 DEBUG "metric": { 03:24:56 DEBUG }, 03:24:56 DEBUG { 03:24:56 DEBUG "metric": { 03:24:56 DEBUG "pod": "ds-cts-0" 03:24:56 DEBUG }, 03:24:56 DEBUG "pod": "am-767f6b67b5-9zw22" 03:24:56 DEBUG "pod": "am-767f6b67b5-9zw22" 03:24:56 DEBUG "pod": "am-767f6b67b5-9zw22" 03:24:56 DEBUG "value": [ 03:24:56 DEBUG "instance": "10.142.0.107:9100" 03:24:56 DEBUG "pod": "ds-idrepo-0" 03:24:56 DEBUG "pod": "ds-idrepo-1" 03:24:56 DEBUG }, 03:24:56 DEBUG "value": [ 03:24:56 DEBUG "instance": "10.142.0.107:9100" 03:24:56 DEBUG "value": [ 03:24:56 DEBUG "metric": { 03:24:56 DEBUG "pod": "ds-idrepo-0" 03:24:56 DEBUG }, 03:24:56 DEBUG "value": [ 03:24:56 DEBUG }, 03:24:56 DEBUG }, 03:24:56 DEBUG }, 03:24:56 DEBUG 03:24:56 DEBUG }, 03:24:56 DEBUG }, 03:24:56 DEBUG }, 03:24:56 DEBUG "value": [ 03:24:56 DEBUG 03:24:56 DEBUG }, 03:24:56 DEBUG 03:24:56 DEBUG "pod": "ds-idrepo-0" 03:24:56 DEBUG }, 03:24:56 DEBUG "value": [ 03:24:56 DEBUG 03:24:56 DEBUG "value": [ 03:24:56 DEBUG "value": [ 03:24:56 DEBUG "value": [ 03:24:56 DEBUG 03:24:56 DEBUG "value": [ 03:24:56 DEBUG "value": [ 03:24:56 DEBUG "value": [ 03:24:56 DEBUG 03:24:56 DEBUG 03:24:56 DEBUG "value": [ 03:24:56 DEBUG 03:24:56 DEBUG 03:24:56 DEBUG [...] 03:24:56 DEBUG 03:24:56 DEBUG 03:24:56 DEBUG 03:24:56 DEBUG 03:24:56 DEBUG 03:24:56 DEBUG 03:24:56 DEBUG 03:24:56 DEBUG 03:24:56 DEBUG 03:24:56 DEBUG 03:24:56 DEBUG 03:24:56 DEBUG 03:24:56 DEBUG 03:24:56 DEBUG 03:24:56 DEBUG "value": [ 03:24:56 DEBUG 03:24:56 DEBUG [...] 03:24:56 DEBUG 03:24:56 DEBUG 03:24:56 DEBUG 03:24:56 DEBUG [...] 03:24:56 DEBUG 03:24:56 DEBUG 03:24:56 DEBUG 03:24:56 DEBUG 03:24:56 DEBUG [...] 03:24:56 DEBUG 03:24:56 DEBUG 03:24:56 DEBUG 03:24:56 DEBUG [...] 03:24:56 DEBUG }, 03:24:56 DEBUG 03:24:56 DEBUG 03:24:56 DEBUG 03:24:56 DEBUG 03:24:56 DEBUG 03:24:56 DEBUG 03:24:56 DEBUG 03:24:56 DEBUG 03:24:56 DEBUG [...] 03:24:56 DEBUG 03:24:56 DEBUG 03:24:56 DEBUG 03:24:56 DEBUG 03:24:56 DEBUG 03:24:56 DEBUG "value": [ 03:24:56 DEBUG 03:24:56 DEBUG [...] 03:24:56 DEBUG [...] 03:24:56 DEBUG [...] 03:24:56 DEBUG 03:24:56 DEBUG [...] 03:24:56 DEBUG [...] 03:24:56 DEBUG [...] 03:24:56 DEBUG 03:24:56 DEBUG 03:24:56 DEBUG [...] 03:24:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:24:56 DEBUG 03:24:56 DEBUG 03:24:56 DEBUG 03:24:56 DEBUG 03:24:56 DEBUG 03:24:56 DEBUG 03:24:56 DEBUG 03:24:56 DEBUG 03:24:56 DEBUG 03:24:56 DEBUG 03:24:56 DEBUG 03:24:56 DEBUG 03:24:56 DEBUG 03:24:56 DEBUG 03:24:56 DEBUG }, 03:24:56 DEBUG 03:24:56 DEBUG 03:24:56 DEBUG 03:24:56 DEBUG "pod": "ds-cts-2" 03:24:56 DEBUG }, 03:24:56 DEBUG 03:24:56 DEBUG 03:24:56 DEBUG "pod": "ds-cts-0" 03:24:56 DEBUG 03:24:56 DEBUG 03:24:56 DEBUG 03:24:56 DEBUG 03:24:56 DEBUG "instance": "10.142.0.43:9100" 03:24:56 DEBUG 03:24:56 DEBUG "value": [ 03:24:56 DEBUG [...] 03:24:56 DEBUG "instance": "10.142.0.43:9100" 03:24:56 DEBUG 03:24:56 DEBUG "value": [ 03:24:56 DEBUG 03:24:56 DEBUG 03:24:56 DEBUG 03:24:56 DEBUG }, 03:24:56 DEBUG 03:24:56 DEBUG 03:24:56 DEBUG 03:24:56 DEBUG "pod": "ds-cts-2" 03:24:56 DEBUG }, 03:24:56 DEBUG 03:24:56 DEBUG 1713666296, 03:24:56 DEBUG 03:24:56 DEBUG 03:24:56 DEBUG [...] 03:24:56 DEBUG 1713666296, 03:24:56 DEBUG 03:24:56 DEBUG "pod": "am-767f6b67b5-vbltz" 03:24:56 DEBUG "pod": "am-767f6b67b5-vbltz" 03:24:56 DEBUG "value": [ 03:24:56 DEBUG "instance": "10.142.0.43:9100" 03:24:56 DEBUG }, 03:24:56 DEBUG "pod": "ds-cts-0" 03:24:56 DEBUG }, 03:24:56 DEBUG "value": [ 03:24:56 DEBUG "instance": "10.142.0.43:9100" 03:24:56 DEBUG "1049092096" 03:24:56 DEBUG 03:24:56 DEBUG 03:24:56 DEBUG "0" 03:24:56 DEBUG }, 03:24:56 DEBUG "pod": "am-767f6b67b5-vbltz" 03:24:56 DEBUG }, 03:24:56 DEBUG }, 03:24:56 DEBUG 1713666296, 03:24:56 DEBUG "value": [ 03:24:56 DEBUG "pod": "ds-idrepo-2" 03:24:56 DEBUG }, 03:24:57 DEBUG "value": [ 03:24:56 DEBUG 1713666296, 03:24:56 DEBUG }, 03:24:57 DEBUG "value": [ 03:24:56 DEBUG "pod": "ds-idrepo-2" 03:24:56 DEBUG 03:24:56 DEBUG ] 03:24:56 DEBUG "value": [ 03:24:56 DEBUG }, 03:24:56 DEBUG "value": [ 03:24:56 DEBUG "value": [ 03:24:57 DEBUG 1713666296, 03:24:56 DEBUG 1713666296, 03:24:57 DEBUG }, 03:24:57 DEBUG 1713666296, 03:24:57 DEBUG "100622948.9" 03:24:56 DEBUG "value": [ 03:24:56 DEBUG ] 03:24:57 DEBUG 1713666296, 03:24:57 DEBUG }, 03:24:57 DEBUG 03:24:57 DEBUG } 03:24:57 DEBUG 1713666296, 03:24:57 DEBUG "value": [ 03:24:57 DEBUG 1713666296, 03:24:57 DEBUG "150.86599999999999" 03:24:57 DEBUG "4421878465536" 03:24:57 DEBUG "value": [ 03:24:57 DEBUG "76038327.51392278" 03:24:57 DEBUG ] 03:24:57 DEBUG 1713666296, 03:24:56 DEBUG "84452158.93873976" 03:24:57 DEBUG } 03:24:57 DEBUG "0.0826666666666521" 03:24:57 DEBUG "value": [ 03:24:57 DEBUG "pod": "ds-cts-2" 03:24:57 DEBUG ] 03:24:57 DEBUG "100622465.43333334" 03:24:57 DEBUG 1713666296, 03:24:57 DEBUG "3648.9666666666667" 03:24:57 DEBUG ] 03:24:57 DEBUG ] 03:24:57 DEBUG 1713666296, 03:24:57 DEBUG ] 03:24:57 DEBUG } 03:24:57 DEBUG "29.5" 03:24:57 DEBUG ] 03:24:57 DEBUG ] 03:24:57 DEBUG ] 03:24:57 DEBUG 1713666296, 03:24:57 DEBUG }, 03:24:57 DEBUG } 03:24:57 DEBUG ] 03:24:57 DEBUG "3333.7999999999997" 03:24:57 DEBUG ] 03:24:57 DEBUG } 03:24:57 DEBUG ] 03:24:57 DEBUG "0" 03:24:57 DEBUG } 03:24:57 DEBUG ] 03:24:57 DEBUG ] 03:24:57 DEBUG } 03:24:57 DEBUG } 03:24:57 DEBUG } 03:24:57 DEBUG "0" 03:24:57 DEBUG "value": [ 03:24:57 DEBUG } 03:24:57 DEBUG } 03:24:57 DEBUG ] 03:24:57 DEBUG } 03:24:57 DEBUG } 03:24:57 DEBUG ] 03:24:57 DEBUG } 03:24:57 DEBUG ] 03:24:57 DEBUG } 03:24:57 DEBUG } 03:24:57 DEBUG ] 03:24:57 DEBUG } 03:24:57 DEBUG ] 03:24:57 DEBUG ] 03:24:57 DEBUG 1713666296, 03:24:57 DEBUG ] 03:24:57 DEBUG } 03:24:57 DEBUG ] 03:24:57 DEBUG } 03:24:57 DEBUG } 03:24:57 DEBUG ] 03:24:57 DEBUG } 03:24:57 DEBUG } 03:24:57 DEBUG ] 03:24:57 DEBUG } 03:24:57 DEBUG } 03:24:57 DEBUG } 03:24:57 DEBUG "0" 03:24:57 DEBUG } 03:24:57 DEBUG ] 03:24:57 DEBUG } 03:24:57 DEBUG ] 03:24:57 DEBUG } 03:24:57 DEBUG } 03:24:57 DEBUG } 03:24:57 DEBUG } 03:24:57 DEBUG } 03:24:57 DEBUG ] 03:24:57 DEBUG ] 03:24:57 DEBUG } 03:24:57 DEBUG } 03:24:57 DEBUG } 03:24:57 DEBUG } 03:24:57 DEBUG } 03:24:57 DEBUG } 03:24:57 DEBUG } 03:24:57 DEBUG } 03:24:57 DEBUG } 03:24:57 DEBUG } 03:24:57 DEBUG } 03:24:57 DEBUG ] 03:24:57 DEBUG } 03:24:57 DEBUG } 03:25:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713666353" 03:25:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666353" 03:25:53 ERROR [http_cmd]: ERROR 03:25:53 INFO [http_cmd]: http status code OK 03:25:53 ERROR --- status code --- 03:25:53 DEBUG --- status code --- 03:25:53 ERROR http status code is 400 (expected [200, 401, 403, 404]) 03:25:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:25:53 ERROR --- http response --- 03:25:53 DEBUG --- http response --- 03:25:53 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 03:25:53 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 03:25:53 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 03:25:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666355" 03:25:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666355" 03:25:55 INFO [http_cmd]: http status code OK 03:25:55 DEBUG --- status code --- 03:25:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:25:55 DEBUG --- http response --- 03:25:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:25:55 DEBUG ----- output ----- 03:25:55 DEBUG { 03:25:55 DEBUG "status": "success", 03:25:55 DEBUG "data": { 03:25:55 DEBUG "resultType": "vector", 03:25:55 DEBUG "result": [ 03:25:55 DEBUG { 03:25:55 DEBUG "metric": { 03:25:55 DEBUG "instance": "10.142.0.107:9100" 03:25:55 DEBUG }, 03:25:55 DEBUG "value": [ 03:25:55 DEBUG 03:25:55 DEBUG 03:25:55 DEBUG 03:25:55 DEBUG [...] 03:25:55 INFO [http_cmd]: http status code OK 03:25:55 DEBUG 03:25:55 DEBUG --- status code --- 03:25:55 DEBUG 03:25:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:25:55 DEBUG 03:25:55 DEBUG --- http response --- 03:25:55 DEBUG "instance": "10.142.0.43:9100" 03:25:55 DEBUG }, 03:25:55 DEBUG "value": [ 03:25:55 DEBUG 1713666355, 03:25:55 DEBUG "0.39503333333331625" 03:25:55 DEBUG ] 03:25:55 DEBUG } 03:25:55 DEBUG ] 03:25:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:25:55 DEBUG ----- output ----- 03:25:55 DEBUG { 03:25:55 DEBUG "status": "success", 03:25:55 DEBUG "data": { 03:25:55 DEBUG "resultType": "vector", 03:25:55 DEBUG "result": [ 03:25:55 DEBUG { 03:25:55 DEBUG "metric": { 03:25:55 DEBUG "pod": "ds-cts-0" 03:25:55 DEBUG }, 03:25:55 DEBUG "value": [ 03:25:55 DEBUG 03:25:55 DEBUG 03:25:55 DEBUG 03:25:55 DEBUG [...] 03:25:55 DEBUG 03:25:55 DEBUG 03:25:55 DEBUG 03:25:55 DEBUG "pod": "ds-cts-2" 03:25:55 DEBUG }, 03:25:55 DEBUG "value": [ 03:25:55 DEBUG 1713666355, 03:25:55 DEBUG "0.5842333333333329" 03:25:55 DEBUG ] 03:25:55 DEBUG } 03:25:55 DEBUG ] 03:25:55 DEBUG } 03:25:55 DEBUG } 03:25:55 DEBUG } 03:25:55 DEBUG } 03:25:55 INFO 03:25:55 INFO [loop_until]: kubectl --namespace=xlou top pods 03:25:55 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:25:55 INFO [loop_until]: OK (rc = 0) 03:25:55 DEBUG --- stdout --- 03:25:55 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 22661m 8803Mi am-767f6b67b5-qcqz2 22515m 9128Mi am-767f6b67b5-vbltz 24470m 8845Mi ds-cts-0 19439m 12477Mi ds-cts-1 19500m 12490Mi ds-cts-2 19422m 12554Mi ds-idrepo-0 9425m 3919Mi ds-idrepo-1 9389m 3946Mi ds-idrepo-2 9541m 3916Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1179Mi idm-b85f9d978-wnhxb 3m 1239Mi lodemon-54c87f8cbb-jzrjs 15m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-68c5d5b884-p97w5 11872m 909Mi 03:25:55 DEBUG --- stderr --- 03:25:55 DEBUG 03:25:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666355" 03:25:55 INFO [http_cmd]: http status code OK 03:25:55 DEBUG --- status code --- 03:25:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:25:55 DEBUG --- http response --- 03:25:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:25:55 DEBUG ----- output ----- 03:25:55 DEBUG { 03:25:55 DEBUG "status": "success", 03:25:55 DEBUG "data": { 03:25:55 DEBUG "resultType": "vector", 03:25:55 DEBUG "result": [ 03:25:55 DEBUG { 03:25:55 DEBUG "metric": { 03:25:55 DEBUG "pod": "am-767f6b67b5-9zw22" 03:25:55 DEBUG }, 03:25:55 DEBUG "value": [ 03:25:55 DEBUG 03:25:55 DEBUG 03:25:55 DEBUG 03:25:55 DEBUG [...] 03:25:55 DEBUG 03:25:55 DEBUG 03:25:55 DEBUG 03:25:55 DEBUG "pod": "am-767f6b67b5-vbltz" 03:25:55 DEBUG }, 03:25:55 DEBUG "value": [ 03:25:55 DEBUG 1713666355, 03:25:55 DEBUG "3324.7" 03:25:55 DEBUG ] 03:25:55 DEBUG } 03:25:55 DEBUG ] 03:25:55 DEBUG } 03:25:55 DEBUG } 03:25:56 INFO 03:25:56 INFO [loop_until]: kubectl --namespace=xlou top node 03:25:56 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:25:56 INFO [loop_until]: OK (rc = 0) 03:25:56 DEBUG --- stdout --- 03:25:56 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 6281Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 9881m 33% 7757Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 22059m 73% 11623Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 85m 0% 3361Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 19781m 66% 15794Mi 14% gke-xlou-cdm-default-pool-1b0769c8-wpwl 162m 0% 5250Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 22406m 75% 12058Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 19702m 65% 3172Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 24857m 83% 11101Mi 9% gke-xlou-cdm-ds-0f601751-j488 10339m 34% 7356Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 19715m 66% 17630Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 10153m 34% 8002Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20315m 68% 15280Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 12040m 75% 11230Mi 19% 03:25:56 DEBUG --- stderr --- 03:25:56 DEBUG 03:25:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666356" 03:25:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666356" 03:25:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666356" 03:25:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666356" 03:25:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666356" 03:25:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713666356" 03:25:56 INFO [http_cmd]: http status code OK 03:25:56 INFO [http_cmd]: http status code OK 03:25:56 DEBUG --- status code --- 03:25:56 DEBUG --- status code --- 03:25:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:25:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:25:56 DEBUG --- http response --- 03:25:56 DEBUG --- http response --- 03:25:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 03:25:56 INFO [http_cmd]: http status code OK 03:25:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 03:25:56 INFO [http_cmd]: http status code OK 03:25:56 DEBUG ----- output ----- 03:25:56 DEBUG --- status code --- 03:25:56 DEBUG ----- output ----- 03:25:56 DEBUG --- status code --- 03:25:56 DEBUG { 03:25:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:25:56 DEBUG { 03:25:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:25:56 DEBUG "status": "success", 03:25:56 DEBUG --- http response --- 03:25:56 DEBUG "status": "success", 03:25:56 DEBUG --- http response --- 03:25:56 DEBUG "data": { 03:25:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:25:56 DEBUG "data": { 03:25:56 INFO [http_cmd]: http status code OK 03:25:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:25:56 DEBUG "resultType": "vector", 03:25:56 DEBUG ----- output ----- 03:25:56 DEBUG "resultType": "vector", 03:25:56 DEBUG --- status code --- 03:25:56 DEBUG ----- output ----- 03:25:56 DEBUG "result": [ 03:25:56 DEBUG { 03:25:56 DEBUG "result": [ 03:25:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:25:56 DEBUG { 03:25:56 DEBUG { 03:25:56 DEBUG "status": "success", 03:25:56 DEBUG { 03:25:56 DEBUG --- http response --- 03:25:56 DEBUG "status": "success", 03:25:56 DEBUG "metric": { 03:25:56 DEBUG "data": { 03:25:56 DEBUG "metric": { 03:25:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:25:56 DEBUG "data": { 03:25:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:25:56 DEBUG "resultType": "vector", 03:25:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:25:56 DEBUG ----- output ----- 03:25:56 DEBUG "resultType": "vector", 03:25:56 DEBUG }, 03:25:56 DEBUG "result": [ 03:25:56 DEBUG }, 03:25:56 DEBUG { 03:25:56 DEBUG "result": [ 03:25:56 DEBUG "value": [ 03:25:56 DEBUG { 03:25:56 DEBUG "value": [ 03:25:56 DEBUG "status": "success", 03:25:56 DEBUG { 03:25:56 DEBUG 03:25:56 DEBUG "metric": { 03:25:56 INFO [http_cmd]: http status code OK 03:25:56 DEBUG 03:25:56 DEBUG "data": { 03:25:56 DEBUG "metric": { 03:25:56 DEBUG 03:25:56 DEBUG "pod": "ds-idrepo-0" 03:25:56 DEBUG --- status code --- 03:25:56 DEBUG 03:25:56 DEBUG "resultType": "vector", 03:25:56 DEBUG "pod": "am-767f6b67b5-9zw22" 03:25:56 DEBUG 03:25:56 DEBUG }, 03:25:56 DEBUG "value": [ 03:25:56 DEBUG 03:25:56 DEBUG "result": [ 03:25:56 DEBUG }, 03:25:56 DEBUG [...] 03:25:56 DEBUG 03:25:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:25:56 DEBUG [...] 03:25:56 DEBUG { 03:25:56 DEBUG "metric": { 03:25:56 DEBUG 03:25:56 DEBUG --- http response --- 03:25:56 DEBUG 03:25:56 DEBUG 03:25:56 DEBUG "value": [ 03:25:56 DEBUG "pod": "am-767f6b67b5-9zw22" 03:25:56 DEBUG 03:25:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:25:56 DEBUG 03:25:56 DEBUG 03:25:56 DEBUG 03:25:56 DEBUG }, 03:25:56 DEBUG "pod": "ds-cts-0" 03:25:56 DEBUG ----- output ----- 03:25:56 DEBUG 03:25:56 DEBUG 03:25:56 DEBUG 03:25:56 DEBUG "value": [ 03:25:56 DEBUG }, 03:25:56 DEBUG { 03:25:56 DEBUG [...] 03:25:56 DEBUG "pod": "ds-cts-0" 03:25:56 DEBUG 03:25:56 DEBUG 03:25:56 DEBUG "value": [ 03:25:56 DEBUG "status": "success", 03:25:56 DEBUG "data": { 03:25:56 DEBUG "resultType": "vector", 03:25:56 DEBUG }, 03:25:56 DEBUG 03:25:56 DEBUG 1713666356, 03:25:56 DEBUG [...] 03:25:56 DEBUG "result": [ 03:25:56 DEBUG "value": [ 03:25:56 DEBUG 03:25:56 DEBUG 03:25:56 DEBUG [...] 03:25:56 DEBUG 03:25:56 DEBUG 03:25:56 DEBUG 1713666356, 03:25:56 DEBUG "0" 03:25:56 DEBUG "0" 03:25:56 DEBUG 03:25:56 DEBUG { 03:25:56 DEBUG "metric": { 03:25:56 DEBUG ] 03:25:56 DEBUG 03:25:56 DEBUG ] 03:25:56 DEBUG 03:25:56 DEBUG 03:25:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:25:56 DEBUG } 03:25:56 DEBUG 03:25:56 DEBUG } 03:25:56 DEBUG 03:25:56 DEBUG "pod": "am-767f6b67b5-vbltz" 03:25:56 DEBUG }, 03:25:56 DEBUG ] 03:25:56 DEBUG "pod": "ds-idrepo-2" 03:25:56 DEBUG ] 03:25:56 DEBUG "pod": "am-767f6b67b5-vbltz" 03:25:56 DEBUG }, 03:25:56 DEBUG "value": [ 03:25:56 DEBUG } 03:25:56 DEBUG }, 03:25:56 DEBUG } 03:25:56 DEBUG }, 03:25:56 DEBUG "value": [ 03:25:56 DEBUG 03:25:56 DEBUG } 03:25:56 DEBUG "value": [ 03:25:56 DEBUG 1713666356, 03:25:56 DEBUG "value": [ 03:25:56 DEBUG 1713666356, 03:25:56 DEBUG 03:25:56 DEBUG "0" 03:25:56 DEBUG } 03:25:56 DEBUG 1713666356, 03:25:56 DEBUG "0" 03:25:56 DEBUG 03:25:56 DEBUG ] 03:25:56 DEBUG "3323.733333333333" 03:25:56 DEBUG ] 03:25:56 DEBUG [...] 03:25:56 DEBUG } 03:25:56 DEBUG ] 03:25:56 DEBUG } 03:25:56 DEBUG 03:25:56 DEBUG } 03:25:56 DEBUG ] 03:25:56 DEBUG 03:25:56 DEBUG } 03:25:56 DEBUG ] 03:25:56 DEBUG } 03:25:56 DEBUG 03:25:56 DEBUG } 03:25:56 DEBUG ] 03:25:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:25:56 DEBUG } 03:25:56 DEBUG } 03:25:56 DEBUG }, 03:25:56 DEBUG } 03:25:56 DEBUG "value": [ 03:25:56 DEBUG 1713666356, 03:25:56 DEBUG "11.795772563083537" 03:25:56 DEBUG ] 03:25:56 DEBUG } 03:25:56 DEBUG ] 03:25:56 DEBUG } 03:25:56 DEBUG } 03:25:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666356" 03:25:56 INFO [http_cmd]: http status code OK 03:25:56 DEBUG --- status code --- 03:25:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:25:56 DEBUG --- http response --- 03:25:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:25:56 DEBUG ----- output ----- 03:25:56 DEBUG { 03:25:56 DEBUG "status": "success", 03:25:56 DEBUG "data": { 03:25:56 DEBUG "resultType": "vector", 03:25:56 DEBUG "result": [ 03:25:56 DEBUG { 03:25:56 DEBUG "metric": { 03:25:56 DEBUG "pod": "am-767f6b67b5-9zw22" 03:25:56 DEBUG }, 03:25:56 DEBUG "value": [ 03:25:56 DEBUG 03:25:56 DEBUG 03:25:56 DEBUG 03:25:56 DEBUG [...] 03:25:56 DEBUG 03:25:56 DEBUG 03:25:56 DEBUG 03:25:56 DEBUG "pod": "am-767f6b67b5-vbltz" 03:25:56 DEBUG }, 03:25:56 DEBUG "value": [ 03:25:56 DEBUG 1713666356, 03:25:56 DEBUG "600069.5" 03:25:56 DEBUG ] 03:25:56 DEBUG } 03:25:56 DEBUG ] 03:25:56 DEBUG } 03:25:56 DEBUG } 03:25:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666357" 03:25:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713666357" 03:25:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713666357" 03:25:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666357" 03:25:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666357" 03:25:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666357" 03:25:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666357" 03:25:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666357" 03:25:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666357" 03:25:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666357" 03:25:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666357" 03:25:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713666357" 03:25:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666357" 03:25:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666357" 03:25:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666357" 03:25:57 INFO [http_cmd]: http status code OK 03:25:57 INFO [http_cmd]: http status code OK 03:25:57 INFO [http_cmd]: http status code OK 03:25:57 INFO [http_cmd]: http status code OK 03:25:57 INFO [http_cmd]: http status code OK 03:25:57 INFO [http_cmd]: http status code OK 03:25:57 INFO [http_cmd]: http status code OK 03:25:57 INFO [http_cmd]: http status code OK 03:25:57 INFO [http_cmd]: http status code OK 03:25:57 DEBUG --- status code --- 03:25:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:25:57 DEBUG --- status code --- 03:25:57 DEBUG --- status code --- 03:25:57 DEBUG --- status code --- 03:25:57 INFO [http_cmd]: http status code OK 03:25:57 DEBUG --- status code --- 03:25:57 DEBUG --- status code --- 03:25:57 DEBUG --- status code --- 03:25:57 DEBUG --- status code --- 03:25:57 DEBUG --- status code --- 03:25:57 INFO [http_cmd]: http status code OK 03:25:57 INFO [http_cmd]: http status code OK 03:25:57 DEBUG --- http response --- 03:25:57 INFO [http_cmd]: http status code OK 03:25:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:25:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:25:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:25:57 DEBUG --- status code --- 03:25:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:25:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:25:57 INFO [http_cmd]: http status code OK 03:25:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:25:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:25:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:25:57 DEBUG --- status code --- 03:25:57 DEBUG --- status code --- 03:25:57 DEBUG --- status code --- 03:25:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:25:57 DEBUG --- http response --- 03:25:57 INFO [http_cmd]: http status code OK 03:25:57 DEBUG --- http response --- 03:25:57 DEBUG --- http response --- 03:25:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:25:57 DEBUG --- http response --- 03:25:57 DEBUG --- http response --- 03:25:57 DEBUG --- status code --- 03:25:57 DEBUG --- http response --- 03:25:57 DEBUG --- http response --- 03:25:57 DEBUG --- http response --- 03:25:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:25:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:25:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:25:57 DEBUG ----- output ----- 03:25:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:25:57 DEBUG --- status code --- 03:25:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:25:57 DEBUG ----- output ----- 03:25:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 03:25:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:25:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:25:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:25:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 03:25:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:25:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:25:57 DEBUG --- http response --- 03:25:57 DEBUG --- http response --- 03:25:57 DEBUG --- http response --- 03:25:57 DEBUG { 03:25:57 DEBUG ----- output ----- 03:25:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:25:57 DEBUG --- http response --- 03:25:57 DEBUG { 03:25:57 DEBUG "status": "success", 03:25:57 DEBUG --- http response --- 03:25:57 DEBUG ----- output ----- 03:25:57 DEBUG ----- output ----- 03:25:57 DEBUG ----- output ----- 03:25:57 DEBUG ----- output ----- 03:25:57 DEBUG ----- output ----- 03:25:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:25:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:25:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:25:57 DEBUG "status": "success", 03:25:57 DEBUG { 03:25:57 DEBUG --- http response --- 03:25:57 DEBUG ----- output ----- 03:25:57 DEBUG { 03:25:57 DEBUG "data": { 03:25:57 DEBUG { 03:25:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:25:57 DEBUG { 03:25:57 DEBUG { 03:25:57 DEBUG "status": "success", 03:25:57 DEBUG { 03:25:57 DEBUG ----- output ----- 03:25:57 DEBUG ----- output ----- 03:25:57 DEBUG ----- output ----- 03:25:57 DEBUG "data": { 03:25:57 DEBUG "status": "success", 03:25:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:25:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:25:57 DEBUG "status": "success", 03:25:57 DEBUG "resultType": "vector", 03:25:57 DEBUG "status": "success", 03:25:57 DEBUG ----- output ----- 03:25:57 DEBUG "status": "success", 03:25:57 DEBUG { 03:25:57 DEBUG "data": { 03:25:57 DEBUG "status": "success", 03:25:57 DEBUG { 03:25:57 DEBUG { 03:25:57 DEBUG { 03:25:57 DEBUG "resultType": "vector", 03:25:57 DEBUG "data": { 03:25:57 DEBUG ----- output ----- 03:25:57 DEBUG ----- output ----- 03:25:57 DEBUG "data": { 03:25:57 DEBUG "result": [ 03:25:57 DEBUG "data": { 03:25:57 DEBUG { 03:25:57 DEBUG "data": { 03:25:57 DEBUG "status": "success", 03:25:57 DEBUG "resultType": "vector", 03:25:57 DEBUG "data": { 03:25:57 DEBUG "status": "success", 03:25:57 DEBUG "status": "success", 03:25:57 DEBUG "status": "success", 03:25:57 DEBUG "result": [ 03:25:57 DEBUG "resultType": "vector", 03:25:57 DEBUG { 03:25:57 DEBUG { 03:25:57 DEBUG "resultType": "vector", 03:25:57 DEBUG { 03:25:57 DEBUG "resultType": "vector", 03:25:57 DEBUG "status": "success", 03:25:57 DEBUG "resultType": "vector", 03:25:57 DEBUG "data": { 03:25:57 DEBUG "result": [ 03:25:57 DEBUG "resultType": "vector", 03:25:57 DEBUG "data": { 03:25:57 DEBUG "data": { 03:25:57 DEBUG "data": { 03:25:57 DEBUG { 03:25:57 DEBUG "result": [ 03:25:57 DEBUG "status": "success", 03:25:57 DEBUG "status": "success", 03:25:57 DEBUG "result": [ 03:25:57 DEBUG "metric": { 03:25:57 DEBUG "result": [ 03:25:57 DEBUG "data": { 03:25:57 DEBUG "result": [ 03:25:57 DEBUG "resultType": "vector", 03:25:57 DEBUG { 03:25:57 DEBUG "result": [ 03:25:57 DEBUG { 03:25:57 DEBUG "resultType": "vector", 03:25:57 DEBUG "resultType": "vector", 03:25:57 DEBUG "metric": { 03:25:57 DEBUG { 03:25:57 DEBUG "data": { 03:25:57 DEBUG "data": { 03:25:57 DEBUG { 03:25:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:25:57 DEBUG { 03:25:57 DEBUG "resultType": "vector", 03:25:57 DEBUG { 03:25:57 DEBUG "result": [ 03:25:57 DEBUG "metric": { 03:25:57 DEBUG "resultType": "vector", 03:25:57 DEBUG "metric": { 03:25:57 DEBUG "result": [ 03:25:57 DEBUG "result": [ 03:25:57 DEBUG "pod": "ds-cts-0" 03:25:57 DEBUG "metric": { 03:25:57 DEBUG "resultType": "vector", 03:25:57 DEBUG "resultType": "vector", 03:25:57 DEBUG "metric": { 03:25:57 DEBUG }, 03:25:57 DEBUG "metric": { 03:25:57 DEBUG "result": [ 03:25:57 DEBUG "metric": { 03:25:57 DEBUG { 03:25:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:25:57 DEBUG "result": [ 03:25:57 DEBUG "instance": "10.142.0.107:9100" 03:25:57 DEBUG { 03:25:57 DEBUG { 03:25:57 DEBUG }, 03:25:57 DEBUG "pod": "am-767f6b67b5-9zw22" 03:25:57 DEBUG "result": [ 03:25:57 DEBUG "result": [ 03:25:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:25:57 DEBUG "value": [ 03:25:57 DEBUG "instance": "10.142.0.107:9100" 03:25:57 DEBUG { 03:25:57 DEBUG "instance": "10.142.0.107:9100" 03:25:57 DEBUG "metric": { 03:25:57 DEBUG }, 03:25:57 DEBUG { 03:25:57 DEBUG }, 03:25:57 DEBUG "metric": { 03:25:57 DEBUG "metric": { 03:25:57 DEBUG "value": [ 03:25:57 DEBUG }, 03:25:57 DEBUG { 03:25:57 DEBUG { 03:25:57 DEBUG }, 03:25:57 DEBUG 03:25:57 DEBUG }, 03:25:57 DEBUG "metric": { 03:25:57 DEBUG }, 03:25:57 DEBUG "pod": "am-767f6b67b5-9zw22" 03:25:57 DEBUG "value": [ 03:25:57 DEBUG "metric": { 03:25:57 DEBUG "value": [ 03:25:57 DEBUG "pod": "ds-idrepo-0" 03:25:57 DEBUG "pod": "ds-cts-0" 03:25:57 DEBUG 03:25:57 DEBUG "value": [ 03:25:57 DEBUG "metric": { 03:25:57 DEBUG "metric": { 03:25:57 DEBUG "value": [ 03:25:57 DEBUG 03:25:57 DEBUG "value": [ 03:25:57 DEBUG "pod": "am-767f6b67b5-9zw22" 03:25:57 DEBUG "value": [ 03:25:57 DEBUG }, 03:25:57 DEBUG 03:25:57 DEBUG "pod": "ds-idrepo-0" 03:25:57 DEBUG 03:25:57 DEBUG }, 03:25:57 DEBUG }, 03:25:57 DEBUG "value": [ 03:25:57 DEBUG 03:25:57 DEBUG "pod": "ds-idrepo-0" 03:25:57 DEBUG "instance": "10.142.0.107:9100" 03:25:57 DEBUG 03:25:57 DEBUG 03:25:57 DEBUG 03:25:57 DEBUG }, 03:25:57 DEBUG 03:25:57 DEBUG "value": [ 03:25:57 DEBUG 03:25:57 DEBUG }, 03:25:57 DEBUG 03:25:57 DEBUG "value": [ 03:25:57 DEBUG 03:25:57 DEBUG 03:25:57 DEBUG 03:25:57 DEBUG }, 03:25:57 DEBUG }, 03:25:57 DEBUG 03:25:57 DEBUG [...] 03:25:57 DEBUG 03:25:57 DEBUG "value": [ 03:25:57 DEBUG 03:25:57 DEBUG 03:25:57 DEBUG 03:25:57 DEBUG "value": [ 03:25:57 DEBUG 03:25:57 DEBUG 03:25:57 DEBUG 03:25:57 DEBUG 03:25:57 DEBUG 03:25:57 DEBUG "value": [ 03:25:57 DEBUG "value": [ 03:25:57 DEBUG 03:25:57 DEBUG 03:25:57 DEBUG 03:25:57 DEBUG 03:25:57 DEBUG 03:25:57 DEBUG 03:25:57 DEBUG [...] 03:25:57 DEBUG 03:25:57 DEBUG [...] 03:25:57 DEBUG 03:25:57 DEBUG [...] 03:25:57 DEBUG 03:25:57 DEBUG [...] 03:25:57 DEBUG 03:25:57 DEBUG 03:25:57 DEBUG [...] 03:25:57 DEBUG 03:25:57 DEBUG [...] 03:25:57 DEBUG 03:25:57 DEBUG [...] 03:25:57 DEBUG 03:25:57 DEBUG 03:25:57 DEBUG 03:25:57 DEBUG 03:25:57 DEBUG 03:25:57 DEBUG 03:25:57 DEBUG [...] 03:25:57 DEBUG 03:25:57 DEBUG 03:25:57 DEBUG 03:25:57 DEBUG 03:25:57 DEBUG 03:25:57 DEBUG 03:25:57 DEBUG 03:25:57 DEBUG 03:25:57 DEBUG [...] 03:25:57 DEBUG 03:25:57 DEBUG 03:25:57 DEBUG 03:25:57 DEBUG [...] 03:25:57 DEBUG 03:25:57 DEBUG 03:25:57 DEBUG 03:25:57 DEBUG 03:25:57 DEBUG [...] 03:25:57 DEBUG 03:25:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:25:57 DEBUG 03:25:57 DEBUG [...] 03:25:57 DEBUG 03:25:57 DEBUG 03:25:57 DEBUG 03:25:57 DEBUG [...] 03:25:57 DEBUG 03:25:57 DEBUG 03:25:57 DEBUG 03:25:57 DEBUG 03:25:57 DEBUG 03:25:57 DEBUG 03:25:57 DEBUG 03:25:57 DEBUG 03:25:57 DEBUG }, 03:25:57 DEBUG 03:25:57 DEBUG 03:25:57 DEBUG 03:25:57 DEBUG 03:25:57 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:25:57 DEBUG 03:25:57 DEBUG "instance": "10.142.0.43:9100" 03:25:57 DEBUG 03:25:57 DEBUG "pod": "ds-cts-2" 03:25:57 DEBUG 03:25:57 DEBUG "pod": "am-767f6b67b5-vbltz" 03:25:57 DEBUG [...] 03:25:57 DEBUG 03:25:57 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:25:57 DEBUG "value": [ 03:25:57 DEBUG "instance": "10.142.0.43:9100" 03:25:57 DEBUG 03:25:57 DEBUG "instance": "10.142.0.43:9100" 03:25:57 DEBUG 03:25:57 DEBUG }, 03:25:57 DEBUG 03:25:57 DEBUG }, 03:25:57 DEBUG 03:25:57 DEBUG }, 03:25:57 DEBUG "pod": "ds-cts-2" 03:25:57 DEBUG }, 03:25:57 DEBUG 03:25:57 DEBUG 03:25:57 DEBUG }, 03:25:57 DEBUG 1713666357, 03:25:57 DEBUG }, 03:25:57 DEBUG 03:25:57 DEBUG }, 03:25:57 DEBUG "pod": "am-767f6b67b5-vbltz" 03:25:57 DEBUG "value": [ 03:25:57 DEBUG 03:25:57 DEBUG "value": [ 03:25:57 DEBUG "pod": "ds-idrepo-2" 03:25:57 DEBUG "value": [ 03:25:57 DEBUG }, 03:25:57 DEBUG "value": [ 03:25:57 DEBUG 03:25:57 DEBUG "pod": "ds-idrepo-2" 03:25:57 DEBUG "value": [ 03:25:57 DEBUG "1049993216" 03:25:57 DEBUG "value": [ 03:25:57 DEBUG "pod": "am-767f6b67b5-vbltz" 03:25:57 DEBUG "value": [ 03:25:57 DEBUG }, 03:25:57 DEBUG 1713666357, 03:25:57 DEBUG "pod": "ds-cts-2" 03:25:57 DEBUG 1713666357, 03:25:57 DEBUG }, 03:25:57 DEBUG 1713666357, 03:25:57 DEBUG "value": [ 03:25:57 DEBUG 1713666357, 03:25:57 DEBUG 03:25:57 DEBUG }, 03:25:57 DEBUG 1713666357, 03:25:57 DEBUG ] 03:25:57 DEBUG 1713666357, 03:25:57 DEBUG }, 03:25:57 DEBUG 1713666357, 03:25:57 DEBUG "value": [ 03:25:57 DEBUG "0" 03:25:57 DEBUG }, 03:25:57 DEBUG "0.08466666666666545" 03:25:57 DEBUG "value": [ 03:25:57 DEBUG "0" 03:25:57 DEBUG 1713666357, 03:25:57 DEBUG "152.42950000000002" 03:25:57 DEBUG "instance": "10.142.0.43:9100" 03:25:57 DEBUG "value": [ 03:25:57 DEBUG "0" 03:25:57 DEBUG } 03:25:57 DEBUG "100161436.93333334" 03:25:57 DEBUG "value": [ 03:25:57 DEBUG "100158285.13333333" 03:25:57 DEBUG 1713666357, 03:25:57 DEBUG ] 03:25:57 DEBUG "value": [ 03:25:57 DEBUG ] 03:25:57 DEBUG 1713666357, 03:25:57 DEBUG "0" 03:25:57 DEBUG "29.233333333333334" 03:25:57 DEBUG ] 03:25:57 DEBUG }, 03:25:57 DEBUG 1713666357, 03:25:57 DEBUG ] 03:25:57 DEBUG ] 03:25:57 DEBUG ] 03:25:57 DEBUG 1713666357, 03:25:57 DEBUG ] 03:25:57 DEBUG } 03:25:57 DEBUG } 03:25:57 DEBUG 1713666357, 03:25:57 DEBUG } 03:25:57 DEBUG ] 03:25:57 DEBUG ] 03:25:57 DEBUG ] 03:25:57 DEBUG } 03:25:57 DEBUG "value": [ 03:25:57 DEBUG "0" 03:25:57 DEBUG } 03:25:57 DEBUG } 03:25:57 DEBUG } 03:25:57 DEBUG "3324.1666666666665" 03:25:57 DEBUG ] 03:25:57 DEBUG "3815.2" 03:25:57 DEBUG ] 03:25:57 DEBUG "0" 03:25:57 DEBUG ] 03:25:57 DEBUG } 03:25:57 DEBUG } 03:25:57 DEBUG } 03:25:57 DEBUG ] 03:25:57 DEBUG 1713666357, 03:25:57 DEBUG "4427132447744" 03:25:57 DEBUG ] 03:25:57 DEBUG } 03:25:57 DEBUG ] 03:25:57 DEBUG ] 03:25:57 DEBUG } 03:25:57 DEBUG ] 03:25:57 DEBUG } 03:25:57 DEBUG ] 03:25:57 DEBUG } 03:25:57 DEBUG ] 03:25:57 DEBUG ] 03:25:57 DEBUG ] 03:25:57 DEBUG } 03:25:57 DEBUG ] 03:25:57 DEBUG } 03:25:57 DEBUG } 03:25:57 DEBUG } 03:25:57 DEBUG } 03:25:57 DEBUG } 03:25:57 DEBUG } 03:25:57 DEBUG } 03:25:57 DEBUG } 03:25:57 DEBUG } 03:25:57 DEBUG } 03:25:57 DEBUG } 03:25:57 DEBUG } 03:25:57 DEBUG } 03:25:57 DEBUG ] 03:25:57 DEBUG ] 03:25:57 DEBUG } 03:25:57 DEBUG } 03:25:57 DEBUG ] 03:25:57 DEBUG ] 03:25:57 DEBUG } 03:25:57 DEBUG } 03:25:57 DEBUG } 03:25:57 DEBUG } 03:25:57 DEBUG } 03:25:57 DEBUG ] 03:25:57 DEBUG } 03:25:57 DEBUG } 03:25:57 DEBUG } 03:25:57 DEBUG } 03:25:57 DEBUG } 03:25:57 DEBUG ] 03:25:57 DEBUG } 03:25:57 DEBUG } 03:25:57 DEBUG } 03:25:57 DEBUG } 03:26:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666413" 03:26:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713666413" 03:26:53 ERROR [http_cmd]: ERROR 03:26:53 ERROR --- status code --- 03:26:53 ERROR http status code is 400 (expected [200, 401, 403, 404]) 03:26:53 ERROR --- http response --- 03:26:53 INFO [http_cmd]: http status code OK 03:26:53 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 03:26:53 DEBUG --- status code --- 03:26:53 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 03:26:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:26:53 DEBUG --- http response --- 03:26:53 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 03:26:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666415" 03:26:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666415" 03:26:55 INFO [http_cmd]: http status code OK 03:26:55 DEBUG --- status code --- 03:26:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:26:55 DEBUG --- http response --- 03:26:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:26:55 DEBUG ----- output ----- 03:26:55 DEBUG { 03:26:55 DEBUG "status": "success", 03:26:55 DEBUG "data": { 03:26:55 DEBUG "resultType": "vector", 03:26:55 DEBUG "result": [ 03:26:55 DEBUG { 03:26:55 DEBUG "metric": { 03:26:55 DEBUG "instance": "10.142.0.107:9100" 03:26:55 DEBUG }, 03:26:55 DEBUG "value": [ 03:26:55 DEBUG 03:26:55 DEBUG 03:26:55 DEBUG 03:26:55 DEBUG [...] 03:26:55 DEBUG 03:26:55 DEBUG 03:26:55 DEBUG 03:26:55 DEBUG "instance": "10.142.0.43:9100" 03:26:55 DEBUG }, 03:26:55 DEBUG "value": [ 03:26:55 DEBUG 1713666415, 03:26:55 DEBUG "0.39419999999995525" 03:26:55 DEBUG ] 03:26:55 DEBUG } 03:26:55 DEBUG ] 03:26:55 DEBUG } 03:26:55 DEBUG } 03:26:55 INFO [http_cmd]: http status code OK 03:26:55 DEBUG --- status code --- 03:26:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:26:55 DEBUG --- http response --- 03:26:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:26:55 DEBUG ----- output ----- 03:26:55 DEBUG { 03:26:55 DEBUG "status": "success", 03:26:55 DEBUG "data": { 03:26:55 DEBUG "resultType": "vector", 03:26:55 DEBUG "result": [ 03:26:55 DEBUG { 03:26:55 DEBUG "metric": { 03:26:55 DEBUG "pod": "ds-cts-0" 03:26:55 DEBUG }, 03:26:55 DEBUG "value": [ 03:26:55 DEBUG 03:26:55 DEBUG 03:26:55 DEBUG 03:26:55 DEBUG [...] 03:26:55 DEBUG 03:26:55 DEBUG 03:26:55 DEBUG 03:26:55 DEBUG "pod": "ds-cts-2" 03:26:55 DEBUG }, 03:26:55 DEBUG "value": [ 03:26:55 DEBUG 1713666415, 03:26:55 DEBUG "0.5847000000000018" 03:26:55 DEBUG ] 03:26:55 DEBUG } 03:26:55 DEBUG ] 03:26:55 DEBUG } 03:26:55 DEBUG } 03:26:55 INFO 03:26:55 INFO [loop_until]: kubectl --namespace=xlou top pods 03:26:55 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:26:55 INFO [loop_until]: OK (rc = 0) 03:26:55 DEBUG --- stdout --- 03:26:55 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 22418m 8798Mi am-767f6b67b5-qcqz2 22219m 9140Mi am-767f6b67b5-vbltz 25011m 8850Mi ds-cts-0 20429m 12470Mi ds-cts-1 19213m 12494Mi ds-cts-2 20194m 12570Mi ds-idrepo-0 9396m 3918Mi ds-idrepo-1 9314m 3962Mi ds-idrepo-2 9554m 3915Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1179Mi idm-b85f9d978-wnhxb 3m 1239Mi lodemon-54c87f8cbb-jzrjs 18m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-68c5d5b884-p97w5 11721m 1000Mi 03:26:55 DEBUG --- stderr --- 03:26:55 DEBUG 03:26:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666415" 03:26:55 INFO [http_cmd]: http status code OK 03:26:55 DEBUG --- status code --- 03:26:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:26:55 DEBUG --- http response --- 03:26:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:26:55 DEBUG ----- output ----- 03:26:55 DEBUG { 03:26:55 DEBUG "status": "success", 03:26:55 DEBUG "data": { 03:26:55 DEBUG "resultType": "vector", 03:26:55 DEBUG "result": [ 03:26:55 DEBUG { 03:26:55 DEBUG "metric": { 03:26:55 DEBUG "pod": "am-767f6b67b5-9zw22" 03:26:55 DEBUG }, 03:26:55 DEBUG "value": [ 03:26:55 DEBUG 03:26:55 DEBUG 03:26:55 DEBUG 03:26:55 DEBUG [...] 03:26:55 DEBUG 03:26:55 DEBUG 03:26:55 DEBUG 03:26:55 DEBUG "pod": "am-767f6b67b5-vbltz" 03:26:55 DEBUG }, 03:26:55 DEBUG "value": [ 03:26:55 DEBUG 1713666415, 03:26:55 DEBUG "3343.866666666667" 03:26:55 DEBUG ] 03:26:55 DEBUG } 03:26:55 DEBUG ] 03:26:55 DEBUG } 03:26:55 DEBUG } 03:26:56 INFO 03:26:56 INFO [loop_until]: kubectl --namespace=xlou top node 03:26:56 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:26:56 INFO [loop_until]: OK (rc = 0) 03:26:56 DEBUG --- stdout --- 03:26:56 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 63m 0% 6283Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 9923m 33% 7753Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 21495m 71% 11614Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 87m 0% 3359Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 19905m 66% 15793Mi 14% gke-xlou-cdm-default-pool-1b0769c8-wpwl 160m 0% 5268Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 22421m 75% 12083Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 19784m 66% 3181Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 24703m 82% 11116Mi 9% gke-xlou-cdm-ds-0f601751-j488 10396m 34% 7348Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 19028m 63% 17623Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 10144m 33% 8018Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20391m 68% 15297Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 12026m 75% 11319Mi 19% 03:26:56 DEBUG --- stderr --- 03:26:56 DEBUG 03:26:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666416" 03:26:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666416" 03:26:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666416" 03:26:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666416" 03:26:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666416" 03:26:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713666416" 03:26:56 INFO [http_cmd]: http status code OK 03:26:56 INFO [http_cmd]: http status code OK 03:26:56 DEBUG --- status code --- 03:26:56 INFO [http_cmd]: http status code OK 03:26:56 DEBUG --- status code --- 03:26:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:26:56 DEBUG --- status code --- 03:26:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:26:56 DEBUG --- http response --- 03:26:56 INFO [http_cmd]: http status code OK 03:26:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:26:56 DEBUG --- http response --- 03:26:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 03:26:56 DEBUG --- status code --- 03:26:56 INFO [http_cmd]: http status code OK 03:26:56 DEBUG --- http response --- 03:26:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:26:56 DEBUG ----- output ----- 03:26:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:26:56 DEBUG --- status code --- 03:26:56 INFO [http_cmd]: http status code OK 03:26:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 03:26:56 DEBUG ----- output ----- 03:26:56 DEBUG { 03:26:56 DEBUG --- http response --- 03:26:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:26:56 DEBUG --- http response --- 03:26:56 DEBUG --- status code --- 03:26:56 DEBUG { 03:26:56 DEBUG "status": "success", 03:26:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:26:56 DEBUG ----- output ----- 03:26:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:26:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:26:56 DEBUG "status": "success", 03:26:56 DEBUG "data": { 03:26:56 DEBUG ----- output ----- 03:26:56 DEBUG { 03:26:56 DEBUG --- http response --- 03:26:56 DEBUG ----- output ----- 03:26:56 DEBUG "data": { 03:26:56 DEBUG "resultType": "vector", 03:26:56 DEBUG "result": [ 03:26:56 DEBUG "status": "success", 03:26:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:26:56 DEBUG { 03:26:56 DEBUG "resultType": "vector", 03:26:56 DEBUG { 03:26:56 DEBUG { 03:26:56 DEBUG "data": { 03:26:56 DEBUG ----- output ----- 03:26:56 DEBUG "status": "success", 03:26:56 DEBUG "result": [ 03:26:56 DEBUG "metric": { 03:26:56 DEBUG "status": "success", 03:26:56 DEBUG "data": { 03:26:56 DEBUG { 03:26:56 DEBUG "data": { 03:26:56 DEBUG { 03:26:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:26:56 DEBUG "resultType": "vector", 03:26:56 DEBUG "resultType": "vector", 03:26:56 DEBUG "status": "success", 03:26:56 DEBUG "resultType": "vector", 03:26:56 DEBUG "metric": { 03:26:56 DEBUG }, 03:26:56 DEBUG "result": [ 03:26:56 DEBUG { 03:26:56 DEBUG "data": { 03:26:56 DEBUG "resultType": "vector", 03:26:56 DEBUG "pod": "ds-idrepo-0" 03:26:56 DEBUG "value": [ 03:26:56 DEBUG "metric": { 03:26:56 DEBUG "result": [ 03:26:56 DEBUG "result": [ 03:26:56 DEBUG }, 03:26:56 DEBUG 03:26:56 DEBUG "result": [ 03:26:56 DEBUG "pod": "am-767f6b67b5-9zw22" 03:26:56 DEBUG }, 03:26:56 DEBUG { 03:26:56 DEBUG "value": [ 03:26:56 DEBUG 03:26:56 DEBUG { 03:26:56 DEBUG "value": [ 03:26:56 DEBUG { 03:26:56 DEBUG "metric": { 03:26:56 DEBUG 03:26:56 DEBUG 03:26:56 DEBUG "metric": { 03:26:56 DEBUG 03:26:56 DEBUG 03:26:56 DEBUG "pod": "am-767f6b67b5-9zw22" 03:26:56 DEBUG 03:26:56 DEBUG [...] 03:26:56 DEBUG 03:26:56 DEBUG 03:26:56 DEBUG }, 03:26:56 DEBUG "value": [ 03:26:56 DEBUG 03:26:56 DEBUG 03:26:56 DEBUG 03:26:56 DEBUG 03:26:56 DEBUG 03:26:56 DEBUG "metric": { 03:26:56 DEBUG 03:26:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:26:56 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:26:56 DEBUG [...] 03:26:56 DEBUG 03:26:56 DEBUG 03:26:56 DEBUG [...] 03:26:56 DEBUG }, 03:26:56 DEBUG }, 03:26:56 DEBUG [...] 03:26:56 DEBUG 03:26:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:26:56 DEBUG 03:26:56 DEBUG "value": [ 03:26:56 DEBUG "value": [ 03:26:56 DEBUG 03:26:56 DEBUG "pod": "am-767f6b67b5-vbltz" 03:26:56 DEBUG }, 03:26:56 DEBUG 03:26:56 DEBUG 03:26:56 DEBUG 1713666416, 03:26:56 DEBUG 03:26:56 DEBUG }, 03:26:56 DEBUG "value": [ 03:26:56 DEBUG 03:26:56 DEBUG 03:26:56 DEBUG "39175.31389309894" 03:26:56 DEBUG 03:26:56 DEBUG "pod": "am-767f6b67b5-vbltz" 03:26:56 DEBUG }, 03:26:56 DEBUG "value": [ 03:26:56 DEBUG 03:26:56 DEBUG ] 03:26:56 DEBUG "pod": "ds-idrepo-2" 03:26:56 DEBUG "value": [ 03:26:56 DEBUG 1713666416, 03:26:56 DEBUG [...] 03:26:56 DEBUG } 03:26:56 DEBUG 1713666416, 03:26:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666416" 03:26:56 DEBUG }, 03:26:56 DEBUG "3324.733333333333" 03:26:56 DEBUG 03:26:56 DEBUG 03:26:56 DEBUG ] 03:26:56 DEBUG "0" 03:26:56 DEBUG "value": [ 03:26:56 DEBUG ] 03:26:56 DEBUG 03:26:56 DEBUG 03:26:56 DEBUG } 03:26:56 DEBUG ] 03:26:56 DEBUG 1713666416, 03:26:56 DEBUG } 03:26:56 DEBUG [...] 03:26:56 DEBUG 03:26:56 DEBUG } 03:26:56 DEBUG } 03:26:56 DEBUG "0" 03:26:56 DEBUG ] 03:26:56 DEBUG 03:26:56 DEBUG 03:26:56 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:26:56 DEBUG ] 03:26:56 DEBUG } 03:26:56 DEBUG 03:26:56 DEBUG }, 03:26:56 DEBUG } 03:26:56 DEBUG ] 03:26:56 DEBUG } 03:26:56 DEBUG 03:26:56 DEBUG "value": [ 03:26:56 DEBUG ] 03:26:56 DEBUG } 03:26:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:26:56 DEBUG 1713666416, 03:26:56 DEBUG } 03:26:56 DEBUG } 03:26:56 DEBUG }, 03:26:56 DEBUG "13.994398871976042" 03:26:56 DEBUG } 03:26:56 DEBUG "value": [ 03:26:56 DEBUG ] 03:26:56 DEBUG } 03:26:56 DEBUG ] 03:26:56 DEBUG } 03:26:56 DEBUG 1713666416, 03:26:56 DEBUG } 03:26:56 DEBUG "11.814853200837863" 03:26:56 DEBUG ] 03:26:56 DEBUG } 03:26:56 DEBUG ] 03:26:56 DEBUG } 03:26:56 DEBUG } 03:26:56 INFO [http_cmd]: http status code OK 03:26:56 DEBUG --- status code --- 03:26:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:26:56 DEBUG --- http response --- 03:26:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:26:56 DEBUG ----- output ----- 03:26:56 DEBUG { 03:26:56 DEBUG "status": "success", 03:26:56 DEBUG "data": { 03:26:56 DEBUG "resultType": "vector", 03:26:56 DEBUG "result": [ 03:26:56 DEBUG { 03:26:56 DEBUG "metric": { 03:26:56 DEBUG "pod": "am-767f6b67b5-9zw22" 03:26:56 DEBUG }, 03:26:56 DEBUG "value": [ 03:26:56 DEBUG 03:26:56 DEBUG 03:26:56 DEBUG 03:26:56 DEBUG [...] 03:26:56 DEBUG 03:26:56 DEBUG 03:26:56 DEBUG 03:26:56 DEBUG "pod": "am-767f6b67b5-vbltz" 03:26:56 DEBUG }, 03:26:56 DEBUG "value": [ 03:26:56 DEBUG 1713666416, 03:26:56 DEBUG "600285.5" 03:26:56 DEBUG ] 03:26:56 DEBUG } 03:26:56 DEBUG ] 03:26:56 DEBUG } 03:26:56 DEBUG } 03:26:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713666417" 03:26:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666417" 03:26:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666417" 03:26:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666417" 03:26:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666417" 03:26:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666417" 03:26:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666417" 03:26:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666417" 03:26:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666417" 03:26:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713666417" 03:26:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666417" 03:26:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666417" 03:26:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666417" 03:26:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713666417" 03:26:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666417" 03:26:57 INFO [http_cmd]: http status code OK 03:26:57 INFO [http_cmd]: http status code OK 03:26:57 INFO [http_cmd]: http status code OK 03:26:57 INFO [http_cmd]: http status code OK 03:26:57 INFO [http_cmd]: http status code OK 03:26:57 INFO [http_cmd]: http status code OK 03:26:57 DEBUG --- status code --- 03:26:57 INFO [http_cmd]: http status code OK 03:26:57 INFO [http_cmd]: http status code OK 03:26:57 DEBUG --- status code --- 03:26:57 INFO [http_cmd]: http status code OK 03:26:57 DEBUG --- status code --- 03:26:57 INFO [http_cmd]: http status code OK 03:26:57 DEBUG --- status code --- 03:26:57 INFO [http_cmd]: http status code OK 03:26:57 INFO [http_cmd]: http status code OK 03:26:57 INFO [http_cmd]: http status code OK 03:26:57 INFO [http_cmd]: http status code OK 03:26:57 DEBUG --- status code --- 03:26:57 DEBUG --- status code --- 03:26:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:26:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:26:57 DEBUG --- status code --- 03:26:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:26:57 INFO [http_cmd]: http status code OK 03:26:57 DEBUG --- status code --- 03:26:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:26:57 DEBUG --- status code --- 03:26:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:26:57 DEBUG --- status code --- 03:26:57 DEBUG --- status code --- 03:26:57 DEBUG --- status code --- 03:26:57 DEBUG --- status code --- 03:26:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:26:57 DEBUG --- status code --- 03:26:57 DEBUG --- http response --- 03:26:57 DEBUG --- http response --- 03:26:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:26:57 DEBUG --- http response --- 03:26:57 DEBUG --- status code --- 03:26:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:26:57 DEBUG --- http response --- 03:26:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:26:57 DEBUG --- http response --- 03:26:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:26:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:26:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:26:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:26:57 DEBUG --- http response --- 03:26:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:26:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:26:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:26:57 DEBUG --- http response --- 03:26:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:26:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:26:57 DEBUG --- http response --- 03:26:57 DEBUG --- http response --- 03:26:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:26:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:26:57 DEBUG --- http response --- 03:26:57 DEBUG --- http response --- 03:26:57 DEBUG --- http response --- 03:26:57 DEBUG --- http response --- 03:26:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 03:26:57 DEBUG --- http response --- 03:26:57 DEBUG ----- output ----- 03:26:57 DEBUG ----- output ----- 03:26:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:26:57 DEBUG --- http response --- 03:26:57 DEBUG ----- output ----- 03:26:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:26:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:26:57 DEBUG ----- output ----- 03:26:57 DEBUG ----- output ----- 03:26:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:26:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:26:57 DEBUG ----- output ----- 03:26:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:26:57 DEBUG ----- output ----- 03:26:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:26:57 DEBUG { 03:26:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 03:26:57 DEBUG ----- output ----- 03:26:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:26:57 DEBUG { 03:26:57 DEBUG ----- output ----- 03:26:57 DEBUG ----- output ----- 03:26:57 DEBUG { 03:26:57 DEBUG { 03:26:57 DEBUG ----- output ----- 03:26:57 DEBUG { 03:26:57 DEBUG ----- output ----- 03:26:57 DEBUG { 03:26:57 DEBUG ----- output ----- 03:26:57 DEBUG { 03:26:57 DEBUG "status": "success", 03:26:57 DEBUG ----- output ----- 03:26:57 DEBUG { 03:26:57 DEBUG ----- output ----- 03:26:57 DEBUG "status": "success", 03:26:57 DEBUG { 03:26:57 DEBUG { 03:26:57 DEBUG "status": "success", 03:26:57 DEBUG "status": "success", 03:26:57 DEBUG { 03:26:57 DEBUG "status": "success", 03:26:57 DEBUG { 03:26:57 DEBUG "status": "success", 03:26:57 DEBUG { 03:26:57 DEBUG "status": "success", 03:26:57 DEBUG "data": { 03:26:57 DEBUG { 03:26:57 DEBUG "status": "success", 03:26:57 DEBUG { 03:26:57 DEBUG "data": { 03:26:57 DEBUG "status": "success", 03:26:57 DEBUG "status": "success", 03:26:57 DEBUG "data": { 03:26:57 DEBUG "data": { 03:26:57 DEBUG "status": "success", 03:26:57 DEBUG "data": { 03:26:57 DEBUG "status": "success", 03:26:57 DEBUG "data": { 03:26:57 DEBUG "status": "success", 03:26:57 DEBUG "data": { 03:26:57 DEBUG "resultType": "vector", 03:26:57 DEBUG "status": "success", 03:26:57 DEBUG "data": { 03:26:57 DEBUG "status": "success", 03:26:57 DEBUG "resultType": "vector", 03:26:57 DEBUG "data": { 03:26:57 DEBUG "data": { 03:26:57 DEBUG "resultType": "vector", 03:26:57 DEBUG "resultType": "vector", 03:26:57 DEBUG "data": { 03:26:57 DEBUG "resultType": "vector", 03:26:57 DEBUG "data": { 03:26:57 DEBUG "resultType": "vector", 03:26:57 DEBUG "result": [ 03:26:57 DEBUG "resultType": "vector", 03:26:57 DEBUG "result": [ 03:26:57 DEBUG "data": { 03:26:57 DEBUG "resultType": "vector", 03:26:57 DEBUG "data": { 03:26:57 DEBUG "result": [ 03:26:57 DEBUG "resultType": "vector", 03:26:57 DEBUG "result": [ 03:26:57 DEBUG "result": [ 03:26:57 DEBUG "result": [ 03:26:57 DEBUG "resultType": "vector", 03:26:57 DEBUG "result": [ 03:26:57 DEBUG "resultType": "vector", 03:26:57 DEBUG "data": { 03:26:57 DEBUG { 03:26:57 DEBUG "result": [ 03:26:57 DEBUG { 03:26:57 DEBUG "metric": { 03:26:57 DEBUG "result": [ 03:26:57 DEBUG "resultType": "vector", 03:26:57 DEBUG { 03:26:57 DEBUG { 03:26:57 DEBUG { 03:26:57 DEBUG "resultType": "vector", 03:26:57 DEBUG { 03:26:57 DEBUG "result": [ 03:26:57 DEBUG { 03:26:57 DEBUG "result": [ 03:26:57 DEBUG "resultType": "vector", 03:26:57 DEBUG "metric": { 03:26:57 DEBUG { 03:26:57 DEBUG "pod": "ds-cts-0" 03:26:57 DEBUG "resultType": "vector", 03:26:57 DEBUG { 03:26:57 DEBUG "result": [ 03:26:57 DEBUG "metric": { 03:26:57 DEBUG "metric": { 03:26:57 DEBUG "metric": { 03:26:57 DEBUG "result": [ 03:26:57 DEBUG "metric": { 03:26:57 DEBUG { 03:26:57 DEBUG "metric": { 03:26:57 DEBUG { 03:26:57 DEBUG "result": [ 03:26:57 DEBUG "pod": "ds-idrepo-0" 03:26:57 DEBUG "metric": { 03:26:57 DEBUG }, 03:26:57 DEBUG "result": [ 03:26:57 DEBUG "metric": { 03:26:57 DEBUG { 03:26:57 DEBUG "metric": { 03:26:57 DEBUG "pod": "am-767f6b67b5-9zw22" 03:26:57 DEBUG "instance": "10.142.0.107:9100" 03:26:57 DEBUG { 03:26:57 DEBUG "instance": "10.142.0.107:9100" 03:26:57 DEBUG "metric": { 03:26:57 DEBUG "pod": "am-767f6b67b5-9zw22" 03:26:57 DEBUG "metric": { 03:26:57 DEBUG { 03:26:57 DEBUG }, 03:26:57 DEBUG "instance": "10.142.0.107:9100" 03:26:57 DEBUG "value": [ 03:26:57 DEBUG { 03:26:57 DEBUG "instance": "10.142.0.107:9100" 03:26:57 DEBUG "pod": "ds-idrepo-0" 03:26:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:26:57 DEBUG }, 03:26:57 DEBUG }, 03:26:57 DEBUG "metric": { 03:26:57 DEBUG }, 03:26:57 DEBUG "pod": "ds-idrepo-0" 03:26:57 DEBUG }, 03:26:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:26:57 DEBUG "metric": { 03:26:57 DEBUG "value": [ 03:26:57 DEBUG }, 03:26:57 DEBUG 03:26:57 DEBUG "metric": { 03:26:57 DEBUG }, 03:26:57 DEBUG }, 03:26:57 DEBUG }, 03:26:57 DEBUG "value": [ 03:26:57 DEBUG "value": [ 03:26:57 DEBUG "pod": "am-767f6b67b5-9zw22" 03:26:57 DEBUG "value": [ 03:26:57 DEBUG }, 03:26:57 DEBUG "value": [ 03:26:57 DEBUG }, 03:26:57 DEBUG "pod": "ds-cts-0" 03:26:57 DEBUG 03:26:57 DEBUG "value": [ 03:26:57 DEBUG 03:26:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:26:57 DEBUG "value": [ 03:26:57 DEBUG "value": [ 03:26:57 DEBUG "value": [ 03:26:57 DEBUG 03:26:57 DEBUG 03:26:57 DEBUG }, 03:26:57 DEBUG 03:26:57 DEBUG "value": [ 03:26:57 DEBUG 03:26:57 DEBUG "value": [ 03:26:57 DEBUG }, 03:26:57 DEBUG 03:26:57 DEBUG 03:26:57 DEBUG 03:26:57 DEBUG }, 03:26:57 DEBUG 03:26:57 DEBUG 03:26:57 DEBUG 03:26:57 DEBUG 03:26:57 DEBUG 03:26:57 DEBUG "value": [ 03:26:57 DEBUG 03:26:57 DEBUG 03:26:57 DEBUG 03:26:57 DEBUG 03:26:57 DEBUG "value": [ 03:26:57 DEBUG 03:26:57 DEBUG 03:26:57 DEBUG [...] 03:26:57 DEBUG "value": [ 03:26:57 DEBUG 03:26:57 DEBUG 03:26:57 DEBUG 03:26:57 DEBUG 03:26:57 DEBUG 03:26:57 DEBUG 03:26:57 DEBUG 03:26:57 DEBUG 03:26:57 DEBUG 03:26:57 DEBUG 03:26:57 DEBUG 03:26:57 DEBUG [...] 03:26:57 DEBUG 03:26:57 DEBUG 03:26:57 DEBUG 03:26:57 DEBUG 03:26:57 DEBUG 03:26:57 DEBUG 03:26:57 DEBUG [...] 03:26:57 DEBUG [...] 03:26:57 DEBUG 03:26:57 DEBUG [...] 03:26:57 DEBUG 03:26:57 DEBUG [...] 03:26:57 DEBUG 03:26:57 DEBUG 03:26:57 DEBUG 03:26:57 DEBUG [...] 03:26:57 DEBUG 03:26:57 DEBUG 03:26:57 DEBUG [...] 03:26:57 DEBUG [...] 03:26:57 DEBUG [...] 03:26:57 DEBUG 03:26:57 DEBUG 03:26:57 DEBUG 03:26:57 DEBUG 03:26:57 DEBUG [...] 03:26:57 DEBUG 03:26:57 DEBUG [...] 03:26:57 DEBUG 03:26:57 DEBUG 03:26:57 DEBUG 03:26:57 DEBUG 03:26:57 DEBUG 03:26:57 DEBUG 03:26:57 DEBUG 03:26:57 DEBUG 03:26:57 DEBUG 03:26:57 DEBUG 03:26:57 DEBUG [...] 03:26:57 DEBUG 03:26:57 DEBUG 03:26:57 DEBUG 03:26:57 DEBUG 03:26:57 DEBUG [...] 03:26:57 DEBUG "pod": "ds-cts-2" 03:26:57 DEBUG "pod": "ds-cts-2" 03:26:57 DEBUG 03:26:57 DEBUG [...] 03:26:57 DEBUG 03:26:57 DEBUG 03:26:57 DEBUG 03:26:57 DEBUG 03:26:57 DEBUG 03:26:57 DEBUG 03:26:57 DEBUG 03:26:57 DEBUG 03:26:57 DEBUG 03:26:57 DEBUG 03:26:57 DEBUG 03:26:57 DEBUG }, 03:26:57 DEBUG }, 03:26:57 DEBUG 03:26:57 DEBUG 03:26:57 DEBUG 03:26:57 DEBUG 03:26:57 DEBUG 03:26:57 DEBUG "instance": "10.142.0.43:9100" 03:26:57 DEBUG "pod": "am-767f6b67b5-vbltz" 03:26:57 DEBUG 03:26:57 DEBUG "instance": "10.142.0.43:9100" 03:26:57 DEBUG 03:26:57 DEBUG "pod": "am-767f6b67b5-vbltz" 03:26:57 DEBUG 03:26:57 DEBUG 03:26:57 DEBUG "value": [ 03:26:57 DEBUG "value": [ 03:26:57 DEBUG 03:26:57 DEBUG 03:26:57 DEBUG "pod": "ds-idrepo-2" 03:26:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:26:57 DEBUG "instance": "10.142.0.43:9100" 03:26:57 DEBUG }, 03:26:57 DEBUG }, 03:26:57 DEBUG 03:26:57 DEBUG }, 03:26:57 DEBUG "pod": "ds-idrepo-2" 03:26:57 DEBUG }, 03:26:57 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:26:57 DEBUG 03:26:57 DEBUG 1713666417, 03:26:57 DEBUG 1713666417, 03:26:57 DEBUG 03:26:57 DEBUG "instance": "10.142.0.43:9100" 03:26:57 DEBUG }, 03:26:57 DEBUG "value": [ 03:26:57 DEBUG 1713666417, 03:26:57 DEBUG "value": [ 03:26:57 DEBUG 1713666417, 03:26:57 DEBUG "pod": "am-767f6b67b5-vbltz" 03:26:57 DEBUG }, 03:26:57 DEBUG }, 03:26:57 DEBUG "value": [ 03:26:57 DEBUG }, 03:26:57 DEBUG "pod": "ds-cts-2" 03:26:57 DEBUG "0" 03:26:57 DEBUG "29.666666666666668" 03:26:57 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:26:57 DEBUG }, 03:26:57 DEBUG }, 03:26:57 DEBUG "0" 03:26:57 DEBUG }, 03:26:57 DEBUG "100919795.86666667" 03:26:57 DEBUG "value": [ 03:26:57 DEBUG "value": [ 03:26:57 DEBUG "value": [ 03:26:57 DEBUG "value": [ 03:26:57 DEBUG 1713666417, 03:26:57 DEBUG "value": [ 03:26:57 DEBUG }, 03:26:57 DEBUG ] 03:26:57 DEBUG ] 03:26:57 DEBUG } 03:26:57 DEBUG "value": [ 03:26:57 DEBUG "value": [ 03:26:57 DEBUG ] 03:26:57 DEBUG "value": [ 03:26:57 DEBUG ] 03:26:57 DEBUG 1713666417, 03:26:57 DEBUG 1713666417, 03:26:57 DEBUG 1713666417, 03:26:57 DEBUG 1713666417, 03:26:57 DEBUG "3345.5666666666666" 03:26:57 DEBUG 1713666417, 03:26:57 DEBUG "value": [ 03:26:57 DEBUG } 03:26:57 DEBUG ] 03:26:57 DEBUG }, 03:26:57 DEBUG 1713666417, 03:26:57 DEBUG 1713666417, 03:26:57 DEBUG } 03:26:57 DEBUG 1713666417, 03:26:57 DEBUG } 03:26:57 DEBUG "0.07399999999999904" 03:26:57 DEBUG "3834.8333333333335" 03:26:57 DEBUG "154.4235" 03:26:57 DEBUG "0" 03:26:57 DEBUG ] 03:26:57 DEBUG "0" 03:26:57 DEBUG 1713666417, 03:26:57 DEBUG ] 03:26:57 DEBUG } 03:26:57 DEBUG "value": [ 03:26:57 DEBUG "4432365433856" 03:26:57 DEBUG "1145298944" 03:26:57 DEBUG ] 03:26:57 DEBUG "100921501.23333333" 03:26:57 DEBUG ] 03:26:57 DEBUG ] 03:26:57 DEBUG } 03:26:57 DEBUG ] 03:26:57 DEBUG ] 03:26:57 DEBUG } 03:26:57 DEBUG ] 03:26:57 DEBUG "0" 03:26:57 DEBUG } 03:26:57 DEBUG } 03:26:57 DEBUG 1713666417, 03:26:57 DEBUG ] 03:26:57 DEBUG ] 03:26:57 DEBUG } 03:26:57 DEBUG } 03:26:57 DEBUG } 03:26:57 DEBUG } 03:26:57 DEBUG ] 03:26:57 DEBUG } 03:26:57 DEBUG } 03:26:57 DEBUG ] 03:26:57 DEBUG } 03:26:57 DEBUG ] 03:26:57 DEBUG } 03:26:57 DEBUG "0" 03:26:57 DEBUG } 03:26:57 DEBUG } 03:26:57 DEBUG ] 03:26:57 DEBUG ] 03:26:57 DEBUG } 03:26:57 DEBUG ] 03:26:57 DEBUG ] 03:26:57 DEBUG } 03:26:57 DEBUG ] 03:26:57 DEBUG } 03:26:57 DEBUG ] 03:26:57 DEBUG ] 03:26:57 DEBUG ] 03:26:57 DEBUG } 03:26:57 DEBUG } 03:26:57 DEBUG ] 03:26:57 DEBUG } 03:26:57 DEBUG } 03:26:57 DEBUG } 03:26:57 DEBUG } 03:26:57 DEBUG ] 03:26:57 DEBUG } 03:26:57 DEBUG } 03:26:57 DEBUG } 03:26:57 DEBUG } 03:26:57 DEBUG } 03:26:57 DEBUG } 03:26:57 DEBUG } 03:26:57 DEBUG } 03:26:57 DEBUG } 03:26:57 DEBUG } 03:26:57 DEBUG ] 03:26:57 DEBUG } 03:26:57 DEBUG ] 03:26:57 DEBUG } 03:26:57 DEBUG } 03:26:57 DEBUG } 03:26:57 DEBUG } 03:26:57 DEBUG } 03:26:57 DEBUG } 03:27:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666473" 03:27:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713666473" 03:27:53 INFO [http_cmd]: http status code OK 03:27:53 ERROR [http_cmd]: ERROR 03:27:53 DEBUG --- status code --- 03:27:53 ERROR --- status code --- 03:27:53 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:27:53 ERROR http status code is 400 (expected [200, 401, 403, 404]) 03:27:53 DEBUG --- http response --- 03:27:53 ERROR --- http response --- 03:27:53 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 03:27:53 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 03:27:53 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 03:27:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666475" 03:27:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666475" 03:27:55 INFO [http_cmd]: http status code OK 03:27:55 DEBUG --- status code --- 03:27:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:27:55 DEBUG --- http response --- 03:27:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:27:55 DEBUG ----- output ----- 03:27:55 DEBUG { 03:27:55 DEBUG "status": "success", 03:27:55 DEBUG "data": { 03:27:55 DEBUG "resultType": "vector", 03:27:55 DEBUG "result": [ 03:27:55 DEBUG { 03:27:55 DEBUG "metric": { 03:27:55 DEBUG "pod": "ds-cts-0" 03:27:55 DEBUG }, 03:27:55 DEBUG "value": [ 03:27:55 DEBUG 03:27:55 DEBUG 03:27:55 DEBUG 03:27:55 DEBUG [...] 03:27:55 DEBUG 03:27:55 DEBUG 03:27:55 DEBUG 03:27:55 DEBUG "pod": "ds-cts-2" 03:27:55 DEBUG }, 03:27:55 DEBUG "value": [ 03:27:55 DEBUG 1713666475, 03:27:55 DEBUG "0.5793000000000006" 03:27:55 DEBUG ] 03:27:55 DEBUG } 03:27:55 DEBUG ] 03:27:55 DEBUG } 03:27:55 DEBUG } 03:27:55 INFO [http_cmd]: http status code OK 03:27:55 DEBUG --- status code --- 03:27:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:27:55 DEBUG --- http response --- 03:27:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:27:55 DEBUG ----- output ----- 03:27:55 DEBUG { 03:27:55 DEBUG "status": "success", 03:27:55 DEBUG "data": { 03:27:55 DEBUG "resultType": "vector", 03:27:55 DEBUG "result": [ 03:27:55 DEBUG { 03:27:55 DEBUG "metric": { 03:27:55 DEBUG "instance": "10.142.0.107:9100" 03:27:55 DEBUG }, 03:27:55 DEBUG "value": [ 03:27:55 DEBUG 03:27:55 DEBUG 03:27:55 DEBUG 03:27:55 DEBUG [...] 03:27:55 DEBUG 03:27:55 DEBUG 03:27:55 DEBUG 03:27:55 DEBUG "instance": "10.142.0.43:9100" 03:27:55 DEBUG }, 03:27:55 DEBUG "value": [ 03:27:55 DEBUG 1713666475, 03:27:55 DEBUG "0.4097666666667408" 03:27:55 DEBUG ] 03:27:55 DEBUG } 03:27:55 DEBUG ] 03:27:55 DEBUG } 03:27:55 DEBUG } 03:27:55 INFO 03:27:55 INFO [loop_until]: kubectl --namespace=xlou top pods 03:27:55 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:27:55 INFO [loop_until]: OK (rc = 0) 03:27:55 DEBUG --- stdout --- 03:27:55 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 22412m 8800Mi am-767f6b67b5-qcqz2 22142m 9141Mi am-767f6b67b5-vbltz 24449m 8824Mi ds-cts-0 20099m 12467Mi ds-cts-1 19203m 12494Mi ds-cts-2 20940m 12571Mi ds-idrepo-0 9375m 3918Mi ds-idrepo-1 9313m 3963Mi ds-idrepo-2 9798m 3915Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1179Mi idm-b85f9d978-wnhxb 3m 1239Mi lodemon-54c87f8cbb-jzrjs 32m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-68c5d5b884-p97w5 11657m 1001Mi 03:27:55 DEBUG --- stderr --- 03:27:55 DEBUG 03:27:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666475" 03:27:55 INFO [http_cmd]: http status code OK 03:27:55 DEBUG --- status code --- 03:27:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:27:55 DEBUG --- http response --- 03:27:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:27:55 DEBUG ----- output ----- 03:27:55 DEBUG { 03:27:55 DEBUG "status": "success", 03:27:55 DEBUG "data": { 03:27:55 DEBUG "resultType": "vector", 03:27:55 DEBUG "result": [ 03:27:55 DEBUG { 03:27:55 DEBUG "metric": { 03:27:55 DEBUG "pod": "am-767f6b67b5-9zw22" 03:27:55 DEBUG }, 03:27:55 DEBUG "value": [ 03:27:55 DEBUG 03:27:55 DEBUG 03:27:55 DEBUG 03:27:55 DEBUG [...] 03:27:55 DEBUG 03:27:55 DEBUG 03:27:55 DEBUG 03:27:55 DEBUG "pod": "am-767f6b67b5-vbltz" 03:27:55 DEBUG }, 03:27:55 DEBUG "value": [ 03:27:55 DEBUG 1713666475, 03:27:55 DEBUG "3330.3333333333335" 03:27:55 DEBUG ] 03:27:55 DEBUG } 03:27:55 DEBUG ] 03:27:55 DEBUG } 03:27:55 DEBUG } 03:27:56 INFO 03:27:56 INFO [loop_until]: kubectl --namespace=xlou top node 03:27:56 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:27:56 INFO [loop_until]: OK (rc = 0) 03:27:56 DEBUG --- stdout --- 03:27:56 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 66m 0% 6281Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 9900m 33% 7766Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 22179m 74% 11602Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 88m 0% 3352Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 19815m 66% 15771Mi 14% gke-xlou-cdm-default-pool-1b0769c8-wpwl 158m 0% 5269Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 22690m 76% 12074Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 19699m 65% 3185Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 24124m 80% 11094Mi 9% gke-xlou-cdm-ds-0f601751-j488 10365m 34% 7347Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 19652m 65% 17623Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 10174m 34% 8014Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20572m 68% 15289Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 12003m 75% 11316Mi 19% 03:27:56 DEBUG --- stderr --- 03:27:56 DEBUG 03:27:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666476" 03:27:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666476" 03:27:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666476" 03:27:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666476" 03:27:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666476" 03:27:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666476" 03:27:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713666476" 03:27:56 INFO [http_cmd]: http status code OK 03:27:56 DEBUG --- status code --- 03:27:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:27:56 DEBUG --- http response --- 03:27:56 INFO [http_cmd]: http status code OK 03:27:56 INFO [http_cmd]: http status code OK 03:27:56 DEBUG --- status code --- 03:27:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 03:27:56 DEBUG --- status code --- 03:27:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:27:56 INFO [http_cmd]: http status code OK 03:27:56 INFO [http_cmd]: http status code OK 03:27:56 INFO [http_cmd]: http status code OK 03:27:56 DEBUG ----- output ----- 03:27:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:27:56 DEBUG --- http response --- 03:27:56 DEBUG --- status code --- 03:27:56 DEBUG --- status code --- 03:27:56 DEBUG --- status code --- 03:27:56 DEBUG { 03:27:56 DEBUG --- http response --- 03:27:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:27:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:27:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:27:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:27:56 DEBUG "status": "success", 03:27:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:27:56 DEBUG ----- output ----- 03:27:56 DEBUG --- http response --- 03:27:56 DEBUG --- http response --- 03:27:56 DEBUG --- http response --- 03:27:56 INFO [http_cmd]: http status code OK 03:27:56 DEBUG "data": { 03:27:56 DEBUG ----- output ----- 03:27:56 DEBUG { 03:27:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 03:27:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:27:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:27:56 DEBUG --- status code --- 03:27:56 DEBUG "resultType": "vector", 03:27:56 DEBUG { 03:27:56 DEBUG "status": "success", 03:27:56 DEBUG ----- output ----- 03:27:56 DEBUG ----- output ----- 03:27:56 DEBUG ----- output ----- 03:27:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:27:56 DEBUG "result": [ 03:27:56 DEBUG "status": "success", 03:27:56 DEBUG "data": { 03:27:56 DEBUG { 03:27:56 DEBUG { 03:27:56 DEBUG { 03:27:56 DEBUG --- http response --- 03:27:56 DEBUG { 03:27:56 DEBUG "data": { 03:27:56 DEBUG "resultType": "vector", 03:27:56 DEBUG "status": "success", 03:27:56 DEBUG "status": "success", 03:27:56 DEBUG "status": "success", 03:27:56 DEBUG "metric": { 03:27:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:27:56 DEBUG "resultType": "vector", 03:27:56 DEBUG "result": [ 03:27:56 DEBUG "data": { 03:27:56 DEBUG "data": { 03:27:56 DEBUG "data": { 03:27:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:27:56 DEBUG ----- output ----- 03:27:56 DEBUG "result": [ 03:27:56 DEBUG { 03:27:56 DEBUG "resultType": "vector", 03:27:56 DEBUG "resultType": "vector", 03:27:56 DEBUG "resultType": "vector", 03:27:56 DEBUG }, 03:27:56 DEBUG { 03:27:56 DEBUG { 03:27:56 DEBUG "metric": { 03:27:56 DEBUG "result": [ 03:27:56 DEBUG "result": [ 03:27:56 DEBUG "result": [ 03:27:56 DEBUG "value": [ 03:27:56 DEBUG "status": "success", 03:27:56 DEBUG "metric": { 03:27:56 DEBUG "pod": "am-767f6b67b5-9zw22" 03:27:56 DEBUG { 03:27:56 DEBUG { 03:27:56 DEBUG { 03:27:56 DEBUG 03:27:56 DEBUG "data": { 03:27:56 DEBUG "pod": "am-767f6b67b5-9zw22" 03:27:56 DEBUG }, 03:27:56 DEBUG "metric": { 03:27:56 DEBUG "metric": { 03:27:56 DEBUG "metric": { 03:27:56 DEBUG "pod": "ds-idrepo-0" 03:27:56 DEBUG "resultType": "vector", 03:27:56 DEBUG }, 03:27:56 DEBUG "value": [ 03:27:56 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:27:56 DEBUG "pod": "am-767f6b67b5-9zw22" 03:27:56 DEBUG }, 03:27:56 DEBUG 03:27:56 DEBUG "result": [ 03:27:56 DEBUG { 03:27:56 DEBUG 03:27:56 DEBUG }, 03:27:56 DEBUG }, 03:27:56 DEBUG "value": [ 03:27:56 DEBUG 03:27:56 DEBUG "value": [ 03:27:56 DEBUG "metric": { 03:27:56 DEBUG 03:27:56 DEBUG "value": [ 03:27:56 DEBUG "value": [ 03:27:56 DEBUG 03:27:56 DEBUG [...] 03:27:56 DEBUG 03:27:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:27:56 DEBUG }, 03:27:56 DEBUG 03:27:56 DEBUG 03:27:56 DEBUG 03:27:56 DEBUG 03:27:56 DEBUG 03:27:56 DEBUG "value": [ 03:27:56 DEBUG 03:27:56 DEBUG 03:27:56 DEBUG 03:27:56 DEBUG 03:27:56 DEBUG 03:27:56 DEBUG 03:27:56 DEBUG 03:27:56 DEBUG 03:27:56 DEBUG 03:27:56 DEBUG 03:27:56 DEBUG [...] 03:27:56 DEBUG 03:27:56 DEBUG [...] 03:27:56 DEBUG [...] 03:27:56 DEBUG [...] 03:27:56 DEBUG 03:27:56 DEBUG [...] 03:27:56 DEBUG 03:27:56 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 03:27:56 DEBUG 03:27:56 DEBUG 03:27:56 DEBUG 03:27:56 DEBUG [...] 03:27:56 DEBUG 03:27:56 DEBUG 03:27:56 DEBUG }, 03:27:56 DEBUG 03:27:56 DEBUG 03:27:56 DEBUG 03:27:56 DEBUG 03:27:56 DEBUG 03:27:56 DEBUG 03:27:56 DEBUG "value": [ 03:27:56 DEBUG 03:27:56 DEBUG 03:27:56 DEBUG 03:27:56 DEBUG 03:27:56 DEBUG 03:27:56 DEBUG "pod": "ds-idrepo-2" 03:27:56 DEBUG 1713666476, 03:27:56 DEBUG "pod": "am-767f6b67b5-vbltz" 03:27:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:27:56 DEBUG "pod": "am-767f6b67b5-vbltz" 03:27:56 DEBUG 03:27:56 DEBUG "pod": "am-767f6b67b5-vbltz" 03:27:56 DEBUG }, 03:27:56 DEBUG "0" 03:27:56 DEBUG }, 03:27:56 DEBUG }, 03:27:56 DEBUG }, 03:27:56 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 03:27:56 DEBUG }, 03:27:56 DEBUG "value": [ 03:27:56 DEBUG ] 03:27:56 DEBUG "value": [ 03:27:56 DEBUG "value": [ 03:27:56 DEBUG "value": [ 03:27:56 DEBUG }, 03:27:56 DEBUG "value": [ 03:27:56 DEBUG 1713666476, 03:27:56 DEBUG } 03:27:56 DEBUG 1713666476, 03:27:56 DEBUG 1713666476, 03:27:56 DEBUG 1713666476, 03:27:56 DEBUG "value": [ 03:27:56 DEBUG 1713666476, 03:27:56 DEBUG "0" 03:27:56 DEBUG ] 03:27:56 DEBUG "3336.5333333333333" 03:27:56 DEBUG "11.776588411681697" 03:27:56 DEBUG "0" 03:27:56 DEBUG 1713666476, 03:27:56 DEBUG "600210.5" 03:27:56 DEBUG ] 03:27:56 DEBUG } 03:27:56 DEBUG ] 03:27:56 DEBUG ] 03:27:56 DEBUG ] 03:27:56 DEBUG "0.000009434731917427936" 03:27:56 DEBUG ] 03:27:56 DEBUG ] 03:27:56 DEBUG } 03:27:56 DEBUG } 03:27:56 DEBUG } 03:27:56 DEBUG } 03:27:56 DEBUG ] 03:27:56 DEBUG } 03:27:56 DEBUG } 03:27:56 DEBUG } 03:27:56 DEBUG ] 03:27:56 DEBUG ] 03:27:56 DEBUG } 03:27:56 DEBUG } 03:27:56 DEBUG ] 03:27:56 DEBUG } 03:27:56 DEBUG } 03:27:56 DEBUG ] 03:27:56 DEBUG } 03:27:56 DEBUG ] 03:27:56 DEBUG } 03:27:56 DEBUG } 03:27:56 DEBUG } 03:27:56 DEBUG } 03:27:56 DEBUG } 03:27:56 DEBUG } 03:27:56 DEBUG } 03:27:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666477" 03:27:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666477" 03:27:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666477" 03:27:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713666477" 03:27:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666477" 03:27:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713666477" 03:27:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666477" 03:27:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713666477" 03:27:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666477" 03:27:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666477" 03:27:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666477" 03:27:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666477" 03:27:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666477" 03:27:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666477" 03:27:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666477" 03:27:57 INFO [http_cmd]: http status code OK 03:27:57 INFO [http_cmd]: http status code OK 03:27:57 INFO [http_cmd]: http status code OK 03:27:57 INFO [http_cmd]: http status code OK 03:27:57 INFO [http_cmd]: http status code OK 03:27:57 INFO [http_cmd]: http status code OK 03:27:57 INFO [http_cmd]: http status code OK 03:27:57 INFO [http_cmd]: http status code OK 03:27:57 DEBUG --- status code --- 03:27:57 DEBUG --- status code --- 03:27:57 INFO [http_cmd]: http status code OK 03:27:57 DEBUG --- status code --- 03:27:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:27:57 DEBUG --- status code --- 03:27:57 DEBUG --- status code --- 03:27:57 INFO [http_cmd]: http status code OK 03:27:57 DEBUG --- status code --- 03:27:57 INFO [http_cmd]: http status code OK 03:27:57 DEBUG --- status code --- 03:27:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:27:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:27:57 DEBUG --- status code --- 03:27:57 DEBUG --- http response --- 03:27:57 DEBUG --- status code --- 03:27:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:27:57 INFO [http_cmd]: http status code OK 03:27:57 INFO [http_cmd]: http status code OK 03:27:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:27:57 INFO [http_cmd]: http status code OK 03:27:57 DEBUG --- status code --- 03:27:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:27:57 DEBUG --- status code --- 03:27:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:27:57 DEBUG --- http response --- 03:27:57 DEBUG --- http response --- 03:27:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:27:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:27:57 INFO [http_cmd]: http status code OK 03:27:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:27:57 DEBUG --- http response --- 03:27:57 DEBUG --- status code --- 03:27:57 DEBUG --- status code --- 03:27:57 DEBUG --- http response --- 03:27:57 DEBUG --- status code --- 03:27:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:27:57 DEBUG --- http response --- 03:27:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:27:57 DEBUG --- http response --- 03:27:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:27:57 DEBUG --- http response --- 03:27:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:27:57 DEBUG ----- output ----- 03:27:57 DEBUG --- status code --- 03:27:57 DEBUG --- http response --- 03:27:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:27:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:27:57 DEBUG --- http response --- 03:27:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:27:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:27:57 DEBUG --- http response --- 03:27:57 DEBUG --- http response --- 03:27:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:27:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:27:57 DEBUG ----- output ----- 03:27:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:27:57 DEBUG ----- output ----- 03:27:57 DEBUG { 03:27:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:27:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:27:57 DEBUG ----- output ----- 03:27:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:27:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:27:57 DEBUG ----- output ----- 03:27:57 DEBUG --- http response --- 03:27:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:27:57 DEBUG ----- output ----- 03:27:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:27:57 DEBUG ----- output ----- 03:27:57 DEBUG { 03:27:57 DEBUG ----- output ----- 03:27:57 DEBUG { 03:27:57 DEBUG "status": "success", 03:27:57 DEBUG --- http response --- 03:27:57 DEBUG ----- output ----- 03:27:57 DEBUG { 03:27:57 DEBUG --- http response --- 03:27:57 DEBUG ----- output ----- 03:27:57 DEBUG { 03:27:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:27:57 DEBUG ----- output ----- 03:27:57 DEBUG { 03:27:57 DEBUG ----- output ----- 03:27:57 DEBUG { 03:27:57 DEBUG "status": "success", 03:27:57 DEBUG { 03:27:57 DEBUG "status": "success", 03:27:57 DEBUG "data": { 03:27:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:27:57 DEBUG { 03:27:57 DEBUG "status": "success", 03:27:57 DEBUG { 03:27:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:27:57 DEBUG "status": "success", 03:27:57 DEBUG ----- output ----- 03:27:57 DEBUG { 03:27:57 DEBUG "status": "success", 03:27:57 DEBUG { 03:27:57 DEBUG "status": "success", 03:27:57 DEBUG "data": { 03:27:57 DEBUG "status": "success", 03:27:57 DEBUG "data": { 03:27:57 DEBUG "resultType": "vector", 03:27:57 DEBUG ----- output ----- 03:27:57 DEBUG "status": "success", 03:27:57 DEBUG "data": { 03:27:57 DEBUG "status": "success", 03:27:57 DEBUG ----- output ----- 03:27:57 DEBUG "data": { 03:27:57 DEBUG { 03:27:57 DEBUG "status": "success", 03:27:57 DEBUG "data": { 03:27:57 DEBUG "status": "success", 03:27:57 DEBUG "data": { 03:27:57 DEBUG "resultType": "vector", 03:27:57 DEBUG "data": { 03:27:57 DEBUG "resultType": "vector", 03:27:57 DEBUG "result": [ 03:27:57 DEBUG { 03:27:57 DEBUG "data": { 03:27:57 DEBUG "resultType": "vector", 03:27:57 DEBUG "data": { 03:27:57 DEBUG { 03:27:57 DEBUG "resultType": "vector", 03:27:57 DEBUG "status": "success", 03:27:57 DEBUG "data": { 03:27:57 DEBUG "resultType": "vector", 03:27:57 DEBUG "data": { 03:27:57 DEBUG "resultType": "vector", 03:27:57 DEBUG "result": [ 03:27:57 DEBUG "resultType": "vector", 03:27:57 DEBUG "result": [ 03:27:57 DEBUG { 03:27:57 DEBUG "status": "success", 03:27:57 DEBUG "resultType": "vector", 03:27:57 DEBUG "result": [ 03:27:57 DEBUG "resultType": "vector", 03:27:57 DEBUG "status": "success", 03:27:57 DEBUG "result": [ 03:27:57 DEBUG "data": { 03:27:57 DEBUG "resultType": "vector", 03:27:57 DEBUG "result": [ 03:27:57 DEBUG "resultType": "vector", 03:27:57 DEBUG "result": [ 03:27:57 DEBUG { 03:27:57 DEBUG "result": [ 03:27:57 DEBUG { 03:27:57 DEBUG "metric": { 03:27:57 DEBUG "data": { 03:27:57 DEBUG "result": [ 03:27:57 DEBUG { 03:27:57 DEBUG "result": [ 03:27:57 DEBUG "data": { 03:27:57 DEBUG { 03:27:57 DEBUG "resultType": "vector", 03:27:57 DEBUG "result": [ 03:27:57 DEBUG { 03:27:57 DEBUG "result": [ 03:27:57 DEBUG { 03:27:57 DEBUG "metric": { 03:27:57 DEBUG { 03:27:57 DEBUG "metric": { 03:27:57 DEBUG "pod": "ds-cts-0" 03:27:57 DEBUG "resultType": "vector", 03:27:57 DEBUG { 03:27:57 DEBUG "metric": { 03:27:57 DEBUG { 03:27:57 DEBUG "resultType": "vector", 03:27:57 DEBUG "metric": { 03:27:57 DEBUG "result": [ 03:27:57 DEBUG { 03:27:57 DEBUG "metric": { 03:27:57 DEBUG { 03:27:57 DEBUG "metric": { 03:27:57 DEBUG "pod": "ds-idrepo-0" 03:27:57 DEBUG "metric": { 03:27:57 DEBUG "instance": "10.142.0.107:9100" 03:27:57 DEBUG }, 03:27:57 DEBUG "result": [ 03:27:57 DEBUG "metric": { 03:27:57 DEBUG "pod": "ds-idrepo-0" 03:27:57 DEBUG "metric": { 03:27:57 DEBUG "result": [ 03:27:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:27:57 DEBUG { 03:27:57 DEBUG "metric": { 03:27:57 DEBUG "instance": "10.142.0.107:9100" 03:27:57 DEBUG "metric": { 03:27:57 DEBUG "instance": "10.142.0.107:9100" 03:27:57 DEBUG }, 03:27:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:27:57 DEBUG }, 03:27:57 DEBUG "value": [ 03:27:57 DEBUG { 03:27:57 DEBUG "pod": "ds-idrepo-0" 03:27:57 DEBUG }, 03:27:57 DEBUG "instance": "10.142.0.107:9100" 03:27:57 DEBUG { 03:27:57 DEBUG }, 03:27:57 DEBUG "metric": { 03:27:57 DEBUG "pod": "am-767f6b67b5-9zw22" 03:27:57 DEBUG }, 03:27:57 DEBUG "pod": "am-767f6b67b5-9zw22" 03:27:57 DEBUG }, 03:27:57 DEBUG "value": [ 03:27:57 DEBUG }, 03:27:57 DEBUG "value": [ 03:27:57 DEBUG 03:27:57 DEBUG "metric": { 03:27:57 DEBUG }, 03:27:57 DEBUG "value": [ 03:27:57 DEBUG }, 03:27:57 DEBUG "metric": { 03:27:57 DEBUG "value": [ 03:27:57 DEBUG "pod": "ds-cts-0" 03:27:57 DEBUG }, 03:27:57 DEBUG "value": [ 03:27:57 DEBUG }, 03:27:57 DEBUG "value": [ 03:27:57 DEBUG 03:27:57 DEBUG "value": [ 03:27:57 DEBUG 03:27:57 DEBUG 03:27:57 DEBUG "pod": "am-767f6b67b5-9zw22" 03:27:57 DEBUG "value": [ 03:27:57 DEBUG 03:27:57 DEBUG "value": [ 03:27:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:27:57 DEBUG 03:27:57 DEBUG }, 03:27:57 DEBUG "value": [ 03:27:57 DEBUG 03:27:57 DEBUG "value": [ 03:27:57 DEBUG 03:27:57 DEBUG 03:27:57 DEBUG 03:27:57 DEBUG 03:27:57 DEBUG 03:27:57 DEBUG }, 03:27:57 DEBUG 03:27:57 DEBUG 03:27:57 DEBUG 03:27:57 DEBUG }, 03:27:57 DEBUG 03:27:57 DEBUG "value": [ 03:27:57 DEBUG 03:27:57 DEBUG 03:27:57 DEBUG 03:27:57 DEBUG 03:27:57 DEBUG 03:27:57 DEBUG 03:27:57 DEBUG 03:27:57 DEBUG [...] 03:27:57 DEBUG "value": [ 03:27:57 DEBUG 03:27:57 DEBUG 03:27:57 DEBUG 03:27:57 DEBUG "value": [ 03:27:57 DEBUG 03:27:57 DEBUG [...] 03:27:57 DEBUG 03:27:57 DEBUG 03:27:57 DEBUG 03:27:57 DEBUG 03:27:57 DEBUG [...] 03:27:57 DEBUG 03:27:57 DEBUG [...] 03:27:57 DEBUG 03:27:57 DEBUG 03:27:57 DEBUG 03:27:57 DEBUG 03:27:57 DEBUG 03:27:57 DEBUG 03:27:57 DEBUG 03:27:57 DEBUG 03:27:57 DEBUG 03:27:57 DEBUG [...] 03:27:57 DEBUG 03:27:57 DEBUG [...] 03:27:57 DEBUG 03:27:57 DEBUG [...] 03:27:57 DEBUG 03:27:57 DEBUG 03:27:57 DEBUG [...] 03:27:57 DEBUG [...] 03:27:57 DEBUG 03:27:57 DEBUG [...] 03:27:57 DEBUG 03:27:57 DEBUG 03:27:57 DEBUG 03:27:57 DEBUG [...] 03:27:57 DEBUG 03:27:57 DEBUG [...] 03:27:57 DEBUG 03:27:57 DEBUG 03:27:57 DEBUG 03:27:57 DEBUG 03:27:57 DEBUG 03:27:57 DEBUG 03:27:57 DEBUG 03:27:57 DEBUG [...] 03:27:57 DEBUG 03:27:57 DEBUG 03:27:57 DEBUG 03:27:57 DEBUG 03:27:57 DEBUG 03:27:57 DEBUG 03:27:57 DEBUG 03:27:57 DEBUG 03:27:57 DEBUG 03:27:57 DEBUG 03:27:57 DEBUG 03:27:57 DEBUG "pod": "ds-cts-2" 03:27:57 DEBUG 03:27:57 DEBUG 03:27:57 DEBUG 03:27:57 DEBUG 03:27:57 DEBUG [...] 03:27:57 DEBUG [...] 03:27:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:27:57 DEBUG 03:27:57 DEBUG 03:27:57 DEBUG 03:27:57 DEBUG "pod": "ds-cts-2" 03:27:57 DEBUG "instance": "10.142.0.43:9100" 03:27:57 DEBUG "instance": "10.142.0.43:9100" 03:27:57 DEBUG 03:27:57 DEBUG }, 03:27:57 DEBUG 03:27:57 DEBUG 03:27:57 DEBUG 03:27:57 DEBUG 03:27:57 DEBUG 03:27:57 DEBUG 03:27:57 DEBUG }, 03:27:57 DEBUG 03:27:57 DEBUG "instance": "10.142.0.43:9100" 03:27:57 DEBUG 03:27:57 DEBUG }, 03:27:57 DEBUG }, 03:27:57 DEBUG }, 03:27:57 DEBUG 03:27:57 DEBUG "value": [ 03:27:57 DEBUG "pod": "ds-idrepo-2" 03:27:57 DEBUG "pod": "am-767f6b67b5-vbltz" 03:27:57 DEBUG 03:27:57 DEBUG "instance": "10.142.0.43:9100" 03:27:57 DEBUG 03:27:57 DEBUG 03:27:57 DEBUG "value": [ 03:27:57 DEBUG "pod": "am-767f6b67b5-vbltz" 03:27:57 DEBUG }, 03:27:57 DEBUG "pod": "am-767f6b67b5-vbltz" 03:27:57 DEBUG "value": [ 03:27:57 DEBUG "value": [ 03:27:57 DEBUG "value": [ 03:27:57 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 03:27:57 DEBUG 1713666477, 03:27:57 DEBUG }, 03:27:57 DEBUG }, 03:27:57 DEBUG "pod": "ds-idrepo-2" 03:27:57 DEBUG }, 03:27:57 DEBUG "pod": "end-user-ui-58878d7668-z8s6j" 03:27:57 DEBUG 03:27:57 DEBUG 1713666477, 03:27:57 DEBUG }, 03:27:57 DEBUG "value": [ 03:27:57 DEBUG }, 03:27:57 DEBUG 1713666477, 03:27:57 DEBUG 1713666477, 03:27:57 DEBUG 1713666477, 03:27:57 DEBUG }, 03:27:57 DEBUG "29.4" 03:27:57 DEBUG "value": [ 03:27:57 DEBUG "value": [ 03:27:57 DEBUG }, 03:27:57 DEBUG "value": [ 03:27:57 DEBUG }, 03:27:57 DEBUG 03:27:57 DEBUG "1147392000" 03:27:57 DEBUG "value": [ 03:27:57 DEBUG 1713666477, 03:27:57 DEBUG "value": [ 03:27:57 DEBUG "0" 03:27:57 DEBUG "0.08566666666666264" 03:27:57 DEBUG "101338215.66666666" 03:27:57 DEBUG "value": [ 03:27:57 DEBUG ] 03:27:57 DEBUG 1713666477, 03:27:57 DEBUG 1713666477, 03:27:57 DEBUG "value": [ 03:27:57 DEBUG 1713666477, 03:27:57 DEBUG "value": [ 03:27:57 DEBUG "pod": "ds-cts-2" 03:27:57 DEBUG ] 03:27:57 DEBUG 1713666477, 03:27:57 DEBUG "4437348363264" 03:27:57 DEBUG 1713666477, 03:27:57 DEBUG ] 03:27:57 DEBUG ] 03:27:57 DEBUG ] 03:27:57 DEBUG 1713666477, 03:27:57 DEBUG } 03:27:57 DEBUG "0" 03:27:57 DEBUG "3672.1" 03:27:57 DEBUG 1713666477, 03:27:57 DEBUG "101340787.19999999" 03:27:57 DEBUG 1713666477, 03:27:57 DEBUG }, 03:27:57 DEBUG } 03:27:57 DEBUG "158.05200000000002" 03:27:57 DEBUG ] 03:27:57 DEBUG } 03:27:57 DEBUG } 03:27:57 DEBUG } 03:27:57 DEBUG } 03:27:57 DEBUG "0" 03:27:57 DEBUG ] 03:27:57 DEBUG ] 03:27:57 DEBUG ] 03:27:57 DEBUG "0" 03:27:57 DEBUG ] 03:27:57 DEBUG "0" 03:27:57 DEBUG "value": [ 03:27:57 DEBUG ] 03:27:57 DEBUG ] 03:27:57 DEBUG "3330.366666666667" 03:27:57 DEBUG ] 03:27:57 DEBUG ] 03:27:57 DEBUG ] 03:27:57 DEBUG ] 03:27:57 DEBUG ] 03:27:57 DEBUG } 03:27:57 DEBUG } 03:27:57 DEBUG } 03:27:57 DEBUG ] 03:27:57 DEBUG } 03:27:57 DEBUG ] 03:27:57 DEBUG 1713666477, 03:27:57 DEBUG } 03:27:57 DEBUG } 03:27:57 DEBUG ] 03:27:57 DEBUG } 03:27:57 DEBUG } 03:27:57 DEBUG } 03:27:57 DEBUG } 03:27:57 DEBUG } 03:27:57 DEBUG } 03:27:57 DEBUG ] 03:27:57 DEBUG ] 03:27:57 DEBUG } 03:27:57 DEBUG ] 03:27:57 DEBUG } 03:27:57 DEBUG "0" 03:27:57 DEBUG } 03:27:57 DEBUG ] 03:27:57 DEBUG } 03:27:57 DEBUG } 03:27:57 DEBUG } 03:27:57 DEBUG } 03:27:57 DEBUG } 03:27:57 DEBUG ] 03:27:57 DEBUG } 03:27:57 DEBUG } 03:27:57 DEBUG ] 03:27:57 DEBUG } 03:27:57 DEBUG ] 03:27:57 DEBUG ] 03:27:57 DEBUG } 03:27:57 DEBUG ] 03:27:57 DEBUG } 03:27:57 DEBUG } 03:27:57 DEBUG } 03:27:57 DEBUG } 03:27:57 DEBUG } 03:27:57 DEBUG } 03:27:57 DEBUG } 03:27:57 DEBUG } 03:27:57 DEBUG } 03:27:57 DEBUG } 03:27:57 DEBUG } 03:27:57 DEBUG } 03:27:57 DEBUG ] 03:27:57 DEBUG } 03:27:57 DEBUG } 03:27:57 DEBUG } 03:28:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713666533" 03:28:53 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666533" 03:28:54 ERROR [http_cmd]: ERROR 03:28:54 ERROR --- status code --- 03:28:54 ERROR http status code is 400 (expected [200, 401, 403, 404]) 03:28:54 ERROR --- http response --- 03:28:54 INFO [http_cmd]: http status code OK 03:28:54 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 03:28:54 DEBUG --- status code --- 03:28:54 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 03:28:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:28:54 DEBUG --- http response --- 03:28:54 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 03:28:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666535" 03:28:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666535" 03:28:55 INFO [http_cmd]: http status code OK 03:28:55 DEBUG --- status code --- 03:28:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:28:55 DEBUG --- http response --- 03:28:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:28:55 DEBUG ----- output ----- 03:28:55 DEBUG { 03:28:55 DEBUG "status": "success", 03:28:55 DEBUG "data": { 03:28:55 DEBUG "resultType": "vector", 03:28:55 DEBUG "result": [ 03:28:55 DEBUG { 03:28:55 DEBUG "metric": { 03:28:55 DEBUG "pod": "ds-cts-0" 03:28:55 DEBUG }, 03:28:55 DEBUG "value": [ 03:28:55 DEBUG 03:28:55 DEBUG 03:28:55 DEBUG 03:28:55 DEBUG [...] 03:28:55 DEBUG 03:28:55 DEBUG 03:28:55 DEBUG 03:28:55 DEBUG "pod": "ds-cts-2" 03:28:55 DEBUG }, 03:28:55 DEBUG "value": [ 03:28:55 DEBUG 1713666535, 03:28:55 DEBUG "0.5939999999999998" 03:28:55 DEBUG ] 03:28:55 DEBUG } 03:28:55 DEBUG ] 03:28:55 DEBUG } 03:28:55 DEBUG } 03:28:55 INFO [http_cmd]: http status code OK 03:28:55 DEBUG --- status code --- 03:28:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:28:55 DEBUG --- http response --- 03:28:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:28:55 DEBUG ----- output ----- 03:28:55 DEBUG { 03:28:55 DEBUG "status": "success", 03:28:55 DEBUG "data": { 03:28:55 DEBUG "resultType": "vector", 03:28:55 DEBUG "result": [ 03:28:55 DEBUG { 03:28:55 DEBUG "metric": { 03:28:55 DEBUG "instance": "10.142.0.107:9100" 03:28:55 DEBUG }, 03:28:55 DEBUG "value": [ 03:28:55 DEBUG 03:28:55 DEBUG 03:28:55 DEBUG 03:28:55 DEBUG [...] 03:28:55 DEBUG 03:28:55 DEBUG 03:28:55 DEBUG 03:28:55 DEBUG "instance": "10.142.0.43:9100" 03:28:55 DEBUG }, 03:28:55 DEBUG "value": [ 03:28:55 DEBUG 1713666535, 03:28:55 DEBUG "0.39229999999999826" 03:28:55 DEBUG ] 03:28:55 DEBUG } 03:28:55 DEBUG ] 03:28:55 DEBUG } 03:28:55 DEBUG } 03:28:55 INFO 03:28:55 INFO [loop_until]: kubectl --namespace=xlou top pods 03:28:55 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:28:55 INFO [loop_until]: OK (rc = 0) 03:28:55 DEBUG --- stdout --- 03:28:55 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 22757m 8800Mi am-767f6b67b5-qcqz2 22794m 9141Mi am-767f6b67b5-vbltz 24396m 8906Mi ds-cts-0 21174m 12504Mi ds-cts-1 19876m 12489Mi ds-cts-2 20106m 12564Mi ds-idrepo-0 9643m 3918Mi ds-idrepo-1 9289m 3963Mi ds-idrepo-2 9552m 3916Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1179Mi idm-b85f9d978-wnhxb 4m 1239Mi lodemon-54c87f8cbb-jzrjs 14m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-68c5d5b884-p97w5 12101m 1007Mi 03:28:55 DEBUG --- stderr --- 03:28:55 DEBUG 03:28:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666535" 03:28:55 INFO [http_cmd]: http status code OK 03:28:55 DEBUG --- status code --- 03:28:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:28:55 DEBUG --- http response --- 03:28:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:28:55 DEBUG ----- output ----- 03:28:55 DEBUG { 03:28:55 DEBUG "status": "success", 03:28:55 DEBUG "data": { 03:28:55 DEBUG "resultType": "vector", 03:28:55 DEBUG "result": [ 03:28:55 DEBUG { 03:28:55 DEBUG "metric": { 03:28:55 DEBUG "pod": "am-767f6b67b5-9zw22" 03:28:55 DEBUG }, 03:28:55 DEBUG "value": [ 03:28:55 DEBUG 03:28:55 DEBUG 03:28:55 DEBUG 03:28:55 DEBUG [...] 03:28:55 DEBUG 03:28:55 DEBUG 03:28:55 DEBUG 03:28:55 DEBUG "pod": "am-767f6b67b5-vbltz" 03:28:55 DEBUG }, 03:28:55 DEBUG "value": [ 03:28:55 DEBUG 1713666535, 03:28:55 DEBUG "3366.7" 03:28:55 DEBUG ] 03:28:55 DEBUG } 03:28:55 DEBUG ] 03:28:55 DEBUG } 03:28:55 DEBUG } 03:28:56 INFO 03:28:56 INFO [loop_until]: kubectl --namespace=xlou top node 03:28:56 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:28:56 INFO [loop_until]: OK (rc = 0) 03:28:56 DEBUG --- stdout --- 03:28:56 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 66m 0% 6281Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 9875m 33% 7755Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 22345m 74% 11610Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 82m 0% 3352Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 20513m 68% 15806Mi 14% gke-xlou-cdm-default-pool-1b0769c8-wpwl 161m 0% 5245Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 22967m 76% 12079Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 19781m 66% 3193Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 24983m 83% 11085Mi 9% gke-xlou-cdm-ds-0f601751-j488 10324m 34% 7360Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 19535m 65% 17614Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 10153m 34% 8008Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20036m 67% 15286Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 12108m 76% 11317Mi 19% 03:28:56 DEBUG --- stderr --- 03:28:56 DEBUG 03:28:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666536" 03:28:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666536" 03:28:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666536" 03:28:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713666536" 03:28:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666536" 03:28:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666536" 03:28:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666536" 03:28:57 INFO [http_cmd]: http status code OK 03:28:57 DEBUG --- status code --- 03:28:57 INFO [http_cmd]: http status code OK 03:28:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:28:57 DEBUG --- status code --- 03:28:57 DEBUG --- http response --- 03:28:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:28:57 INFO [http_cmd]: http status code OK 03:28:57 INFO [http_cmd]: http status code OK 03:28:57 DEBUG --- http response --- 03:28:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 03:28:57 INFO [http_cmd]: http status code OK 03:28:57 DEBUG --- status code --- 03:28:57 DEBUG --- status code --- 03:28:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:28:57 DEBUG ----- output ----- 03:28:57 INFO [http_cmd]: http status code OK 03:28:57 DEBUG --- status code --- 03:28:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:28:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:28:57 DEBUG --- http response --- 03:28:57 DEBUG { 03:28:57 DEBUG --- status code --- 03:28:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:28:57 DEBUG --- http response --- 03:28:57 DEBUG ----- output ----- 03:28:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:28:57 DEBUG "status": "success", 03:28:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:28:57 DEBUG --- http response --- 03:28:57 INFO [http_cmd]: http status code OK 03:28:57 DEBUG --- status code --- 03:28:57 DEBUG { 03:28:57 DEBUG ----- output ----- 03:28:57 DEBUG { 03:28:57 DEBUG --- http response --- 03:28:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:28:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:28:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:28:57 DEBUG "status": "success", 03:28:57 DEBUG "status": "success", 03:28:57 DEBUG "data": { 03:28:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:28:57 DEBUG ----- output ----- 03:28:57 DEBUG --- http response --- 03:28:57 DEBUG ----- output ----- 03:28:57 DEBUG "data": { 03:28:57 DEBUG "data": { 03:28:57 DEBUG "resultType": "vector", 03:28:57 DEBUG ----- output ----- 03:28:57 DEBUG { 03:28:57 DEBUG { 03:28:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 03:28:57 DEBUG "resultType": "vector", 03:28:57 DEBUG "resultType": "vector", 03:28:57 DEBUG "result": [ 03:28:57 DEBUG { 03:28:57 DEBUG "status": "success", 03:28:57 DEBUG "status": "success", 03:28:57 DEBUG ----- output ----- 03:28:57 DEBUG "result": [ 03:28:57 DEBUG "result": [ 03:28:57 DEBUG { 03:28:57 DEBUG "status": "success", 03:28:57 DEBUG "data": { 03:28:57 DEBUG "data": { 03:28:57 DEBUG { 03:28:57 DEBUG { 03:28:57 DEBUG { 03:28:57 DEBUG "metric": { 03:28:57 DEBUG "resultType": "vector", 03:28:57 DEBUG "data": { 03:28:57 DEBUG "resultType": "vector", 03:28:57 DEBUG "status": "success", 03:28:57 DEBUG "metric": { 03:28:57 DEBUG "metric": { 03:28:57 DEBUG "pod": "am-767f6b67b5-9zw22" 03:28:57 DEBUG "result": [ 03:28:57 DEBUG "resultType": "vector", 03:28:57 DEBUG "result": [ 03:28:57 DEBUG "data": { 03:28:57 DEBUG "pod": "ds-idrepo-0" 03:28:57 DEBUG }, 03:28:57 DEBUG "pod": "ds-idrepo-1" 03:28:57 DEBUG { 03:28:57 DEBUG "result": [ 03:28:57 DEBUG { 03:28:57 DEBUG "resultType": "vector", 03:28:57 DEBUG }, 03:28:57 DEBUG "value": [ 03:28:57 DEBUG }, 03:28:57 DEBUG "value": [ 03:28:57 DEBUG { 03:28:57 DEBUG "metric": { 03:28:57 DEBUG "result": [ 03:28:57 DEBUG "value": [ 03:28:57 DEBUG 03:28:57 DEBUG "metric": { 03:28:57 DEBUG 03:28:57 DEBUG "metric": { 03:28:57 DEBUG "pod": "am-767f6b67b5-9zw22" 03:28:57 DEBUG { 03:28:57 DEBUG 03:28:57 DEBUG 03:28:57 DEBUG "pod": "am-767f6b67b5-9zw22" 03:28:57 DEBUG 03:28:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:28:57 DEBUG }, 03:28:57 DEBUG "metric": { 03:28:57 DEBUG 03:28:57 DEBUG 03:28:57 DEBUG [...] 03:28:57 DEBUG 03:28:57 DEBUG 03:28:57 DEBUG }, 03:28:57 DEBUG "pod": "ds-idrepo-1" 03:28:57 DEBUG }, 03:28:57 DEBUG "value": [ 03:28:57 DEBUG "value": [ 03:28:57 DEBUG 03:28:57 DEBUG "pod": "am-767f6b67b5-vbltz" 03:28:57 DEBUG "value": [ 03:28:57 DEBUG 03:28:57 DEBUG [...] 03:28:57 DEBUG 03:28:57 DEBUG }, 03:28:57 DEBUG }, 03:28:57 DEBUG 03:28:57 DEBUG 03:28:57 DEBUG 03:28:57 DEBUG 03:28:57 DEBUG 03:28:57 DEBUG "value": [ 03:28:57 DEBUG "value": [ 03:28:57 DEBUG [...] 03:28:57 DEBUG 03:28:57 DEBUG 03:28:57 DEBUG 03:28:57 DEBUG 03:28:57 DEBUG 1713666536, 03:28:57 DEBUG 03:28:57 DEBUG 03:28:57 DEBUG 03:28:57 DEBUG 03:28:57 DEBUG 03:28:57 DEBUG [...] 03:28:57 DEBUG "3336.733333333333" 03:28:57 DEBUG 03:28:57 DEBUG 03:28:57 DEBUG [...] 03:28:57 DEBUG [...] 03:28:57 DEBUG "pod": "ds-idrepo-2" 03:28:57 DEBUG 03:28:57 DEBUG ] 03:28:57 DEBUG 03:28:57 DEBUG 03:28:57 DEBUG 03:28:57 DEBUG 03:28:57 DEBUG }, 03:28:57 DEBUG 03:28:57 DEBUG 03:28:57 DEBUG "pod": "am-767f6b67b5-vbltz" 03:28:57 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:28:57 DEBUG [...] 03:28:57 DEBUG 03:28:57 DEBUG 03:28:57 DEBUG }, 03:28:57 DEBUG 03:28:57 DEBUG }, 03:28:57 DEBUG 03:28:57 DEBUG } 03:28:57 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:28:57 DEBUG "value": [ 03:28:57 DEBUG "value": [ 03:28:57 DEBUG 03:28:57 DEBUG "value": [ 03:28:57 DEBUG 03:28:57 DEBUG ] 03:28:57 DEBUG }, 03:28:57 DEBUG 1713666536, 03:28:57 DEBUG "0" 03:28:57 DEBUG "pod": "am-767f6b67b5-vbltz" 03:28:57 DEBUG }, 03:28:57 DEBUG 03:28:57 DEBUG } 03:28:57 DEBUG "value": [ 03:28:57 DEBUG ] 03:28:57 DEBUG "value": [ 03:28:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:28:57 DEBUG 1713666536, 03:28:57 DEBUG "0" 03:28:57 DEBUG 1713666536, 03:28:57 DEBUG "32431227.00328634" 03:28:57 DEBUG } 03:28:57 DEBUG ] 03:28:57 DEBUG }, 03:28:57 DEBUG ] 03:28:57 DEBUG ] 03:28:57 DEBUG 1713666536, 03:28:57 DEBUG "11.821918197886843" 03:28:57 DEBUG ] 03:28:57 DEBUG "value": [ 03:28:57 DEBUG } 03:28:57 DEBUG ] 03:28:57 DEBUG 1713666536, 03:28:57 DEBUG } 03:28:57 DEBUG } 03:28:57 DEBUG 1713666536, 03:28:57 DEBUG } 03:28:57 DEBUG } 03:28:57 DEBUG } 03:28:57 DEBUG "599630.5" 03:28:57 DEBUG ] 03:28:57 DEBUG "11.833315620094655" 03:28:57 DEBUG } 03:28:57 DEBUG ] 03:28:57 DEBUG } 03:28:57 DEBUG ] 03:28:57 DEBUG } 03:28:57 DEBUG } 03:28:57 DEBUG } 03:28:57 DEBUG } 03:28:57 DEBUG ] 03:28:57 DEBUG } 03:28:57 DEBUG } 03:28:57 DEBUG } 03:28:57 DEBUG ] 03:28:57 DEBUG } 03:28:57 DEBUG ] 03:28:57 DEBUG } 03:28:57 DEBUG } 03:28:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666537" 03:28:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713666537" 03:28:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666537" 03:28:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666537" 03:28:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713666537" 03:28:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666537" 03:28:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666537" 03:28:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666537" 03:28:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666537" 03:28:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666537" 03:28:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666537" 03:28:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713666537" 03:28:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666537" 03:28:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666537" 03:28:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666537" 03:28:57 INFO [http_cmd]: http status code OK 03:28:57 INFO [http_cmd]: http status code OK 03:28:57 INFO [http_cmd]: http status code OK 03:28:57 INFO [http_cmd]: http status code OK 03:28:57 INFO [http_cmd]: http status code OK 03:28:57 INFO [http_cmd]: http status code OK 03:28:57 INFO [http_cmd]: http status code OK 03:28:57 INFO [http_cmd]: http status code OK 03:28:57 INFO [http_cmd]: http status code OK 03:28:57 INFO [http_cmd]: http status code OK 03:28:57 INFO [http_cmd]: http status code OK 03:28:57 DEBUG --- status code --- 03:28:57 DEBUG --- status code --- 03:28:57 DEBUG --- status code --- 03:28:57 DEBUG --- status code --- 03:28:57 DEBUG --- status code --- 03:28:57 INFO [http_cmd]: http status code OK 03:28:57 DEBUG --- status code --- 03:28:57 DEBUG --- status code --- 03:28:57 DEBUG --- status code --- 03:28:57 DEBUG --- status code --- 03:28:57 DEBUG --- status code --- 03:28:57 DEBUG --- status code --- 03:28:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:28:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:28:57 INFO [http_cmd]: http status code OK 03:28:57 DEBUG --- status code --- 03:28:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:28:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:28:57 DEBUG --- status code --- 03:28:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:28:57 DEBUG --- http response --- 03:28:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:28:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:28:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:28:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:28:57 DEBUG --- http response --- 03:28:57 DEBUG --- http response --- 03:28:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:28:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:28:57 INFO [http_cmd]: http status code OK 03:28:57 DEBUG --- http response --- 03:28:57 DEBUG --- http response --- 03:28:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:28:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:28:57 DEBUG --- http response --- 03:28:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:28:57 DEBUG --- http response --- 03:28:57 DEBUG --- http response --- 03:28:57 INFO [http_cmd]: http status code OK 03:28:57 DEBUG --- http response --- 03:28:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:28:57 DEBUG --- http response --- 03:28:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:28:57 DEBUG --- http response --- 03:28:57 DEBUG --- status code --- 03:28:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:28:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:28:57 DEBUG --- http response --- 03:28:57 DEBUG --- http response --- 03:28:57 DEBUG ----- output ----- 03:28:57 DEBUG { 03:28:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:28:57 DEBUG ----- output ----- 03:28:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:28:57 DEBUG ----- output ----- 03:28:57 DEBUG ----- output ----- 03:28:57 DEBUG ----- output ----- 03:28:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:28:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:28:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:28:57 DEBUG ----- output ----- 03:28:57 DEBUG ----- output ----- 03:28:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:28:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:28:57 DEBUG "status": "success", 03:28:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:28:57 DEBUG --- status code --- 03:28:57 DEBUG { 03:28:57 DEBUG { 03:28:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 03:28:57 DEBUG { 03:28:57 DEBUG { 03:28:57 DEBUG ----- output ----- 03:28:57 DEBUG ----- output ----- 03:28:57 DEBUG --- http response --- 03:28:57 DEBUG { 03:28:57 DEBUG { 03:28:57 DEBUG ----- output ----- 03:28:57 DEBUG ----- output ----- 03:28:57 DEBUG "data": { 03:28:57 DEBUG ----- output ----- 03:28:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:28:57 DEBUG "status": "success", 03:28:57 DEBUG "status": "success", 03:28:57 DEBUG ----- output ----- 03:28:57 DEBUG "status": "success", 03:28:57 DEBUG "status": "success", 03:28:57 DEBUG { 03:28:57 DEBUG { 03:28:57 DEBUG "status": "success", 03:28:57 DEBUG "data": { 03:28:57 DEBUG "status": "success", 03:28:57 DEBUG { 03:28:57 DEBUG { 03:28:57 DEBUG "resultType": "vector", 03:28:57 DEBUG { 03:28:57 DEBUG --- http response --- 03:28:57 DEBUG "data": { 03:28:57 DEBUG "data": { 03:28:57 DEBUG { 03:28:57 DEBUG "data": { 03:28:57 DEBUG "data": { 03:28:57 DEBUG "status": "success", 03:28:57 DEBUG "status": "success", 03:28:57 DEBUG "resultType": "vector", 03:28:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 03:28:57 DEBUG "data": { 03:28:57 DEBUG "status": "success", 03:28:57 DEBUG "status": "success", 03:28:57 DEBUG "result": [ 03:28:57 DEBUG "status": "success", 03:28:57 DEBUG "data": { 03:28:57 DEBUG "resultType": "vector", 03:28:57 DEBUG "resultType": "vector", 03:28:57 DEBUG "status": "success", 03:28:57 DEBUG "resultType": "vector", 03:28:57 DEBUG "resultType": "vector", 03:28:57 DEBUG "data": { 03:28:57 DEBUG "data": { 03:28:57 DEBUG "result": [ 03:28:57 DEBUG ----- output ----- 03:28:57 DEBUG "resultType": "vector", 03:28:57 DEBUG "data": { 03:28:57 DEBUG "data": { 03:28:57 DEBUG { 03:28:57 DEBUG "resultType": "vector", 03:28:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:28:57 DEBUG "result": [ 03:28:57 DEBUG "result": [ 03:28:57 DEBUG "data": { 03:28:57 DEBUG "result": [ 03:28:57 DEBUG "result": [ 03:28:57 DEBUG "resultType": "vector", 03:28:57 DEBUG "resultType": "vector", 03:28:57 DEBUG { 03:28:57 DEBUG { 03:28:57 DEBUG "result": [ 03:28:57 DEBUG "resultType": "vector", 03:28:57 DEBUG "resultType": "vector", 03:28:57 DEBUG "metric": { 03:28:57 DEBUG "result": [ 03:28:57 DEBUG ----- output ----- 03:28:57 DEBUG { 03:28:57 DEBUG { 03:28:57 DEBUG "resultType": "vector", 03:28:57 DEBUG { 03:28:57 DEBUG "metric": { 03:28:57 DEBUG "result": [ 03:28:57 DEBUG "result": [ 03:28:57 DEBUG { 03:28:57 DEBUG "status": "success", 03:28:57 DEBUG { 03:28:57 DEBUG "metric": { 03:28:57 DEBUG "result": [ 03:28:57 DEBUG "pod": "am-767f6b67b5-9zw22" 03:28:57 DEBUG { 03:28:57 DEBUG { 03:28:57 DEBUG "status": "success", 03:28:57 DEBUG "metric": { 03:28:57 DEBUG "result": [ 03:28:57 DEBUG { 03:28:57 DEBUG "pod": "ds-cts-0" 03:28:57 DEBUG { 03:28:57 DEBUG "metric": { 03:28:57 DEBUG "metric": { 03:28:57 DEBUG "data": { 03:28:57 DEBUG "pod": "ds-idrepo-0" 03:28:57 DEBUG "result": [ 03:28:57 DEBUG { 03:28:57 DEBUG }, 03:28:57 DEBUG "value": [ 03:28:57 DEBUG 03:28:57 DEBUG "metric": { 03:28:57 DEBUG "metric": { 03:28:57 DEBUG { 03:28:57 DEBUG "metric": { 03:28:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:28:57 DEBUG }, 03:28:57 DEBUG "instance": "10.142.0.107:9100" 03:28:57 DEBUG "pod": "ds-cts-0" 03:28:57 DEBUG "resultType": "vector", 03:28:57 DEBUG }, 03:28:57 DEBUG { 03:28:57 DEBUG "metric": { 03:28:57 DEBUG "data": { 03:28:57 DEBUG 03:28:57 DEBUG "instance": "10.142.0.107:9100" 03:28:57 DEBUG "pod": "am-767f6b67b5-9zw22" 03:28:57 DEBUG "instance": "10.142.0.107:9100" 03:28:57 DEBUG "metric": { 03:28:57 DEBUG "value": [ 03:28:57 DEBUG "metric": { 03:28:57 DEBUG }, 03:28:57 DEBUG }, 03:28:57 DEBUG }, 03:28:57 DEBUG "result": [ 03:28:57 DEBUG "value": [ 03:28:57 DEBUG "metric": { 03:28:57 DEBUG "pod": "ds-idrepo-0" 03:28:57 DEBUG "resultType": "vector", 03:28:57 DEBUG 03:28:57 DEBUG [...] 03:28:57 DEBUG }, 03:28:57 DEBUG }, 03:28:57 DEBUG "pod": "ds-idrepo-1" 03:28:57 DEBUG 03:28:57 DEBUG "instance": "10.142.0.107:9100" 03:28:57 DEBUG "value": [ 03:28:57 DEBUG "value": [ 03:28:57 DEBUG "value": [ 03:28:57 DEBUG { 03:28:57 DEBUG 03:28:57 DEBUG "pod": "ds-idrepo-0" 03:28:57 DEBUG }, 03:28:57 DEBUG "result": [ 03:28:57 DEBUG 03:28:57 DEBUG }, 03:28:57 DEBUG "value": [ 03:28:57 DEBUG "value": [ 03:28:57 DEBUG }, 03:28:57 DEBUG 03:28:57 DEBUG }, 03:28:57 DEBUG 03:28:57 DEBUG 03:28:57 DEBUG 03:28:57 DEBUG "metric": { 03:28:57 DEBUG "pod": "ds-idrepo-1" 03:28:57 DEBUG 03:28:57 DEBUG "value": [ 03:28:57 DEBUG { 03:28:57 DEBUG 03:28:57 DEBUG "value": [ 03:28:57 DEBUG 03:28:57 DEBUG 03:28:57 DEBUG "value": [ 03:28:57 DEBUG 03:28:57 DEBUG "value": [ 03:28:57 DEBUG 03:28:57 DEBUG 03:28:57 DEBUG 03:28:57 DEBUG }, 03:28:57 DEBUG }, 03:28:57 DEBUG 03:28:57 DEBUG 03:28:57 DEBUG "metric": { 03:28:57 DEBUG 03:28:57 DEBUG 03:28:57 DEBUG 03:28:57 DEBUG 03:28:57 DEBUG 03:28:57 DEBUG [...] 03:28:57 DEBUG 03:28:57 DEBUG 03:28:57 DEBUG 03:28:57 DEBUG 03:28:57 DEBUG "value": [ 03:28:57 DEBUG "value": [ 03:28:57 DEBUG [...] 03:28:57 DEBUG 03:28:57 DEBUG "pod": "am-767f6b67b5-9zw22" 03:28:57 DEBUG }, 03:28:57 DEBUG "value": [ 03:28:57 DEBUG 03:28:57 DEBUG 03:28:57 DEBUG [...] 03:28:57 DEBUG 03:28:57 DEBUG 03:28:57 DEBUG [...] 03:28:57 DEBUG 03:28:57 DEBUG [...] 03:28:57 DEBUG 03:28:57 DEBUG 03:28:57 DEBUG 03:28:57 DEBUG 03:28:57 DEBUG 03:28:57 DEBUG 03:28:57 DEBUG "pod": "am-767f6b67b5-vbltz" 03:28:57 DEBUG [...] 03:28:57 DEBUG 03:28:57 DEBUG 03:28:57 DEBUG 03:28:57 DEBUG 03:28:57 DEBUG 03:28:57 DEBUG 03:28:57 DEBUG 03:28:57 DEBUG 03:28:57 DEBUG [...] 03:28:57 DEBUG 03:28:57 DEBUG [...] 03:28:57 DEBUG 03:28:57 DEBUG 03:28:57 DEBUG }, 03:28:57 DEBUG 03:28:57 DEBUG 03:28:57 DEBUG 03:28:57 DEBUG 03:28:57 DEBUG [...] 03:28:57 DEBUG 03:28:57 DEBUG 03:28:57 DEBUG 03:28:57 DEBUG 03:28:57 DEBUG 03:28:57 DEBUG 03:28:57 DEBUG 03:28:57 DEBUG [...] 03:28:57 DEBUG 03:28:57 DEBUG "value": [ 03:28:57 DEBUG 03:28:57 DEBUG 03:28:57 DEBUG [...] 03:28:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:28:57 DEBUG 03:28:57 DEBUG "instance": "10.142.0.43:9100" 03:28:57 DEBUG 03:28:57 DEBUG [...] 03:28:57 DEBUG [...] 03:28:57 DEBUG 03:28:57 DEBUG "pod": "ds-cts-2" 03:28:57 DEBUG 03:28:57 DEBUG 03:28:57 DEBUG 03:28:57 DEBUG 1713666537, 03:28:57 DEBUG 03:28:57 DEBUG "instance": "10.142.0.43:9100" 03:28:57 DEBUG 03:28:57 DEBUG }, 03:28:57 DEBUG 03:28:57 DEBUG }, 03:28:57 DEBUG "pod": "ds-cts-2" 03:28:57 DEBUG 03:28:57 DEBUG 03:28:57 DEBUG 03:28:57 DEBUG }, 03:28:57 DEBUG 03:28:57 DEBUG 03:28:57 DEBUG "163.1105" 03:28:57 DEBUG "pod": "am-767f6b67b5-vbltz" 03:28:57 DEBUG [...] 03:28:57 DEBUG }, 03:28:57 DEBUG 03:28:57 DEBUG "value": [ 03:28:57 DEBUG 03:28:57 DEBUG "value": [ 03:28:57 DEBUG }, 03:28:57 DEBUG 03:28:57 DEBUG 03:28:57 DEBUG "pod": "ds-cts-2" 03:28:57 DEBUG "value": [ 03:28:57 DEBUG "pod": "ds-idrepo-2" 03:28:57 DEBUG "instance": "10.142.0.43:9100" 03:28:57 DEBUG ] 03:28:57 DEBUG }, 03:28:57 DEBUG 03:28:57 DEBUG "value": [ 03:28:57 DEBUG 03:28:57 DEBUG 1713666537, 03:28:57 DEBUG "instance": "10.142.0.43:9100" 03:28:57 DEBUG 1713666537, 03:28:57 DEBUG "value": [ 03:28:57 DEBUG "pod": "ds-idrepo-2" 03:28:57 DEBUG 03:28:57 DEBUG }, 03:28:57 DEBUG 1713666537, 03:28:57 DEBUG }, 03:28:57 DEBUG }, 03:28:57 DEBUG } 03:28:57 DEBUG "value": [ 03:28:57 DEBUG 03:28:57 DEBUG 1713666537, 03:28:57 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:28:57 DEBUG "1157902336" 03:28:57 DEBUG }, 03:28:57 DEBUG "4442509227008" 03:28:57 DEBUG 1713666537, 03:28:57 DEBUG }, 03:28:57 DEBUG 03:28:57 DEBUG "value": [ 03:28:57 DEBUG "0" 03:28:57 DEBUG "value": [ 03:28:57 DEBUG "value": [ 03:28:57 DEBUG ] 03:28:57 DEBUG 1713666537, 03:28:57 DEBUG 03:28:57 DEBUG "0.0806666666666577" 03:28:57 DEBUG }, 03:28:57 DEBUG ] 03:28:57 DEBUG "value": [ 03:28:57 DEBUG ] 03:28:57 DEBUG "0" 03:28:57 DEBUG "value": [ 03:28:57 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:28:57 DEBUG 1713666537, 03:28:57 DEBUG ] 03:28:57 DEBUG 1713666537, 03:28:57 DEBUG 1713666537, 03:28:57 DEBUG } 03:28:57 DEBUG "3856.5666666666666" 03:28:57 DEBUG "pod": "am-767f6b67b5-vbltz" 03:28:57 DEBUG ] 03:28:57 DEBUG } 03:28:57 DEBUG ] 03:28:57 DEBUG 1713666537, 03:28:57 DEBUG } 03:28:57 DEBUG ] 03:28:57 DEBUG } 03:28:57 DEBUG }, 03:28:57 DEBUG "29.3" 03:28:57 DEBUG } 03:28:57 DEBUG "0" 03:28:57 DEBUG "101643525.4" 03:28:57 DEBUG } 03:28:57 DEBUG ] 03:28:57 DEBUG }, 03:28:57 DEBUG "value": [ 03:28:57 DEBUG } 03:28:57 DEBUG } 03:28:57 DEBUG "101640167.76666667" 03:28:57 DEBUG ] 03:28:57 DEBUG ] 03:28:57 DEBUG 1713666537, 03:28:57 DEBUG "value": [ 03:28:57 DEBUG ] 03:28:57 DEBUG ] 03:28:57 DEBUG ] 03:28:57 DEBUG ] 03:28:57 DEBUG } 03:28:57 DEBUG "value": [ 03:28:57 DEBUG 1713666537, 03:28:57 DEBUG } 03:28:57 DEBUG ] 03:28:57 DEBUG ] 03:28:57 DEBUG } 03:28:57 DEBUG } 03:28:57 DEBUG "0" 03:28:57 DEBUG 1713666537, 03:28:57 DEBUG } 03:28:57 DEBUG } 03:28:57 DEBUG } 03:28:57 DEBUG } 03:28:57 DEBUG ] 03:28:57 DEBUG 1713666537, 03:28:57 DEBUG "0" 03:28:57 DEBUG } 03:28:57 DEBUG } 03:28:57 DEBUG ] 03:28:57 DEBUG } 03:28:57 DEBUG ] 03:28:57 DEBUG } 03:28:57 DEBUG ] 03:28:57 DEBUG } 03:28:57 DEBUG ] 03:28:57 DEBUG ] 03:28:57 DEBUG } 03:28:57 DEBUG "3365.5666666666666" 03:28:57 DEBUG ] 03:28:57 DEBUG } 03:28:57 DEBUG } 03:28:57 DEBUG } 03:28:57 DEBUG ] 03:28:57 DEBUG "0" 03:28:57 DEBUG } 03:28:57 DEBUG } 03:28:57 DEBUG } 03:28:57 DEBUG } 03:28:57 DEBUG ] 03:28:57 DEBUG } 03:28:57 DEBUG } 03:28:57 DEBUG } 03:28:57 DEBUG ] 03:28:57 DEBUG } 03:28:57 DEBUG } 03:28:57 DEBUG } 03:28:57 DEBUG } 03:28:57 DEBUG ] 03:28:57 DEBUG } 03:28:57 DEBUG } 03:28:57 DEBUG ] 03:28:57 DEBUG } 03:28:57 DEBUG ] 03:28:57 DEBUG } 03:28:57 DEBUG } 03:28:57 DEBUG } 03:28:57 DEBUG } 03:28:57 DEBUG } 03:29:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713666594" 03:29:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666594" 03:29:54 INFO [http_cmd]: http status code OK 03:29:54 DEBUG --- status code --- 03:29:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:29:54 DEBUG --- http response --- 03:29:54 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 03:29:54 ERROR [http_cmd]: ERROR 03:29:54 ERROR --- status code --- 03:29:54 ERROR http status code is 400 (expected [200, 401, 403, 404]) 03:29:54 ERROR --- http response --- 03:29:54 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 03:29:54 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 03:29:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666595" 03:29:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666595" 03:29:55 INFO [http_cmd]: http status code OK 03:29:55 DEBUG --- status code --- 03:29:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:29:55 DEBUG --- http response --- 03:29:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:29:55 DEBUG ----- output ----- 03:29:55 DEBUG { 03:29:55 DEBUG "status": "success", 03:29:55 DEBUG "data": { 03:29:55 DEBUG "resultType": "vector", 03:29:55 DEBUG "result": [ 03:29:55 DEBUG { 03:29:55 DEBUG "metric": { 03:29:55 DEBUG "pod": "ds-cts-0" 03:29:55 DEBUG }, 03:29:55 DEBUG "value": [ 03:29:55 DEBUG 03:29:55 DEBUG 03:29:55 DEBUG 03:29:55 DEBUG [...] 03:29:55 DEBUG 03:29:55 DEBUG 03:29:55 DEBUG 03:29:55 DEBUG "pod": "ds-cts-2" 03:29:55 DEBUG }, 03:29:55 DEBUG "value": [ 03:29:55 DEBUG 1713666595, 03:29:55 DEBUG "0.6070333333333338" 03:29:55 DEBUG ] 03:29:55 DEBUG } 03:29:55 DEBUG ] 03:29:55 DEBUG } 03:29:55 DEBUG } 03:29:55 INFO [http_cmd]: http status code OK 03:29:55 DEBUG --- status code --- 03:29:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:29:55 DEBUG --- http response --- 03:29:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:29:55 DEBUG ----- output ----- 03:29:55 DEBUG { 03:29:55 DEBUG "status": "success", 03:29:55 DEBUG "data": { 03:29:55 DEBUG "resultType": "vector", 03:29:55 DEBUG "result": [ 03:29:55 DEBUG { 03:29:55 DEBUG "metric": { 03:29:55 DEBUG "instance": "10.142.0.107:9100" 03:29:55 DEBUG }, 03:29:55 DEBUG "value": [ 03:29:55 DEBUG 03:29:55 DEBUG 03:29:55 DEBUG 03:29:55 DEBUG [...] 03:29:55 DEBUG 03:29:55 DEBUG 03:29:55 DEBUG 03:29:55 DEBUG "instance": "10.142.0.43:9100" 03:29:55 DEBUG }, 03:29:55 DEBUG "value": [ 03:29:55 DEBUG 1713666595, 03:29:55 DEBUG "0.4021666666666628" 03:29:55 DEBUG ] 03:29:55 DEBUG } 03:29:55 DEBUG ] 03:29:55 DEBUG } 03:29:55 DEBUG } 03:29:55 INFO 03:29:55 INFO [loop_until]: kubectl --namespace=xlou top pods 03:29:55 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:29:56 INFO [loop_until]: OK (rc = 0) 03:29:56 DEBUG --- stdout --- 03:29:56 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 22533m 8800Mi am-767f6b67b5-qcqz2 23015m 9157Mi am-767f6b67b5-vbltz 25201m 8945Mi ds-cts-0 20496m 12502Mi ds-cts-1 19519m 12491Mi ds-cts-2 20434m 12566Mi ds-idrepo-0 9761m 3918Mi ds-idrepo-1 9606m 3963Mi ds-idrepo-2 9489m 3915Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1179Mi idm-b85f9d978-wnhxb 3m 1239Mi lodemon-54c87f8cbb-jzrjs 16m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-68c5d5b884-p97w5 11834m 1013Mi 03:29:56 DEBUG --- stderr --- 03:29:56 DEBUG 03:29:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666596" 03:29:56 INFO [http_cmd]: http status code OK 03:29:56 DEBUG --- status code --- 03:29:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:29:56 DEBUG --- http response --- 03:29:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:29:56 DEBUG ----- output ----- 03:29:56 DEBUG { 03:29:56 DEBUG "status": "success", 03:29:56 DEBUG "data": { 03:29:56 DEBUG "resultType": "vector", 03:29:56 DEBUG "result": [ 03:29:56 DEBUG { 03:29:56 DEBUG "metric": { 03:29:56 DEBUG "pod": "am-767f6b67b5-9zw22" 03:29:56 DEBUG }, 03:29:56 DEBUG "value": [ 03:29:56 DEBUG 03:29:56 DEBUG 03:29:56 DEBUG 03:29:56 DEBUG [...] 03:29:56 DEBUG 03:29:56 DEBUG 03:29:56 DEBUG 03:29:56 DEBUG "pod": "am-767f6b67b5-vbltz" 03:29:56 DEBUG }, 03:29:56 DEBUG "value": [ 03:29:56 DEBUG 1713666596, 03:29:56 DEBUG "3340.4666666666667" 03:29:56 DEBUG ] 03:29:56 DEBUG } 03:29:56 DEBUG ] 03:29:56 DEBUG } 03:29:56 DEBUG } 03:29:56 INFO 03:29:56 INFO [loop_until]: kubectl --namespace=xlou top node 03:29:56 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:29:56 INFO [loop_until]: OK (rc = 0) 03:29:56 DEBUG --- stdout --- 03:29:56 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 63m 0% 6285Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 9572m 32% 7760Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 22037m 73% 11618Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 84m 0% 3359Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 20167m 67% 15820Mi 14% gke-xlou-cdm-default-pool-1b0769c8-wpwl 155m 0% 5246Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 22200m 74% 12094Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 19709m 66% 3200Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 25036m 83% 11210Mi 10% gke-xlou-cdm-ds-0f601751-j488 10362m 34% 7359Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 19468m 65% 17622Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 10180m 34% 8016Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20407m 68% 15290Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 11644m 73% 11319Mi 19% 03:29:56 DEBUG --- stderr --- 03:29:56 DEBUG 03:29:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666597" 03:29:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666597" 03:29:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666597" 03:29:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666597" 03:29:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666597" 03:29:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666597" 03:29:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713666597" 03:29:57 INFO [http_cmd]: http status code OK 03:29:57 DEBUG --- status code --- 03:29:57 INFO [http_cmd]: http status code OK 03:29:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:29:57 DEBUG --- status code --- 03:29:57 DEBUG --- http response --- 03:29:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:29:57 INFO [http_cmd]: http status code OK 03:29:57 INFO [http_cmd]: http status code OK 03:29:57 INFO [http_cmd]: http status code OK 03:29:57 DEBUG --- http response --- 03:29:57 INFO [http_cmd]: http status code OK 03:29:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:29:57 DEBUG --- status code --- 03:29:57 DEBUG --- status code --- 03:29:57 DEBUG --- status code --- 03:29:57 DEBUG --- status code --- 03:29:57 DEBUG ----- output ----- 03:29:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:29:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:29:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:29:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:29:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:29:57 DEBUG { 03:29:57 INFO [http_cmd]: http status code OK 03:29:57 DEBUG ----- output ----- 03:29:57 DEBUG --- http response --- 03:29:57 DEBUG --- http response --- 03:29:57 DEBUG --- http response --- 03:29:57 DEBUG --- http response --- 03:29:57 DEBUG "status": "success", 03:29:57 DEBUG --- status code --- 03:29:57 DEBUG { 03:29:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 03:29:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:29:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:29:57 DEBUG "data": { 03:29:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 03:29:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:29:57 DEBUG "status": "success", 03:29:57 DEBUG ----- output ----- 03:29:57 DEBUG ----- output ----- 03:29:57 DEBUG ----- output ----- 03:29:57 DEBUG "resultType": "vector", 03:29:57 DEBUG ----- output ----- 03:29:57 DEBUG --- http response --- 03:29:57 DEBUG "data": { 03:29:57 DEBUG { 03:29:57 DEBUG { 03:29:57 DEBUG { 03:29:57 DEBUG "result": [ 03:29:57 DEBUG { 03:29:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:29:57 DEBUG "resultType": "vector", 03:29:57 DEBUG "status": "success", 03:29:57 DEBUG "status": "success", 03:29:57 DEBUG "status": "success", 03:29:57 DEBUG { 03:29:57 DEBUG "status": "success", 03:29:57 DEBUG ----- output ----- 03:29:57 DEBUG "result": [ 03:29:57 DEBUG "data": { 03:29:57 DEBUG "data": { 03:29:57 DEBUG "data": { 03:29:57 DEBUG "metric": { 03:29:57 DEBUG "data": { 03:29:57 DEBUG { 03:29:57 DEBUG { 03:29:57 DEBUG "resultType": "vector", 03:29:57 DEBUG "resultType": "vector", 03:29:57 DEBUG "resultType": "vector", 03:29:57 DEBUG "pod": "ds-idrepo-0" 03:29:57 DEBUG "resultType": "vector", 03:29:57 DEBUG "status": "success", 03:29:57 DEBUG "metric": { 03:29:57 DEBUG "result": [ 03:29:57 DEBUG "result": [ 03:29:57 DEBUG "result": [ 03:29:57 DEBUG }, 03:29:57 DEBUG "result": [ 03:29:57 DEBUG "data": { 03:29:57 DEBUG "pod": "am-767f6b67b5-9zw22" 03:29:57 DEBUG { 03:29:57 DEBUG { 03:29:57 DEBUG { 03:29:57 DEBUG "value": [ 03:29:57 DEBUG { 03:29:57 DEBUG "resultType": "vector", 03:29:57 DEBUG }, 03:29:57 DEBUG "metric": { 03:29:57 DEBUG "metric": { 03:29:57 DEBUG "metric": { 03:29:57 DEBUG 03:29:57 DEBUG "metric": { 03:29:57 DEBUG "result": [ 03:29:57 DEBUG "value": [ 03:29:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:29:57 DEBUG "pod": "am-767f6b67b5-9zw22" 03:29:57 DEBUG "pod": "am-767f6b67b5-9zw22" 03:29:57 DEBUG 03:29:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:29:57 DEBUG { 03:29:57 DEBUG 03:29:57 DEBUG }, 03:29:57 DEBUG }, 03:29:57 DEBUG }, 03:29:57 DEBUG 03:29:57 DEBUG }, 03:29:57 DEBUG "metric": { 03:29:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:29:57 DEBUG "value": [ 03:29:57 DEBUG 03:29:57 DEBUG 03:29:57 DEBUG [...] 03:29:57 DEBUG "value": [ 03:29:57 DEBUG }, 03:29:57 DEBUG 03:29:57 DEBUG "value": [ 03:29:57 DEBUG 03:29:57 DEBUG "value": [ 03:29:57 DEBUG 03:29:57 DEBUG "value": [ 03:29:57 DEBUG [...] 03:29:57 DEBUG 03:29:57 DEBUG 03:29:57 DEBUG 03:29:57 DEBUG 03:29:57 DEBUG 03:29:57 DEBUG 03:29:57 DEBUG 03:29:57 DEBUG 03:29:57 DEBUG 03:29:57 DEBUG 03:29:57 DEBUG "pod": "am-767f6b67b5-vbltz" 03:29:57 DEBUG 03:29:57 DEBUG 03:29:57 DEBUG 03:29:57 DEBUG 03:29:57 DEBUG 03:29:57 DEBUG 03:29:57 DEBUG }, 03:29:57 DEBUG 03:29:57 DEBUG [...] 03:29:57 DEBUG "pod": "ds-idrepo-2" 03:29:57 DEBUG [...] 03:29:57 DEBUG [...] 03:29:57 DEBUG [...] 03:29:57 DEBUG "value": [ 03:29:57 DEBUG 03:29:57 DEBUG 03:29:57 DEBUG }, 03:29:57 DEBUG 03:29:57 DEBUG 03:29:57 DEBUG 03:29:57 DEBUG 1713666597, 03:29:57 DEBUG [...] 03:29:57 DEBUG 03:29:57 DEBUG "value": [ 03:29:57 DEBUG 03:29:57 DEBUG 03:29:57 DEBUG 03:29:57 DEBUG "3335.5" 03:29:57 DEBUG 03:29:57 DEBUG 03:29:57 DEBUG 1713666597, 03:29:57 DEBUG 03:29:57 DEBUG "pod": "am-767f6b67b5-vbltz" 03:29:57 DEBUG ] 03:29:57 DEBUG 03:29:57 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:29:57 DEBUG "0" 03:29:57 DEBUG 03:29:57 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:29:57 DEBUG }, 03:29:57 DEBUG } 03:29:57 DEBUG 03:29:57 DEBUG }, 03:29:57 DEBUG ] 03:29:57 DEBUG 03:29:57 DEBUG }, 03:29:57 DEBUG "value": [ 03:29:57 DEBUG ] 03:29:57 DEBUG } 03:29:57 DEBUG "value": [ 03:29:57 DEBUG } 03:29:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:29:57 DEBUG "value": [ 03:29:57 DEBUG 1713666597, 03:29:57 DEBUG } 03:29:57 DEBUG 1713666597, 03:29:57 DEBUG "pod": "am-767f6b67b5-vbltz" 03:29:57 DEBUG ] 03:29:57 DEBUG }, 03:29:57 DEBUG "value": [ 03:29:57 DEBUG "600151.5" 03:29:57 DEBUG "32064933.883563496" 03:29:57 DEBUG }, 03:29:57 DEBUG } 03:29:57 DEBUG 1713666597, 03:29:57 DEBUG ] 03:29:57 DEBUG 1713666597, 03:29:57 DEBUG ] 03:29:57 DEBUG "value": [ 03:29:57 DEBUG } 03:29:57 DEBUG "11.799440417564282" 03:29:57 DEBUG } 03:29:57 DEBUG "5.686583075294264" 03:29:57 DEBUG } 03:29:57 DEBUG 1713666597, 03:29:57 DEBUG ] 03:29:57 DEBUG } 03:29:57 DEBUG ] 03:29:57 DEBUG ] 03:29:57 DEBUG "0" 03:29:57 DEBUG ] 03:29:57 DEBUG } 03:29:57 DEBUG ] 03:29:57 DEBUG } 03:29:57 DEBUG ] 03:29:57 DEBUG } 03:29:57 DEBUG ] 03:29:57 DEBUG } 03:29:57 DEBUG } 03:29:57 DEBUG } 03:29:57 DEBUG } 03:29:57 DEBUG } 03:29:57 DEBUG } 03:29:57 DEBUG ] 03:29:57 DEBUG } 03:29:57 DEBUG } 03:29:57 DEBUG } 03:29:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666597" 03:29:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666597" 03:29:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666597" 03:29:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666597" 03:29:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713666597" 03:29:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666597" 03:29:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666597" 03:29:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666597" 03:29:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713666597" 03:29:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713666597" 03:29:57 INFO [http_cmd]: http status code OK 03:29:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666597" 03:29:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666597" 03:29:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666597" 03:29:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666597" 03:29:57 INFO [http_cmd]: http status code OK 03:29:57 DEBUG --- status code --- 03:29:57 INFO [http_cmd]: http status code OK 03:29:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666597" 03:29:57 INFO [http_cmd]: http status code OK 03:29:57 INFO [http_cmd]: http status code OK 03:29:57 INFO [http_cmd]: http status code OK 03:29:57 INFO [http_cmd]: http status code OK 03:29:57 INFO [http_cmd]: http status code OK 03:29:57 INFO [http_cmd]: http status code OK 03:29:57 DEBUG --- status code --- 03:29:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:29:57 DEBUG --- status code --- 03:29:57 INFO [http_cmd]: http status code OK 03:29:57 INFO [http_cmd]: http status code OK 03:29:57 INFO [http_cmd]: http status code OK 03:29:57 DEBUG --- status code --- 03:29:57 DEBUG --- status code --- 03:29:57 DEBUG --- status code --- 03:29:57 DEBUG --- status code --- 03:29:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:29:57 DEBUG --- status code --- 03:29:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:29:57 INFO [http_cmd]: http status code OK 03:29:57 DEBUG --- http response --- 03:29:57 INFO [http_cmd]: http status code OK 03:29:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:29:57 DEBUG --- status code --- 03:29:57 DEBUG --- status code --- 03:29:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:29:57 DEBUG --- status code --- 03:29:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:29:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:29:57 DEBUG --- http response --- 03:29:57 DEBUG --- status code --- 03:29:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:29:57 DEBUG --- http response --- 03:29:57 DEBUG --- status code --- 03:29:57 DEBUG --- status code --- 03:29:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 03:29:57 DEBUG --- http response --- 03:29:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:29:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:29:57 DEBUG --- http response --- 03:29:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:29:57 DEBUG --- http response --- 03:29:57 DEBUG --- http response --- 03:29:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:29:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:29:57 DEBUG --- http response --- 03:29:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:29:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:29:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:29:57 DEBUG ----- output ----- 03:29:57 DEBUG --- http response --- 03:29:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:29:57 DEBUG --- http response --- 03:29:57 INFO [http_cmd]: http status code OK 03:29:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:29:57 DEBUG --- http response --- 03:29:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:29:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:29:57 DEBUG ----- output ----- 03:29:57 DEBUG --- http response --- 03:29:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:29:57 DEBUG --- http response --- 03:29:57 DEBUG ----- output ----- 03:29:57 DEBUG --- http response --- 03:29:57 DEBUG { 03:29:57 DEBUG ----- output ----- 03:29:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:29:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:29:57 DEBUG --- status code --- 03:29:57 DEBUG ----- output ----- 03:29:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:29:57 DEBUG ----- output ----- 03:29:57 DEBUG { 03:29:57 DEBUG { 03:29:57 DEBUG ----- output ----- 03:29:57 DEBUG { 03:29:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:29:57 DEBUG { 03:29:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:29:57 DEBUG "status": "success", 03:29:57 DEBUG { 03:29:57 DEBUG ----- output ----- 03:29:57 DEBUG ----- output ----- 03:29:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:29:57 DEBUG { 03:29:57 DEBUG ----- output ----- 03:29:57 DEBUG "status": "success", 03:29:57 DEBUG "status": "success", 03:29:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 03:29:57 DEBUG "status": "success", 03:29:57 DEBUG ----- output ----- 03:29:57 DEBUG ----- output ----- 03:29:57 DEBUG "status": "success", 03:29:57 DEBUG ----- output ----- 03:29:57 DEBUG "data": { 03:29:57 DEBUG "status": "success", 03:29:57 DEBUG { 03:29:57 DEBUG { 03:29:57 DEBUG --- http response --- 03:29:57 DEBUG "status": "success", 03:29:57 DEBUG { 03:29:57 DEBUG "data": { 03:29:57 DEBUG "data": { 03:29:57 DEBUG ----- output ----- 03:29:57 DEBUG "data": { 03:29:57 DEBUG { 03:29:57 DEBUG { 03:29:57 DEBUG "status": "success", 03:29:57 DEBUG { 03:29:57 DEBUG "resultType": "vector", 03:29:57 DEBUG "data": { 03:29:57 DEBUG "status": "success", 03:29:57 DEBUG "status": "success", 03:29:57 DEBUG "data": { 03:29:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:29:57 DEBUG "status": "success", 03:29:57 DEBUG "resultType": "vector", 03:29:57 DEBUG "resultType": "vector", 03:29:57 DEBUG { 03:29:57 DEBUG "resultType": "vector", 03:29:57 DEBUG "status": "success", 03:29:57 DEBUG "data": { 03:29:57 DEBUG "data": { 03:29:57 DEBUG "status": "success", 03:29:57 DEBUG "data": { 03:29:57 DEBUG "resultType": "vector", 03:29:57 DEBUG "data": { 03:29:57 DEBUG "data": { 03:29:57 DEBUG "resultType": "vector", 03:29:57 DEBUG "result": [ 03:29:57 DEBUG "data": { 03:29:57 DEBUG "result": [ 03:29:57 DEBUG "result": [ 03:29:57 DEBUG "status": "success", 03:29:57 DEBUG "data": { 03:29:57 DEBUG "result": [ 03:29:57 DEBUG "resultType": "vector", 03:29:57 DEBUG "resultType": "vector", 03:29:57 DEBUG "result": [ 03:29:57 DEBUG "result": [ 03:29:57 DEBUG "result": [ 03:29:57 DEBUG "resultType": "vector", 03:29:57 DEBUG "resultType": "vector", 03:29:57 DEBUG { 03:29:57 DEBUG "metric": { 03:29:57 DEBUG { 03:29:57 DEBUG { 03:29:57 DEBUG "resultType": "vector", 03:29:57 DEBUG ----- output ----- 03:29:57 DEBUG "data": { 03:29:57 DEBUG { 03:29:57 DEBUG "result": [ 03:29:57 DEBUG { 03:29:57 DEBUG { 03:29:57 DEBUG "resultType": "vector", 03:29:57 DEBUG { 03:29:57 DEBUG "result": [ 03:29:57 DEBUG "result": [ 03:29:57 DEBUG "pod": "am-767f6b67b5-9zw22" 03:29:57 DEBUG "metric": { 03:29:57 DEBUG "resultType": "vector", 03:29:57 DEBUG "metric": { 03:29:57 DEBUG "result": [ 03:29:57 DEBUG { 03:29:57 DEBUG "resultType": "vector", 03:29:57 DEBUG "metric": { 03:29:57 DEBUG { 03:29:57 DEBUG "metric": { 03:29:57 DEBUG "metric": { 03:29:57 DEBUG "result": [ 03:29:57 DEBUG { 03:29:57 DEBUG { 03:29:57 DEBUG { 03:29:57 DEBUG }, 03:29:57 DEBUG "pod": "ds-idrepo-0" 03:29:57 DEBUG "result": [ 03:29:57 DEBUG "pod": "am-767f6b67b5-9zw22" 03:29:57 DEBUG { 03:29:57 DEBUG "status": "success", 03:29:57 DEBUG "result": [ 03:29:57 DEBUG "pod": "am-767f6b67b5-9zw22" 03:29:57 DEBUG "instance": "10.142.0.107:9100" 03:29:57 DEBUG "metric": { 03:29:57 DEBUG "pod": "ds-cts-0" 03:29:57 DEBUG "metric": { 03:29:57 DEBUG "metric": { 03:29:57 DEBUG "metric": { 03:29:57 DEBUG "metric": { 03:29:57 DEBUG "value": [ 03:29:57 DEBUG }, 03:29:57 DEBUG { 03:29:57 DEBUG }, 03:29:57 DEBUG "metric": { 03:29:57 DEBUG "data": { 03:29:57 DEBUG { 03:29:57 DEBUG }, 03:29:57 DEBUG }, 03:29:57 DEBUG "instance": "10.142.0.107:9100" 03:29:57 DEBUG }, 03:29:57 DEBUG "pod": "ds-idrepo-0" 03:29:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:29:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:29:57 DEBUG "pod": "ds-cts-0" 03:29:57 DEBUG 03:29:57 DEBUG "value": [ 03:29:57 DEBUG "metric": { 03:29:57 DEBUG "value": [ 03:29:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:29:57 DEBUG "resultType": "vector", 03:29:57 DEBUG "metric": { 03:29:57 DEBUG "value": [ 03:29:57 DEBUG "value": [ 03:29:57 DEBUG }, 03:29:57 DEBUG "value": [ 03:29:57 DEBUG }, 03:29:57 DEBUG }, 03:29:57 DEBUG }, 03:29:57 DEBUG }, 03:29:57 DEBUG 03:29:57 DEBUG 03:29:57 DEBUG "instance": "10.142.0.107:9100" 03:29:57 DEBUG 03:29:57 DEBUG }, 03:29:57 DEBUG "result": [ 03:29:57 DEBUG "instance": "10.142.0.107:9100" 03:29:57 DEBUG 03:29:57 DEBUG 03:29:57 DEBUG "value": [ 03:29:57 DEBUG 03:29:57 DEBUG "value": [ 03:29:57 DEBUG "value": [ 03:29:57 DEBUG "value": [ 03:29:57 DEBUG "value": [ 03:29:57 DEBUG 03:29:57 DEBUG 03:29:57 DEBUG }, 03:29:57 DEBUG 03:29:57 DEBUG "value": [ 03:29:57 DEBUG { 03:29:57 DEBUG }, 03:29:57 DEBUG 03:29:57 DEBUG 03:29:57 DEBUG 03:29:57 DEBUG 03:29:57 DEBUG 03:29:57 DEBUG 03:29:57 DEBUG 03:29:57 DEBUG 03:29:57 DEBUG 03:29:57 DEBUG 03:29:57 DEBUG "value": [ 03:29:57 DEBUG 03:29:57 DEBUG 03:29:57 DEBUG "metric": { 03:29:57 DEBUG "value": [ 03:29:57 DEBUG 03:29:57 DEBUG 03:29:57 DEBUG 03:29:57 DEBUG 03:29:57 DEBUG 03:29:57 DEBUG 03:29:57 DEBUG 03:29:57 DEBUG [...] 03:29:57 DEBUG [...] 03:29:57 DEBUG [...] 03:29:57 DEBUG 03:29:57 DEBUG [...] 03:29:57 DEBUG 03:29:57 DEBUG 03:29:57 DEBUG [...] 03:29:57 DEBUG [...] 03:29:57 DEBUG [...] 03:29:57 DEBUG 03:29:57 DEBUG [...] 03:29:57 DEBUG 03:29:57 DEBUG 03:29:57 DEBUG 03:29:57 DEBUG 03:29:57 DEBUG 03:29:57 DEBUG 03:29:57 DEBUG 03:29:57 DEBUG 03:29:57 DEBUG 03:29:57 DEBUG "pod": "ds-idrepo-0" 03:29:57 DEBUG 03:29:57 DEBUG 03:29:57 DEBUG 03:29:57 DEBUG [...] 03:29:57 DEBUG 03:29:57 DEBUG [...] 03:29:57 DEBUG [...] 03:29:57 DEBUG 03:29:57 DEBUG [...] 03:29:57 DEBUG 03:29:57 DEBUG 03:29:57 DEBUG 03:29:57 DEBUG 03:29:57 DEBUG 03:29:57 DEBUG }, 03:29:57 DEBUG 03:29:57 DEBUG 03:29:57 DEBUG 03:29:57 DEBUG 03:29:57 DEBUG 03:29:57 DEBUG 03:29:57 DEBUG 03:29:57 DEBUG 03:29:57 DEBUG "pod": "am-767f6b67b5-vbltz" 03:29:57 DEBUG [...] 03:29:57 DEBUG 03:29:57 DEBUG 03:29:57 DEBUG 03:29:57 DEBUG 03:29:57 DEBUG "value": [ 03:29:57 DEBUG 03:29:57 DEBUG 03:29:57 DEBUG 03:29:57 DEBUG 03:29:57 DEBUG 03:29:57 DEBUG 03:29:57 DEBUG 03:29:57 DEBUG 03:29:57 DEBUG }, 03:29:57 DEBUG 03:29:57 DEBUG 03:29:57 DEBUG "pod": "ds-cts-2" 03:29:57 DEBUG 03:29:57 DEBUG [...] 03:29:57 DEBUG 03:29:57 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:29:57 DEBUG "pod": "am-767f6b67b5-vbltz" 03:29:57 DEBUG "instance": "10.142.0.43:9100" 03:29:57 DEBUG 03:29:57 DEBUG "pod": "ds-cts-2" 03:29:57 DEBUG 03:29:57 DEBUG 03:29:57 DEBUG 03:29:57 DEBUG "value": [ 03:29:57 DEBUG 03:29:57 DEBUG "pod": "am-767f6b67b5-vbltz" 03:29:57 DEBUG }, 03:29:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:29:57 DEBUG 03:29:57 DEBUG 03:29:57 DEBUG }, 03:29:57 DEBUG }, 03:29:57 DEBUG }, 03:29:57 DEBUG "instance": "10.142.0.43:9100" 03:29:57 DEBUG }, 03:29:57 DEBUG "pod": "ds-idrepo-2" 03:29:57 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:29:57 DEBUG "pod": "ds-cts-2" 03:29:57 DEBUG 1713666597, 03:29:57 DEBUG 03:29:57 DEBUG }, 03:29:57 DEBUG "value": [ 03:29:57 DEBUG }, 03:29:57 DEBUG 03:29:57 DEBUG 03:29:57 DEBUG "value": [ 03:29:57 DEBUG "value": [ 03:29:57 DEBUG "value": [ 03:29:57 DEBUG }, 03:29:57 DEBUG "value": [ 03:29:57 DEBUG }, 03:29:57 DEBUG }, 03:29:57 DEBUG }, 03:29:57 DEBUG "3849.1666666666665" 03:29:57 DEBUG "instance": "10.142.0.43:9100" 03:29:57 DEBUG "value": [ 03:29:57 DEBUG 1713666597, 03:29:57 DEBUG "value": [ 03:29:57 DEBUG 03:29:57 DEBUG [...] 03:29:57 DEBUG 1713666597, 03:29:57 DEBUG "0" 03:29:57 DEBUG 1713666597, 03:29:57 DEBUG 1713666597, 03:29:57 DEBUG 1713666597, 03:29:57 DEBUG "value": [ 03:29:57 DEBUG "value": [ 03:29:57 DEBUG "value": [ 03:29:57 DEBUG ] 03:29:57 DEBUG }, 03:29:57 DEBUG 1713666597, 03:29:57 DEBUG "0" 03:29:57 DEBUG 1713666597, 03:29:57 DEBUG "instance": "10.142.0.43:9100" 03:29:57 DEBUG 03:29:57 DEBUG ] 03:29:57 DEBUG "value": [ 03:29:57 DEBUG "101126278.43333334" 03:29:57 DEBUG "167.9995" 03:29:57 DEBUG "0" 03:29:57 DEBUG 1713666597, 03:29:57 DEBUG 1713666597, 03:29:57 DEBUG 1713666597, 03:29:57 DEBUG } 03:29:57 DEBUG "value": [ 03:29:57 DEBUG "3342.733333333333" 03:29:57 DEBUG ] 03:29:57 DEBUG "1162711040" 03:29:57 DEBUG }, 03:29:57 DEBUG 03:29:57 DEBUG } 03:29:57 DEBUG 1713666597, 03:29:57 DEBUG ] 03:29:57 DEBUG ] 03:29:57 DEBUG ] 03:29:57 DEBUG "0" 03:29:57 DEBUG "0" 03:29:57 DEBUG "31.766666666666666" 03:29:57 DEBUG ] 03:29:57 DEBUG 1713666597, 03:29:57 DEBUG ] 03:29:57 DEBUG } 03:29:57 DEBUG ] 03:29:57 DEBUG "value": [ 03:29:57 DEBUG 03:29:57 DEBUG ] 03:29:57 DEBUG "101121246.80000001" 03:29:57 DEBUG } 03:29:57 DEBUG } 03:29:57 DEBUG } 03:29:57 DEBUG ] 03:29:57 DEBUG ] 03:29:57 DEBUG } 03:29:57 DEBUG ] 03:29:57 DEBUG "0.08499999999999087" 03:29:57 DEBUG } 03:29:57 DEBUG ] 03:29:57 DEBUG } 03:29:57 DEBUG 1713666597, 03:29:57 DEBUG "pod": "ds-idrepo-2" 03:29:57 DEBUG } 03:29:57 DEBUG ] 03:29:57 DEBUG ] 03:29:57 DEBUG ] 03:29:57 DEBUG ] 03:29:57 DEBUG } 03:29:57 DEBUG } 03:29:57 DEBUG ] 03:29:57 DEBUG } 03:29:57 DEBUG } 03:29:57 DEBUG } 03:29:57 DEBUG ] 03:29:57 DEBUG ] 03:29:57 DEBUG "4447645789184" 03:29:57 DEBUG }, 03:29:57 DEBUG } 03:29:57 DEBUG } 03:29:57 DEBUG } 03:29:57 DEBUG } 03:29:57 DEBUG } 03:29:57 DEBUG ] 03:29:57 DEBUG ] 03:29:57 DEBUG } 03:29:57 DEBUG ] 03:29:57 DEBUG } 03:29:57 DEBUG } 03:29:57 DEBUG } 03:29:57 DEBUG ] 03:29:57 DEBUG "value": [ 03:29:57 DEBUG ] 03:29:57 DEBUG } 03:29:57 DEBUG } 03:29:57 DEBUG } 03:29:57 DEBUG } 03:29:57 DEBUG } 03:29:57 DEBUG } 03:29:57 DEBUG } 03:29:57 DEBUG } 03:29:57 DEBUG } 03:29:57 DEBUG } 03:29:57 DEBUG 1713666597, 03:29:57 DEBUG } 03:29:57 DEBUG } 03:29:57 DEBUG ] 03:29:57 DEBUG ] 03:29:57 DEBUG "0" 03:29:57 DEBUG } 03:29:57 DEBUG } 03:29:57 DEBUG } 03:29:57 DEBUG } 03:29:57 DEBUG ] 03:29:57 DEBUG } 03:29:57 DEBUG } 03:29:57 DEBUG } 03:29:57 DEBUG ] 03:29:57 DEBUG } 03:29:57 DEBUG } 03:30:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713666654" 03:30:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666654" 03:30:54 INFO [http_cmd]: http status code OK 03:30:54 DEBUG --- status code --- 03:30:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:30:54 ERROR [http_cmd]: ERROR 03:30:54 DEBUG --- http response --- 03:30:54 ERROR --- status code --- 03:30:54 ERROR http status code is 400 (expected [200, 401, 403, 404]) 03:30:54 ERROR --- http response --- 03:30:54 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 03:30:54 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 03:30:54 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 03:30:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666655" 03:30:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666655" 03:30:55 INFO [http_cmd]: http status code OK 03:30:55 DEBUG --- status code --- 03:30:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:30:55 DEBUG --- http response --- 03:30:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:30:55 DEBUG ----- output ----- 03:30:55 DEBUG { 03:30:55 DEBUG "status": "success", 03:30:55 DEBUG "data": { 03:30:55 DEBUG "resultType": "vector", 03:30:55 DEBUG "result": [ 03:30:55 DEBUG { 03:30:55 DEBUG "metric": { 03:30:55 DEBUG "pod": "ds-cts-0" 03:30:55 DEBUG }, 03:30:55 DEBUG "value": [ 03:30:55 DEBUG 03:30:55 DEBUG 03:30:55 DEBUG 03:30:55 DEBUG [...] 03:30:55 DEBUG 03:30:55 DEBUG 03:30:55 DEBUG 03:30:55 DEBUG "pod": "ds-cts-2" 03:30:55 DEBUG }, 03:30:55 DEBUG "value": [ 03:30:55 DEBUG 1713666655, 03:30:55 DEBUG "0.5441333333333356" 03:30:55 DEBUG ] 03:30:55 DEBUG } 03:30:55 DEBUG ] 03:30:55 DEBUG } 03:30:55 DEBUG } 03:30:55 INFO [http_cmd]: http status code OK 03:30:55 DEBUG --- status code --- 03:30:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:30:55 DEBUG --- http response --- 03:30:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:30:55 DEBUG ----- output ----- 03:30:55 DEBUG { 03:30:55 DEBUG "status": "success", 03:30:55 DEBUG "data": { 03:30:55 DEBUG "resultType": "vector", 03:30:55 DEBUG "result": [ 03:30:55 DEBUG { 03:30:55 DEBUG "metric": { 03:30:55 DEBUG "instance": "10.142.0.107:9100" 03:30:55 DEBUG }, 03:30:55 DEBUG "value": [ 03:30:55 DEBUG 03:30:55 DEBUG 03:30:55 DEBUG 03:30:55 DEBUG [...] 03:30:55 DEBUG 03:30:55 DEBUG 03:30:55 DEBUG 03:30:55 DEBUG "instance": "10.142.0.43:9100" 03:30:55 DEBUG }, 03:30:55 DEBUG "value": [ 03:30:55 DEBUG 1713666655, 03:30:55 DEBUG "0.3828666666666038" 03:30:55 DEBUG ] 03:30:55 DEBUG } 03:30:55 DEBUG ] 03:30:55 DEBUG } 03:30:55 DEBUG } 03:30:56 INFO 03:30:56 INFO [loop_until]: kubectl --namespace=xlou top pods 03:30:56 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:30:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666656" 03:30:56 INFO [http_cmd]: http status code OK 03:30:56 DEBUG --- status code --- 03:30:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:30:56 DEBUG --- http response --- 03:30:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:30:56 DEBUG ----- output ----- 03:30:56 DEBUG { 03:30:56 DEBUG "status": "success", 03:30:56 DEBUG "data": { 03:30:56 DEBUG "resultType": "vector", 03:30:56 DEBUG "result": [ 03:30:56 DEBUG { 03:30:56 DEBUG "metric": { 03:30:56 DEBUG "pod": "am-767f6b67b5-9zw22" 03:30:56 DEBUG }, 03:30:56 DEBUG "value": [ 03:30:56 DEBUG 03:30:56 DEBUG 03:30:56 DEBUG 03:30:56 DEBUG [...] 03:30:56 DEBUG 03:30:56 DEBUG 03:30:56 DEBUG 03:30:56 DEBUG "pod": "am-767f6b67b5-vbltz" 03:30:56 DEBUG }, 03:30:56 DEBUG "value": [ 03:30:56 DEBUG 1713666656, 03:30:56 DEBUG "3336.633333333333" 03:30:56 DEBUG ] 03:30:56 DEBUG } 03:30:56 DEBUG ] 03:30:56 DEBUG } 03:30:56 DEBUG } 03:30:56 INFO [loop_until]: OK (rc = 0) 03:30:56 DEBUG --- stdout --- 03:30:56 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 22640m 8803Mi am-767f6b67b5-qcqz2 22504m 9144Mi am-767f6b67b5-vbltz 24694m 8913Mi ds-cts-0 21114m 12537Mi ds-cts-1 19773m 12489Mi ds-cts-2 20466m 12572Mi ds-idrepo-0 9530m 3919Mi ds-idrepo-1 9532m 3962Mi ds-idrepo-2 9359m 3916Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 3m 1179Mi idm-b85f9d978-wnhxb 3m 1239Mi lodemon-54c87f8cbb-jzrjs 16m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-68c5d5b884-p97w5 12156m 1019Mi 03:30:56 DEBUG --- stderr --- 03:30:56 DEBUG 03:30:57 INFO 03:30:57 INFO [loop_until]: kubectl --namespace=xlou top node 03:30:57 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:30:57 INFO [loop_until]: OK (rc = 0) 03:30:57 DEBUG --- stdout --- 03:30:57 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 62m 0% 6287Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 9866m 33% 7756Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 22434m 75% 11610Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 87m 0% 3353Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 20226m 67% 15856Mi 14% gke-xlou-cdm-default-pool-1b0769c8-wpwl 157m 0% 5281Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 22808m 76% 12094Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 19873m 66% 3216Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 25095m 84% 11188Mi 10% gke-xlou-cdm-ds-0f601751-j488 9942m 33% 7363Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 19737m 66% 17612Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 9866m 33% 8026Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 19608m 65% 15277Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 12126m 76% 11324Mi 19% 03:30:57 DEBUG --- stderr --- 03:30:57 DEBUG 03:30:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666657" 03:30:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666657" 03:30:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666657" 03:30:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666657" 03:30:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713666657" 03:30:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666657" 03:30:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666657" 03:30:57 INFO [http_cmd]: http status code OK 03:30:57 DEBUG --- status code --- 03:30:57 INFO [http_cmd]: http status code OK 03:30:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:30:57 INFO [http_cmd]: http status code OK 03:30:57 INFO [http_cmd]: http status code OK 03:30:57 DEBUG --- status code --- 03:30:57 DEBUG --- http response --- 03:30:57 DEBUG --- status code --- 03:30:57 DEBUG --- status code --- 03:30:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:30:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:30:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:30:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:30:57 DEBUG --- http response --- 03:30:57 DEBUG ----- output ----- 03:30:57 DEBUG --- http response --- 03:30:57 DEBUG --- http response --- 03:30:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:30:57 DEBUG ----- output ----- 03:30:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:30:57 DEBUG { 03:30:57 DEBUG { 03:30:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:30:57 DEBUG ----- output ----- 03:30:57 DEBUG "status": "success", 03:30:57 DEBUG "status": "success", 03:30:57 DEBUG "data": { 03:30:57 DEBUG { 03:30:57 DEBUG "data": { 03:30:57 INFO [http_cmd]: http status code OK 03:30:57 DEBUG "resultType": "vector", 03:30:57 DEBUG "result": [ 03:30:57 DEBUG ----- output ----- 03:30:57 DEBUG "resultType": "vector", 03:30:57 DEBUG --- status code --- 03:30:57 INFO [http_cmd]: http status code OK 03:30:57 DEBUG { 03:30:57 DEBUG "status": "success", 03:30:57 INFO [http_cmd]: http status code OK 03:30:57 DEBUG { 03:30:57 DEBUG "result": [ 03:30:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:30:57 DEBUG --- status code --- 03:30:57 DEBUG "metric": { 03:30:57 DEBUG "data": { 03:30:57 DEBUG --- status code --- 03:30:57 DEBUG "status": "success", 03:30:57 DEBUG { 03:30:57 DEBUG --- http response --- 03:30:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:30:57 DEBUG "pod": "ds-idrepo-0" 03:30:57 DEBUG "resultType": "vector", 03:30:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:30:57 DEBUG "data": { 03:30:57 DEBUG "metric": { 03:30:57 DEBUG --- http response --- 03:30:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:30:57 DEBUG }, 03:30:57 DEBUG "value": [ 03:30:57 DEBUG --- http response --- 03:30:57 DEBUG "resultType": "vector", 03:30:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:30:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:30:57 DEBUG ----- output ----- 03:30:57 DEBUG 03:30:57 DEBUG 03:30:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:30:57 DEBUG "result": [ 03:30:57 DEBUG }, 03:30:57 DEBUG ----- output ----- 03:30:57 DEBUG { 03:30:57 DEBUG "status": "success", 03:30:57 DEBUG "data": { 03:30:57 DEBUG "result": [ 03:30:57 DEBUG { 03:30:57 DEBUG "value": [ 03:30:57 DEBUG { 03:30:57 DEBUG "resultType": "vector", 03:30:57 DEBUG "result": [ 03:30:57 DEBUG { 03:30:57 DEBUG ----- output ----- 03:30:57 DEBUG "metric": { 03:30:57 DEBUG 03:30:57 DEBUG "status": "success", 03:30:57 DEBUG 03:30:57 DEBUG { 03:30:57 DEBUG "metric": { 03:30:57 DEBUG { 03:30:57 DEBUG "pod": "am-767f6b67b5-9zw22" 03:30:57 DEBUG 03:30:57 DEBUG 03:30:57 DEBUG [...] 03:30:57 DEBUG "metric": { 03:30:57 DEBUG "pod": "am-767f6b67b5-9zw22" 03:30:57 DEBUG "status": "success", 03:30:57 DEBUG }, 03:30:57 DEBUG [...] 03:30:57 DEBUG "data": { 03:30:57 DEBUG 03:30:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:30:57 DEBUG }, 03:30:57 DEBUG "data": { 03:30:57 DEBUG "value": [ 03:30:57 DEBUG 03:30:57 DEBUG "resultType": "vector", 03:30:57 DEBUG 03:30:57 DEBUG 03:30:57 DEBUG "value": [ 03:30:57 DEBUG 03:30:57 DEBUG 03:30:57 DEBUG 03:30:57 DEBUG 03:30:57 DEBUG }, 03:30:57 DEBUG "pod": "ds-idrepo-2" 03:30:57 DEBUG 03:30:57 DEBUG "resultType": "vector", 03:30:57 DEBUG 03:30:57 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:30:57 DEBUG "result": [ 03:30:57 DEBUG "value": [ 03:30:57 DEBUG }, 03:30:57 DEBUG 03:30:57 DEBUG "result": [ 03:30:57 DEBUG 03:30:57 DEBUG }, 03:30:57 DEBUG { 03:30:57 DEBUG 03:30:57 DEBUG "value": [ 03:30:57 DEBUG [...] 03:30:57 DEBUG { 03:30:57 DEBUG [...] 03:30:57 DEBUG "value": [ 03:30:57 DEBUG "metric": { 03:30:57 DEBUG 03:30:57 DEBUG 1713666657, 03:30:57 DEBUG 03:30:57 DEBUG "metric": { 03:30:57 DEBUG 03:30:57 DEBUG 1713666657, 03:30:57 DEBUG "pod": "am-767f6b67b5-9zw22" 03:30:57 DEBUG 03:30:57 DEBUG "0" 03:30:57 DEBUG 03:30:57 DEBUG 03:30:57 DEBUG "pod": "am-767f6b67b5-vbltz" 03:30:57 DEBUG "11.863510925010187" 03:30:57 DEBUG }, 03:30:57 DEBUG [...] 03:30:57 DEBUG ] 03:30:57 DEBUG } 03:30:57 DEBUG ] 03:30:57 DEBUG } 03:30:57 DEBUG "value": [ 03:30:57 DEBUG 03:30:57 DEBUG 03:30:57 DEBUG }, 03:30:57 DEBUG ] 03:30:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:30:57 DEBUG 03:30:57 DEBUG 03:30:57 DEBUG [...] 03:30:57 DEBUG 03:30:57 DEBUG "value": [ 03:30:57 DEBUG } 03:30:57 DEBUG }, 03:30:57 DEBUG 03:30:57 DEBUG 03:30:57 DEBUG } 03:30:57 DEBUG 03:30:57 DEBUG 03:30:57 DEBUG ] 03:30:57 DEBUG "value": [ 03:30:57 DEBUG 03:30:57 DEBUG 03:30:57 DEBUG 1713666657, 03:30:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:30:57 DEBUG } 03:30:57 DEBUG 03:30:57 DEBUG 03:30:57 DEBUG [...] 03:30:57 DEBUG 03:30:57 DEBUG }, 03:30:57 DEBUG } 03:30:57 DEBUG "3342.5333333333333" 03:30:57 DEBUG 03:30:57 DEBUG "pod": "am-767f6b67b5-vbltz" 03:30:57 DEBUG "pod": "am-767f6b67b5-vbltz" 03:30:57 DEBUG "value": [ 03:30:57 DEBUG ] 03:30:57 DEBUG 03:30:57 DEBUG }, 03:30:57 DEBUG }, 03:30:57 DEBUG 1713666657, 03:30:57 DEBUG } 03:30:57 DEBUG 03:30:57 DEBUG "value": [ 03:30:57 DEBUG "value": [ 03:30:57 DEBUG "11.903439791123805" 03:30:57 DEBUG ] 03:30:57 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:30:57 DEBUG 1713666657, 03:30:57 DEBUG 1713666657, 03:30:57 DEBUG ] 03:30:57 DEBUG } 03:30:57 DEBUG }, 03:30:57 DEBUG "600108.5" 03:30:57 DEBUG "0" 03:30:57 DEBUG } 03:30:57 DEBUG } 03:30:57 DEBUG "value": [ 03:30:57 DEBUG ] 03:30:57 DEBUG ] 03:30:57 DEBUG ] 03:30:57 DEBUG 1713666657, 03:30:57 DEBUG } 03:30:57 DEBUG } 03:30:57 DEBUG ] 03:30:57 DEBUG "3848336.170787731" 03:30:57 DEBUG ] 03:30:57 DEBUG } 03:30:57 DEBUG } 03:30:57 DEBUG ] 03:30:57 DEBUG } 03:30:57 DEBUG } 03:30:57 DEBUG } 03:30:57 DEBUG } 03:30:57 DEBUG } 03:30:57 DEBUG ] 03:30:57 DEBUG } 03:30:57 DEBUG } 03:30:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666657" 03:30:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666657" 03:30:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666657" 03:30:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713666657" 03:30:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666657" 03:30:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666657" 03:30:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666657" 03:30:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713666657" 03:30:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713666657" 03:30:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666657" 03:30:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666657" 03:30:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666657" 03:30:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666657" 03:30:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666657" 03:30:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666657" 03:30:57 INFO [http_cmd]: http status code OK 03:30:57 INFO [http_cmd]: http status code OK 03:30:57 INFO [http_cmd]: http status code OK 03:30:57 INFO [http_cmd]: http status code OK 03:30:57 INFO [http_cmd]: http status code OK 03:30:57 INFO [http_cmd]: http status code OK 03:30:57 INFO [http_cmd]: http status code OK 03:30:57 INFO [http_cmd]: http status code OK 03:30:57 DEBUG --- status code --- 03:30:57 INFO [http_cmd]: http status code OK 03:30:57 INFO [http_cmd]: http status code OK 03:30:57 DEBUG --- status code --- 03:30:57 DEBUG --- status code --- 03:30:57 INFO [http_cmd]: http status code OK 03:30:57 INFO [http_cmd]: http status code OK 03:30:57 DEBUG --- status code --- 03:30:57 DEBUG --- status code --- 03:30:57 DEBUG --- status code --- 03:30:57 DEBUG --- status code --- 03:30:57 DEBUG --- status code --- 03:30:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:30:57 DEBUG --- status code --- 03:30:57 DEBUG --- status code --- 03:30:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:30:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:30:57 DEBUG --- status code --- 03:30:57 DEBUG --- status code --- 03:30:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:30:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:30:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:30:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:30:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:30:57 INFO [http_cmd]: http status code OK 03:30:57 DEBUG --- http response --- 03:30:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:30:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:30:57 DEBUG --- http response --- 03:30:57 DEBUG --- http response --- 03:30:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:30:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:30:57 DEBUG --- http response --- 03:30:57 DEBUG --- http response --- 03:30:57 DEBUG --- http response --- 03:30:57 DEBUG --- http response --- 03:30:57 DEBUG --- http response --- 03:30:57 DEBUG --- status code --- 03:30:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:30:57 DEBUG --- http response --- 03:30:57 DEBUG --- http response --- 03:30:57 INFO [http_cmd]: http status code OK 03:30:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:30:57 DEBUG --- http response --- 03:30:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:30:57 DEBUG --- http response --- 03:30:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:30:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:30:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:30:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:30:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:30:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:30:57 DEBUG ----- output ----- 03:30:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:30:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:30:58 DEBUG --- status code --- 03:30:58 DEBUG ----- output ----- 03:30:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:30:58 DEBUG ----- output ----- 03:30:58 INFO [http_cmd]: http status code OK 03:30:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:30:58 DEBUG ----- output ----- 03:30:58 DEBUG ----- output ----- 03:30:58 DEBUG ----- output ----- 03:30:58 DEBUG ----- output ----- 03:30:58 DEBUG ----- output ----- 03:30:58 DEBUG --- http response --- 03:30:58 DEBUG { 03:30:58 DEBUG ----- output ----- 03:30:58 DEBUG ----- output ----- 03:30:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:30:58 DEBUG { 03:30:58 DEBUG ----- output ----- 03:30:58 DEBUG { 03:30:58 DEBUG --- status code --- 03:30:58 DEBUG ----- output ----- 03:30:58 DEBUG { 03:30:58 DEBUG { 03:30:58 DEBUG { 03:30:58 DEBUG { 03:30:58 DEBUG { 03:30:58 DEBUG "status": "success", 03:30:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:30:58 DEBUG { 03:30:58 DEBUG { 03:30:58 DEBUG --- http response --- 03:30:58 DEBUG "status": "success", 03:30:58 DEBUG { 03:30:58 DEBUG "status": "success", 03:30:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:30:58 DEBUG { 03:30:58 DEBUG "status": "success", 03:30:58 DEBUG "status": "success", 03:30:58 DEBUG "status": "success", 03:30:58 DEBUG "status": "success", 03:30:58 DEBUG "status": "success", 03:30:58 DEBUG "data": { 03:30:58 DEBUG ----- output ----- 03:30:58 DEBUG "status": "success", 03:30:58 DEBUG "data": { 03:30:58 DEBUG "data": { 03:30:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:30:58 DEBUG ----- output ----- 03:30:58 DEBUG { 03:30:58 DEBUG --- http response --- 03:30:58 DEBUG "status": "success", 03:30:58 DEBUG "data": { 03:30:58 DEBUG "data": { 03:30:58 DEBUG "data": { 03:30:58 DEBUG "data": { 03:30:58 DEBUG "data": { 03:30:58 DEBUG "resultType": "vector", 03:30:58 DEBUG { 03:30:58 DEBUG "status": "success", 03:30:58 DEBUG "resultType": "vector", 03:30:58 DEBUG "resultType": "vector", 03:30:58 DEBUG "status": "success", 03:30:58 DEBUG "status": "success", 03:30:58 DEBUG "data": { 03:30:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:30:58 DEBUG "data": { 03:30:58 DEBUG "resultType": "vector", 03:30:58 DEBUG "resultType": "vector", 03:30:58 DEBUG "resultType": "vector", 03:30:58 DEBUG "resultType": "vector", 03:30:58 DEBUG "resultType": "vector", 03:30:58 DEBUG "result": [ 03:30:58 DEBUG "status": "success", 03:30:58 DEBUG "data": { 03:30:58 DEBUG "result": [ 03:30:58 DEBUG "result": [ 03:30:58 DEBUG "data": { 03:30:58 DEBUG "data": { 03:30:58 DEBUG "resultType": "vector", 03:30:58 DEBUG ----- output ----- 03:30:58 DEBUG { 03:30:58 DEBUG "result": [ 03:30:58 DEBUG "result": [ 03:30:58 DEBUG "result": [ 03:30:58 DEBUG "result": [ 03:30:58 DEBUG "result": [ 03:30:58 DEBUG { 03:30:58 DEBUG "data": { 03:30:58 DEBUG "resultType": "vector", 03:30:58 DEBUG { 03:30:58 DEBUG { 03:30:58 DEBUG "resultType": "vector", 03:30:58 DEBUG "resultType": "vector", 03:30:58 DEBUG "result": [ 03:30:58 DEBUG "status": "success", 03:30:58 DEBUG "resultType": "vector", 03:30:58 DEBUG { 03:30:58 DEBUG "metric": { 03:30:58 DEBUG { 03:30:58 DEBUG { 03:30:58 DEBUG { 03:30:58 DEBUG "metric": { 03:30:58 DEBUG "resultType": "vector", 03:30:58 DEBUG "result": [ 03:30:58 DEBUG "metric": { 03:30:58 DEBUG "metric": { 03:30:58 DEBUG "result": [ 03:30:58 DEBUG "result": [ 03:30:58 DEBUG { 03:30:58 DEBUG "data": { 03:30:58 DEBUG "result": [ 03:30:58 DEBUG "pod": "am-767f6b67b5-9zw22" 03:30:58 DEBUG { 03:30:58 DEBUG "metric": { 03:30:58 DEBUG "metric": { 03:30:58 DEBUG "metric": { 03:30:58 DEBUG "pod": "am-767f6b67b5-9zw22" 03:30:58 DEBUG "result": [ 03:30:58 DEBUG { 03:30:58 DEBUG "instance": "10.142.0.107:9100" 03:30:58 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:30:58 DEBUG { 03:30:58 DEBUG { 03:30:58 DEBUG "metric": { 03:30:58 DEBUG "resultType": "vector", 03:30:58 DEBUG { 03:30:58 DEBUG }, 03:30:58 DEBUG "metric": { 03:30:58 DEBUG "pod": "ds-idrepo-0" 03:30:58 DEBUG "instance": "10.142.0.107:9100" 03:30:58 DEBUG "pod": "ds-cts-0" 03:30:58 DEBUG }, 03:30:58 DEBUG { 03:30:58 DEBUG "metric": { 03:30:58 DEBUG }, 03:30:58 DEBUG }, 03:30:58 DEBUG "value": [ 03:30:58 DEBUG "metric": { 03:30:58 DEBUG "pod": "am-767f6b67b5-9zw22" 03:30:58 DEBUG }, 03:30:58 DEBUG "metric": { 03:30:58 DEBUG "value": [ 03:30:58 DEBUG "pod": "ds-cts-0" 03:30:58 DEBUG }, 03:30:58 DEBUG }, 03:30:58 DEBUG }, 03:30:58 DEBUG "value": [ 03:30:58 DEBUG "metric": { 03:30:58 DEBUG "pod": "ds-idrepo-0" 03:30:58 DEBUG "value": [ 03:30:58 DEBUG 03:30:58 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:30:58 DEBUG "metric": { 03:30:58 DEBUG "value": [ 03:30:58 DEBUG "result": [ 03:30:58 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:30:58 DEBUG 03:30:58 DEBUG }, 03:30:58 DEBUG "value": [ 03:30:58 DEBUG "value": [ 03:30:58 DEBUG "value": [ 03:30:58 DEBUG 03:30:58 DEBUG "instance": "10.142.0.107:9100" 03:30:58 DEBUG }, 03:30:58 DEBUG 03:30:58 DEBUG 03:30:58 DEBUG }, 03:30:58 DEBUG "instance": "10.142.0.107:9100" 03:30:58 DEBUG 03:30:58 DEBUG { 03:30:58 DEBUG }, 03:30:58 DEBUG 03:30:58 DEBUG "value": [ 03:30:58 DEBUG 03:30:58 DEBUG 03:30:58 DEBUG 03:30:58 DEBUG 03:30:58 DEBUG }, 03:30:58 DEBUG "value": [ 03:30:58 DEBUG 03:30:58 DEBUG 03:30:58 DEBUG "value": [ 03:30:58 DEBUG }, 03:30:58 DEBUG 03:30:58 DEBUG "metric": { 03:30:58 DEBUG "value": [ 03:30:58 DEBUG 03:30:58 DEBUG 03:30:58 DEBUG 03:30:58 DEBUG 03:30:58 DEBUG 03:30:58 DEBUG 03:30:58 DEBUG "value": [ 03:30:58 DEBUG 03:30:58 DEBUG 03:30:58 DEBUG [...] 03:30:58 DEBUG 03:30:58 DEBUG "value": [ 03:30:58 DEBUG 03:30:58 DEBUG [...] 03:30:58 DEBUG 03:30:58 DEBUG 03:30:58 DEBUG 03:30:58 DEBUG 03:30:58 DEBUG 03:30:58 DEBUG 03:30:58 DEBUG [...] 03:30:58 DEBUG 03:30:58 DEBUG 03:30:58 DEBUG 03:30:58 DEBUG 03:30:58 DEBUG [...] 03:30:58 DEBUG 03:30:58 DEBUG 03:30:58 DEBUG "pod": "ds-idrepo-0" 03:30:58 DEBUG 03:30:58 DEBUG 03:30:58 DEBUG [...] 03:30:58 DEBUG [...] 03:30:58 DEBUG [...] 03:30:58 DEBUG [...] 03:30:58 DEBUG 03:30:58 DEBUG 03:30:58 DEBUG [...] 03:30:58 DEBUG 03:30:58 DEBUG 03:30:58 DEBUG 03:30:58 DEBUG 03:30:58 DEBUG 03:30:58 DEBUG }, 03:30:58 DEBUG [...] 03:30:58 DEBUG [...] 03:30:58 DEBUG 03:30:58 DEBUG 03:30:58 DEBUG 03:30:58 DEBUG 03:30:58 DEBUG 03:30:58 DEBUG 03:30:58 DEBUG 03:30:58 DEBUG 03:30:58 DEBUG 03:30:58 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:30:58 DEBUG 03:30:58 DEBUG 03:30:58 DEBUG "value": [ 03:30:58 DEBUG 03:30:58 DEBUG 03:30:58 DEBUG 03:30:58 DEBUG 03:30:58 DEBUG 03:30:58 DEBUG 03:30:58 DEBUG 03:30:58 DEBUG [...] 03:30:58 DEBUG 03:30:58 DEBUG [...] 03:30:58 DEBUG 03:30:58 DEBUG }, 03:30:58 DEBUG [...] 03:30:58 DEBUG 03:30:58 DEBUG 03:30:58 DEBUG 03:30:58 DEBUG 03:30:58 DEBUG 03:30:58 DEBUG 03:30:58 DEBUG 03:30:58 DEBUG 03:30:58 DEBUG "pod": "am-767f6b67b5-vbltz" 03:30:58 DEBUG 03:30:58 DEBUG 03:30:58 DEBUG 03:30:58 DEBUG 03:30:58 DEBUG "value": [ 03:30:58 DEBUG "pod": "am-767f6b67b5-vbltz" 03:30:58 DEBUG 03:30:58 DEBUG 03:30:58 DEBUG 03:30:58 DEBUG 03:30:58 DEBUG "pod": "ds-cts-2" 03:30:58 DEBUG "instance": "10.142.0.43:9100" 03:30:58 DEBUG }, 03:30:58 DEBUG "pod": "ds-cts-2" 03:30:58 DEBUG }, 03:30:58 DEBUG 03:30:58 DEBUG 03:30:58 DEBUG 03:30:58 DEBUG "instance": "10.142.0.43:9100" 03:30:58 DEBUG 1713666657, 03:30:58 DEBUG }, 03:30:58 DEBUG "value": [ 03:30:58 DEBUG 03:30:58 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:30:58 DEBUG "pod": "ds-cts-2" 03:30:58 DEBUG }, 03:30:58 DEBUG "value": [ 03:30:58 DEBUG }, 03:30:58 DEBUG "pod": "am-767f6b67b5-vbltz" 03:30:58 DEBUG "value": [ 03:30:58 DEBUG 1713666657, 03:30:58 DEBUG "instance": "10.142.0.43:9100" 03:30:58 DEBUG 03:30:58 DEBUG }, 03:30:58 DEBUG "276.7193622483448" 03:30:58 DEBUG 03:30:58 DEBUG 1713666657, 03:30:58 DEBUG [...] 03:30:58 DEBUG }, 03:30:58 DEBUG "value": [ 03:30:58 DEBUG }, 03:30:58 DEBUG 1713666657, 03:30:58 DEBUG "value": [ 03:30:58 DEBUG }, 03:30:58 DEBUG "value": [ 03:30:58 DEBUG "pod": "ds-idrepo-2" 03:30:58 DEBUG "3682.166666666667" 03:30:58 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:30:58 DEBUG "value": [ 03:30:58 DEBUG ] 03:30:58 DEBUG "instance": "10.142.0.43:9100" 03:30:58 DEBUG "3335.9666666666667" 03:30:58 DEBUG 03:30:58 DEBUG "value": [ 03:30:58 DEBUG 1713666657, 03:30:58 DEBUG "value": [ 03:30:58 DEBUG "102026527.73333333" 03:30:58 DEBUG 1713666657, 03:30:58 DEBUG 1713666657, 03:30:58 DEBUG }, 03:30:58 DEBUG "value": [ 03:30:58 DEBUG ] 03:30:58 DEBUG }, 03:30:58 DEBUG 1713666657, 03:30:58 DEBUG } 03:30:58 DEBUG }, 03:30:58 DEBUG ] 03:30:58 DEBUG 03:30:58 DEBUG "1277456384" 03:30:58 DEBUG 1713666657, 03:30:58 DEBUG 1713666657, 03:30:58 DEBUG ] 03:30:58 DEBUG "30.233333333333334" 03:30:58 DEBUG "172.4975" 03:30:58 DEBUG }, 03:30:58 DEBUG 1713666657, 03:30:58 DEBUG } 03:30:58 DEBUG "value": [ 03:30:58 DEBUG "4452645989376" 03:30:58 DEBUG ] 03:30:58 DEBUG "value": [ 03:30:58 DEBUG 1713666657, 03:30:58 DEBUG "0.07066666666666871" 03:30:58 DEBUG ] 03:30:58 DEBUG "0" 03:30:58 DEBUG "0" 03:30:58 DEBUG } 03:30:58 DEBUG ] 03:30:58 DEBUG ] 03:30:58 DEBUG "value": [ 03:30:58 DEBUG "102021487.53333333" 03:30:58 DEBUG ] 03:30:58 DEBUG 1713666657, 03:30:58 DEBUG ] 03:30:58 DEBUG } 03:30:58 DEBUG 03:30:58 DEBUG "pod": "ds-idrepo-2" 03:30:58 DEBUG } 03:30:58 DEBUG } 03:30:58 DEBUG ] 03:30:58 DEBUG ] 03:30:58 DEBUG ] 03:30:58 DEBUG } 03:30:58 DEBUG } 03:30:58 DEBUG 1713666657, 03:30:58 DEBUG ] 03:30:58 DEBUG } 03:30:58 DEBUG "5672.746926091068" 03:30:58 DEBUG } 03:30:58 DEBUG } 03:30:58 DEBUG }, 03:30:58 DEBUG ] 03:30:58 DEBUG ] 03:30:58 DEBUG ] 03:30:58 DEBUG } 03:30:58 DEBUG } 03:30:58 DEBUG } 03:30:58 DEBUG ] 03:30:58 DEBUG } 03:30:58 DEBUG } 03:30:58 DEBUG } 03:30:58 DEBUG } 03:30:58 DEBUG ] 03:30:58 DEBUG ] 03:30:58 DEBUG "value": [ 03:30:58 DEBUG } 03:30:58 DEBUG } 03:30:58 DEBUG } 03:30:58 DEBUG } 03:30:58 DEBUG } 03:30:58 DEBUG ] 03:30:58 DEBUG "0" 03:30:58 DEBUG ] 03:30:58 DEBUG } 03:30:58 DEBUG } 03:30:58 DEBUG } 03:30:58 DEBUG } 03:30:58 DEBUG ] 03:30:58 DEBUG ] 03:30:58 DEBUG } 03:30:58 DEBUG ] 03:30:58 DEBUG } 03:30:58 DEBUG ] 03:30:58 DEBUG 1713666657, 03:30:58 DEBUG } 03:30:58 DEBUG ] 03:30:58 DEBUG } 03:30:58 DEBUG } 03:30:58 DEBUG } 03:30:58 DEBUG } 03:30:58 DEBUG } 03:30:58 DEBUG "0" 03:30:58 DEBUG } 03:30:58 DEBUG } 03:30:58 DEBUG ] 03:30:58 DEBUG } 03:30:58 DEBUG } 03:30:58 DEBUG ] 03:30:58 DEBUG } 03:30:58 DEBUG } 03:30:58 DEBUG } 03:30:58 DEBUG } 03:30:58 DEBUG ] 03:30:58 DEBUG } 03:30:58 DEBUG } 03:31:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666714" 03:31:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713666714" 03:31:54 ERROR [http_cmd]: ERROR 03:31:54 ERROR --- status code --- 03:31:54 ERROR http status code is 400 (expected [200, 401, 403, 404]) 03:31:54 ERROR --- http response --- 03:31:54 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 03:31:54 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 03:31:54 INFO [http_cmd]: http status code OK 03:31:54 DEBUG --- status code --- 03:31:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:31:54 DEBUG --- http response --- 03:31:54 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 03:31:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666715" 03:31:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666715" 03:31:55 INFO [http_cmd]: http status code OK 03:31:55 DEBUG --- status code --- 03:31:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:31:55 DEBUG --- http response --- 03:31:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:31:55 DEBUG ----- output ----- 03:31:55 DEBUG { 03:31:55 DEBUG "status": "success", 03:31:55 DEBUG "data": { 03:31:55 DEBUG "resultType": "vector", 03:31:55 DEBUG "result": [ 03:31:55 DEBUG { 03:31:55 DEBUG "metric": { 03:31:55 DEBUG "pod": "ds-cts-0" 03:31:55 DEBUG }, 03:31:55 DEBUG "value": [ 03:31:55 DEBUG 03:31:55 DEBUG 03:31:55 DEBUG 03:31:55 DEBUG [...] 03:31:55 DEBUG 03:31:55 DEBUG 03:31:55 DEBUG 03:31:55 DEBUG "pod": "ds-cts-2" 03:31:55 DEBUG }, 03:31:55 DEBUG "value": [ 03:31:55 DEBUG 1713666715, 03:31:55 DEBUG "0.4059666666666677" 03:31:55 DEBUG ] 03:31:55 DEBUG } 03:31:55 DEBUG ] 03:31:55 DEBUG } 03:31:55 DEBUG } 03:31:55 INFO [http_cmd]: http status code OK 03:31:55 DEBUG --- status code --- 03:31:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:31:55 DEBUG --- http response --- 03:31:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:31:55 DEBUG ----- output ----- 03:31:55 DEBUG { 03:31:55 DEBUG "status": "success", 03:31:55 DEBUG "data": { 03:31:55 DEBUG "resultType": "vector", 03:31:55 DEBUG "result": [ 03:31:55 DEBUG { 03:31:55 DEBUG "metric": { 03:31:55 DEBUG "instance": "10.142.0.107:9100" 03:31:55 DEBUG }, 03:31:55 DEBUG "value": [ 03:31:55 DEBUG 03:31:55 DEBUG 03:31:55 DEBUG 03:31:55 DEBUG [...] 03:31:55 DEBUG 03:31:55 DEBUG 03:31:55 DEBUG 03:31:55 DEBUG "instance": "10.142.0.43:9100" 03:31:55 DEBUG }, 03:31:55 DEBUG "value": [ 03:31:55 DEBUG 1713666715, 03:31:55 DEBUG "0.4064333333333081" 03:31:55 DEBUG ] 03:31:55 DEBUG } 03:31:55 DEBUG ] 03:31:55 DEBUG } 03:31:55 DEBUG } 03:31:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666716" 03:31:56 INFO 03:31:56 INFO [loop_until]: kubectl --namespace=xlou top pods 03:31:56 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:31:56 INFO [http_cmd]: http status code OK 03:31:56 DEBUG --- status code --- 03:31:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:31:56 DEBUG --- http response --- 03:31:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:31:56 DEBUG ----- output ----- 03:31:56 DEBUG { 03:31:56 DEBUG "status": "success", 03:31:56 DEBUG "data": { 03:31:56 DEBUG "resultType": "vector", 03:31:56 DEBUG "result": [ 03:31:56 DEBUG { 03:31:56 DEBUG "metric": { 03:31:56 DEBUG "pod": "am-767f6b67b5-9zw22" 03:31:56 DEBUG }, 03:31:56 DEBUG "value": [ 03:31:56 DEBUG 03:31:56 DEBUG 03:31:56 DEBUG 03:31:56 DEBUG [...] 03:31:56 DEBUG 03:31:56 DEBUG 03:31:56 DEBUG 03:31:56 DEBUG "pod": "am-767f6b67b5-vbltz" 03:31:56 DEBUG }, 03:31:56 DEBUG "value": [ 03:31:56 DEBUG 1713666716, 03:31:56 DEBUG "2707.5" 03:31:56 DEBUG ] 03:31:56 DEBUG } 03:31:56 DEBUG ] 03:31:56 DEBUG } 03:31:56 DEBUG } 03:31:56 INFO [loop_until]: OK (rc = 0) 03:31:56 DEBUG --- stdout --- 03:31:56 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 22205m 8802Mi am-767f6b67b5-qcqz2 22140m 9142Mi am-767f6b67b5-vbltz 24246m 8895Mi ds-cts-0 20458m 12531Mi ds-cts-1 19522m 12487Mi ds-cts-2 20325m 12573Mi ds-idrepo-0 9236m 3918Mi ds-idrepo-1 9748m 3963Mi ds-idrepo-2 9684m 3915Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 3m 1179Mi idm-b85f9d978-wnhxb 3m 1239Mi lodemon-54c87f8cbb-jzrjs 20m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-68c5d5b884-p97w5 11730m 1120Mi 03:31:56 DEBUG --- stderr --- 03:31:56 DEBUG 03:31:57 INFO 03:31:57 INFO [loop_until]: kubectl --namespace=xlou top node 03:31:57 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:31:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666717" 03:31:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666717" 03:31:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666717" 03:31:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666717" 03:31:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666717" 03:31:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666717" 03:31:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713666717" 03:31:57 INFO [loop_until]: OK (rc = 0) 03:31:57 DEBUG --- stdout --- 03:31:57 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 63m 0% 6281Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 8158m 27% 7749Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 20510m 68% 11635Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 79m 0% 3334Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 19151m 64% 15837Mi 14% gke-xlou-cdm-default-pool-1b0769c8-wpwl 155m 0% 5256Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 16028m 53% 12119Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 18135m 60% 3187Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 23020m 77% 11177Mi 10% gke-xlou-cdm-ds-0f601751-j488 7040m 23% 7349Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 19198m 64% 17602Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 9361m 31% 8026Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20283m 67% 15279Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 11222m 70% 10414Mi 17% 03:31:57 INFO [http_cmd]: http status code OK 03:31:57 DEBUG --- stderr --- 03:31:57 INFO [http_cmd]: http status code OK 03:31:57 INFO [http_cmd]: http status code OK 03:31:57 DEBUG --- status code --- 03:31:57 INFO [http_cmd]: http status code OK 03:31:57 INFO [http_cmd]: http status code OK 03:31:57 DEBUG 03:31:57 DEBUG --- status code --- 03:31:57 DEBUG --- status code --- 03:31:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:31:57 DEBUG --- status code --- 03:31:57 DEBUG --- status code --- 03:31:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:31:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:31:57 DEBUG --- http response --- 03:31:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:31:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:31:57 DEBUG --- http response --- 03:31:57 DEBUG --- http response --- 03:31:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:31:57 DEBUG --- http response --- 03:31:57 DEBUG --- http response --- 03:31:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 03:31:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:31:57 DEBUG ----- output ----- 03:31:57 INFO [http_cmd]: http status code OK 03:31:57 INFO [http_cmd]: http status code OK 03:31:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:31:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 03:31:57 DEBUG ----- output ----- 03:31:57 DEBUG ----- output ----- 03:31:57 DEBUG { 03:31:57 DEBUG --- status code --- 03:31:57 DEBUG --- status code --- 03:31:57 DEBUG ----- output ----- 03:31:57 DEBUG ----- output ----- 03:31:57 DEBUG { 03:31:57 DEBUG { 03:31:57 DEBUG "status": "success", 03:31:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:31:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:31:57 DEBUG { 03:31:57 DEBUG { 03:31:57 DEBUG "status": "success", 03:31:57 DEBUG "status": "success", 03:31:57 DEBUG "data": { 03:31:57 DEBUG --- http response --- 03:31:57 DEBUG --- http response --- 03:31:57 DEBUG "status": "success", 03:31:57 DEBUG "status": "success", 03:31:57 DEBUG "data": { 03:31:57 DEBUG "data": { 03:31:57 DEBUG "resultType": "vector", 03:31:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:31:57 DEBUG "data": { 03:31:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:31:57 DEBUG "data": { 03:31:57 DEBUG "resultType": "vector", 03:31:57 DEBUG "resultType": "vector", 03:31:57 DEBUG "result": [ 03:31:57 DEBUG ----- output ----- 03:31:57 DEBUG "resultType": "vector", 03:31:57 DEBUG ----- output ----- 03:31:57 DEBUG "resultType": "vector", 03:31:57 DEBUG "result": [ 03:31:57 DEBUG "result": [ 03:31:57 DEBUG { 03:31:57 DEBUG { 03:31:57 DEBUG "result": [ 03:31:57 DEBUG { 03:31:57 DEBUG "result": [ 03:31:57 DEBUG { 03:31:57 DEBUG { 03:31:57 DEBUG "metric": { 03:31:57 DEBUG "status": "success", 03:31:57 DEBUG { 03:31:57 DEBUG "status": "success", 03:31:57 DEBUG { 03:31:57 DEBUG "metric": { 03:31:57 DEBUG "metric": { 03:31:57 DEBUG "pod": "ds-idrepo-0" 03:31:57 DEBUG "data": { 03:31:57 DEBUG "metric": { 03:31:57 DEBUG "data": { 03:31:57 DEBUG "metric": { 03:31:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:31:57 DEBUG "pod": "am-767f6b67b5-9zw22" 03:31:57 DEBUG }, 03:31:57 DEBUG "resultType": "vector", 03:31:57 DEBUG "pod": "am-767f6b67b5-9zw22" 03:31:57 DEBUG "resultType": "vector", 03:31:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:31:57 DEBUG }, 03:31:57 DEBUG }, 03:31:57 DEBUG "value": [ 03:31:57 DEBUG "result": [ 03:31:57 DEBUG }, 03:31:57 DEBUG "result": [ 03:31:57 DEBUG }, 03:31:57 DEBUG "value": [ 03:31:57 DEBUG "value": [ 03:31:57 DEBUG 03:31:57 DEBUG { 03:31:57 DEBUG "value": [ 03:31:57 DEBUG { 03:31:57 DEBUG "value": [ 03:31:57 DEBUG 03:31:57 DEBUG 03:31:57 DEBUG 03:31:57 DEBUG "metric": { 03:31:57 DEBUG 03:31:57 DEBUG "metric": { 03:31:57 DEBUG 03:31:57 DEBUG 03:31:57 DEBUG 03:31:57 DEBUG 03:31:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:31:57 DEBUG 03:31:57 DEBUG "pod": "am-767f6b67b5-9zw22" 03:31:57 DEBUG 03:31:57 DEBUG 03:31:57 DEBUG 03:31:57 DEBUG [...] 03:31:57 DEBUG }, 03:31:57 DEBUG 03:31:57 DEBUG }, 03:31:57 DEBUG 03:31:57 DEBUG [...] 03:31:57 DEBUG [...] 03:31:57 DEBUG 03:31:57 DEBUG "value": [ 03:31:57 DEBUG [...] 03:31:57 DEBUG "value": [ 03:31:57 DEBUG [...] 03:31:57 DEBUG 03:31:57 DEBUG 03:31:57 DEBUG 03:31:57 DEBUG 03:31:57 DEBUG 03:31:57 DEBUG 03:31:57 DEBUG 03:31:57 DEBUG 03:31:57 DEBUG 03:31:57 DEBUG 03:31:57 DEBUG 03:31:57 DEBUG 03:31:57 DEBUG 03:31:57 DEBUG 03:31:57 DEBUG 03:31:57 DEBUG 03:31:57 DEBUG "pod": "ds-idrepo-2" 03:31:57 DEBUG 03:31:57 DEBUG 03:31:57 DEBUG 03:31:57 DEBUG 03:31:57 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:31:57 DEBUG "pod": "am-767f6b67b5-vbltz" 03:31:57 DEBUG }, 03:31:57 DEBUG [...] 03:31:57 DEBUG "pod": "am-767f6b67b5-vbltz" 03:31:57 DEBUG [...] 03:31:57 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:31:57 DEBUG }, 03:31:57 DEBUG }, 03:31:57 DEBUG "value": [ 03:31:57 DEBUG 03:31:57 DEBUG }, 03:31:57 DEBUG 03:31:57 DEBUG }, 03:31:57 DEBUG "value": [ 03:31:57 DEBUG "value": [ 03:31:57 DEBUG 1713666717, 03:31:57 DEBUG 03:31:57 DEBUG "value": [ 03:31:57 DEBUG 03:31:57 DEBUG "value": [ 03:31:57 DEBUG 1713666717, 03:31:57 DEBUG 1713666717, 03:31:57 DEBUG "0" 03:31:57 DEBUG 03:31:57 DEBUG 1713666717, 03:31:57 DEBUG 03:31:57 DEBUG 1713666717, 03:31:57 DEBUG "11.717058160547554" 03:31:57 DEBUG "591131" 03:31:57 DEBUG ] 03:31:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:31:57 DEBUG "2703.0666666666666" 03:31:57 DEBUG "pod": "am-767f6b67b5-vbltz" 03:31:57 DEBUG "172385.06876227897" 03:31:57 DEBUG ] 03:31:57 DEBUG ] 03:31:57 DEBUG } 03:31:57 DEBUG }, 03:31:57 DEBUG ] 03:31:57 DEBUG }, 03:31:57 DEBUG ] 03:31:57 DEBUG } 03:31:57 DEBUG } 03:31:57 DEBUG ] 03:31:57 DEBUG "value": [ 03:31:57 DEBUG } 03:31:57 DEBUG "value": [ 03:31:57 DEBUG } 03:31:57 DEBUG ] 03:31:57 DEBUG ] 03:31:57 DEBUG } 03:31:57 DEBUG 1713666717, 03:31:57 DEBUG ] 03:31:57 DEBUG 1713666717, 03:31:57 DEBUG ] 03:31:57 DEBUG } 03:31:57 DEBUG } 03:31:57 DEBUG } 03:31:57 DEBUG "11.762223467144453" 03:31:57 DEBUG } 03:31:57 DEBUG "0" 03:31:57 DEBUG } 03:31:57 DEBUG } 03:31:57 DEBUG } 03:31:57 DEBUG ] 03:31:57 DEBUG } 03:31:57 DEBUG ] 03:31:57 DEBUG } 03:31:57 DEBUG } 03:31:57 DEBUG } 03:31:57 DEBUG ] 03:31:57 DEBUG ] 03:31:57 DEBUG } 03:31:57 DEBUG } 03:31:57 DEBUG } 03:31:57 DEBUG } 03:31:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666718" 03:31:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713666718" 03:31:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666718" 03:31:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666718" 03:31:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666718" 03:31:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666718" 03:31:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666718" 03:31:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713666718" 03:31:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666718" 03:31:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666718" 03:31:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666718" 03:31:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666718" 03:31:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666718" 03:31:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666718" 03:31:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713666718" 03:31:58 INFO [http_cmd]: http status code OK 03:31:58 INFO [http_cmd]: http status code OK 03:31:58 INFO [http_cmd]: http status code OK 03:31:58 INFO [http_cmd]: http status code OK 03:31:58 INFO [http_cmd]: http status code OK 03:31:58 INFO [http_cmd]: http status code OK 03:31:58 INFO [http_cmd]: http status code OK 03:31:58 INFO [http_cmd]: http status code OK 03:31:58 INFO [http_cmd]: http status code OK 03:31:58 DEBUG --- status code --- 03:31:58 INFO [http_cmd]: http status code OK 03:31:58 DEBUG --- status code --- 03:31:58 DEBUG --- status code --- 03:31:58 DEBUG --- status code --- 03:31:58 DEBUG --- status code --- 03:31:58 INFO [http_cmd]: http status code OK 03:31:58 INFO [http_cmd]: http status code OK 03:31:58 DEBUG --- status code --- 03:31:58 DEBUG --- status code --- 03:31:58 DEBUG --- status code --- 03:31:58 DEBUG --- status code --- 03:31:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:31:58 DEBUG --- status code --- 03:31:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:31:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:31:58 DEBUG --- http response --- 03:31:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:31:58 DEBUG --- status code --- 03:31:58 DEBUG --- status code --- 03:31:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:31:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:31:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:31:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:31:58 DEBUG --- http response --- 03:31:58 INFO [http_cmd]: http status code OK 03:31:58 INFO [http_cmd]: http status code OK 03:31:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:31:58 DEBUG --- http response --- 03:31:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:31:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 03:31:58 DEBUG --- http response --- 03:31:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:31:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:31:58 DEBUG --- http response --- 03:31:58 DEBUG --- http response --- 03:31:58 DEBUG --- http response --- 03:31:58 DEBUG --- http response --- 03:31:58 INFO [http_cmd]: http status code OK 03:31:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:31:58 DEBUG --- status code --- 03:31:58 DEBUG --- status code --- 03:31:58 DEBUG --- http response --- 03:31:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:31:58 DEBUG --- http response --- 03:31:58 DEBUG ----- output ----- 03:31:58 DEBUG --- http response --- 03:31:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:31:58 DEBUG --- http response --- 03:31:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:31:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:31:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:31:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:31:58 DEBUG --- status code --- 03:31:58 DEBUG ----- output ----- 03:31:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:31:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:31:58 DEBUG ----- output ----- 03:31:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 03:31:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:31:58 DEBUG { 03:31:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:31:58 DEBUG ----- output ----- 03:31:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:31:58 DEBUG ----- output ----- 03:31:58 DEBUG ----- output ----- 03:31:58 DEBUG ----- output ----- 03:31:58 DEBUG ----- output ----- 03:31:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:31:58 DEBUG { 03:31:58 DEBUG --- http response --- 03:31:58 DEBUG --- http response --- 03:31:58 DEBUG { 03:31:58 DEBUG ----- output ----- 03:31:58 DEBUG ----- output ----- 03:31:58 DEBUG "status": "success", 03:31:58 DEBUG { 03:31:58 DEBUG ----- output ----- 03:31:58 DEBUG ----- output ----- 03:31:58 DEBUG { 03:31:58 DEBUG { 03:31:58 DEBUG { 03:31:58 DEBUG { 03:31:58 DEBUG --- http response --- 03:31:58 DEBUG "status": "success", 03:31:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:31:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:31:58 DEBUG "status": "success", 03:31:58 DEBUG { 03:31:58 DEBUG { 03:31:58 DEBUG "data": { 03:31:58 DEBUG "status": "success", 03:31:58 DEBUG { 03:31:58 DEBUG { 03:31:58 DEBUG "status": "success", 03:31:58 DEBUG "status": "success", 03:31:58 DEBUG "status": "success", 03:31:58 DEBUG "status": "success", 03:31:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:31:58 DEBUG "data": { 03:31:58 DEBUG ----- output ----- 03:31:58 DEBUG ----- output ----- 03:31:58 DEBUG "data": { 03:31:58 DEBUG "status": "success", 03:31:58 DEBUG "status": "success", 03:31:58 DEBUG "resultType": "vector", 03:31:58 DEBUG "data": { 03:31:58 DEBUG "status": "success", 03:31:58 DEBUG "status": "success", 03:31:58 DEBUG "data": { 03:31:58 DEBUG "data": { 03:31:58 DEBUG "data": { 03:31:58 DEBUG "data": { 03:31:58 DEBUG ----- output ----- 03:31:58 DEBUG { 03:31:58 DEBUG { 03:31:58 DEBUG { 03:31:58 DEBUG "resultType": "vector", 03:31:58 DEBUG "data": { 03:31:58 DEBUG "data": { 03:31:58 DEBUG "result": [ 03:31:58 DEBUG "resultType": "vector", 03:31:58 DEBUG "data": { 03:31:58 DEBUG "data": { 03:31:58 DEBUG "resultType": "vector", 03:31:58 DEBUG "resultType": "vector", 03:31:58 DEBUG "resultType": "vector", 03:31:58 DEBUG "resultType": "vector", 03:31:58 DEBUG "status": "success", 03:31:58 DEBUG "resultType": "vector", 03:31:58 DEBUG "status": "success", 03:31:58 DEBUG "status": "success", 03:31:58 DEBUG "result": [ 03:31:58 DEBUG "resultType": "vector", 03:31:58 DEBUG "resultType": "vector", 03:31:58 DEBUG { 03:31:58 DEBUG "result": [ 03:31:58 DEBUG "resultType": "vector", 03:31:58 DEBUG "resultType": "vector", 03:31:58 DEBUG "result": [ 03:31:58 DEBUG "result": [ 03:31:58 DEBUG "result": [ 03:31:58 DEBUG "result": [ 03:31:58 DEBUG "data": { 03:31:58 DEBUG "result": [ 03:31:58 DEBUG "data": { 03:31:58 DEBUG "data": { 03:31:58 DEBUG { 03:31:58 DEBUG "result": [ 03:31:58 DEBUG "result": [ 03:31:58 DEBUG "metric": { 03:31:58 DEBUG { 03:31:58 DEBUG "result": [ 03:31:58 DEBUG "result": [ 03:31:58 DEBUG { 03:31:58 DEBUG { 03:31:58 DEBUG { 03:31:58 DEBUG { 03:31:58 DEBUG "resultType": "vector", 03:31:58 DEBUG { 03:31:58 DEBUG "resultType": "vector", 03:31:58 DEBUG "resultType": "vector", 03:31:58 DEBUG "metric": { 03:31:58 DEBUG { 03:31:58 DEBUG { 03:31:58 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:31:58 DEBUG "metric": { 03:31:58 DEBUG { 03:31:58 DEBUG { 03:31:58 DEBUG "metric": { 03:31:58 DEBUG "metric": { 03:31:58 DEBUG "metric": { 03:31:58 DEBUG "metric": { 03:31:58 DEBUG "result": [ 03:31:58 DEBUG "metric": { 03:31:58 DEBUG "pod": "am-767f6b67b5-9zw22" 03:31:58 DEBUG "result": [ 03:31:58 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:31:58 DEBUG "metric": { 03:31:58 DEBUG "metric": { 03:31:58 DEBUG }, 03:31:58 DEBUG "pod": "ds-cts-0" 03:31:58 DEBUG "metric": { 03:31:58 DEBUG "metric": { 03:31:58 DEBUG "instance": "10.142.0.107:9100" 03:31:58 DEBUG "instance": "10.142.0.107:9100" 03:31:58 DEBUG "pod": "am-767f6b67b5-9zw22" 03:31:58 DEBUG "instance": "10.142.0.107:9100" 03:31:58 DEBUG { 03:31:58 DEBUG "result": [ 03:31:58 DEBUG { 03:31:58 DEBUG { 03:31:58 DEBUG }, 03:31:58 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:31:58 DEBUG "pod": "ds-cts-0" 03:31:58 DEBUG "value": [ 03:31:58 DEBUG }, 03:31:58 DEBUG "pod": "am-767f6b67b5-9zw22" 03:31:58 DEBUG }, 03:31:58 DEBUG "instance": "10.142.0.107:9100" 03:31:58 DEBUG }, 03:31:58 DEBUG }, 03:31:58 DEBUG }, 03:31:58 DEBUG "metric": { 03:31:58 DEBUG "metric": { 03:31:58 DEBUG }, 03:31:58 DEBUG "metric": { 03:31:58 DEBUG "value": [ 03:31:58 DEBUG }, 03:31:58 DEBUG }, 03:31:58 DEBUG 03:31:58 DEBUG "value": [ 03:31:58 DEBUG }, 03:31:58 DEBUG "value": [ 03:31:58 DEBUG }, 03:31:58 DEBUG "value": [ 03:31:58 DEBUG "value": [ 03:31:58 DEBUG 03:31:58 DEBUG "pod": "ds-idrepo-0" 03:31:58 DEBUG "pod": "ds-idrepo-0" 03:31:58 DEBUG "value": [ 03:31:58 DEBUG "pod": "ds-idrepo-0" 03:31:58 DEBUG 03:31:58 DEBUG "value": [ 03:31:58 DEBUG "value": [ 03:31:58 DEBUG 03:31:58 DEBUG 03:31:58 DEBUG 03:31:58 DEBUG 03:31:58 DEBUG "value": [ 03:31:58 DEBUG 03:31:58 DEBUG 03:31:58 DEBUG }, 03:31:58 DEBUG }, 03:31:58 DEBUG "value": [ 03:31:58 DEBUG 03:31:58 DEBUG }, 03:31:58 DEBUG 03:31:58 DEBUG 03:31:58 DEBUG 03:31:58 DEBUG 03:31:58 DEBUG 03:31:58 DEBUG "value": [ 03:31:58 DEBUG 03:31:58 DEBUG 03:31:58 DEBUG 03:31:58 DEBUG 03:31:58 DEBUG "value": [ 03:31:58 DEBUG "value": [ 03:31:58 DEBUG 03:31:58 DEBUG 03:31:58 DEBUG "value": [ 03:31:58 DEBUG 03:31:58 DEBUG 03:31:58 DEBUG 03:31:58 DEBUG [...] 03:31:58 DEBUG [...] 03:31:58 DEBUG 03:31:58 DEBUG 03:31:58 DEBUG 03:31:58 DEBUG 03:31:58 DEBUG [...] 03:31:58 DEBUG 03:31:58 DEBUG 03:31:58 DEBUG 03:31:58 DEBUG 03:31:58 DEBUG 03:31:58 DEBUG [...] 03:31:58 DEBUG 03:31:58 DEBUG 03:31:58 DEBUG 03:31:58 DEBUG 03:31:58 DEBUG 03:31:58 DEBUG [...] 03:31:58 DEBUG 03:31:58 DEBUG [...] 03:31:58 DEBUG 03:31:58 DEBUG 03:31:58 DEBUG 03:31:58 DEBUG 03:31:58 DEBUG [...] 03:31:58 DEBUG 03:31:58 DEBUG 03:31:58 DEBUG [...] 03:31:58 DEBUG [...] 03:31:58 DEBUG 03:31:58 DEBUG 03:31:58 DEBUG 03:31:58 DEBUG 03:31:58 DEBUG [...] 03:31:58 DEBUG 03:31:58 DEBUG 03:31:58 DEBUG 03:31:58 DEBUG 03:31:58 DEBUG [...] 03:31:58 DEBUG 03:31:58 DEBUG 03:31:58 DEBUG 03:31:58 DEBUG 03:31:58 DEBUG 03:31:58 DEBUG 03:31:58 DEBUG 03:31:58 DEBUG [...] 03:31:58 DEBUG 03:31:58 DEBUG 03:31:58 DEBUG 03:31:58 DEBUG 03:31:58 DEBUG [...] 03:31:58 DEBUG [...] 03:31:58 DEBUG 03:31:58 DEBUG 03:31:58 DEBUG [...] 03:31:58 DEBUG 03:31:58 DEBUG 03:31:58 DEBUG 03:31:58 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:31:58 DEBUG "pod": "ds-cts-2" 03:31:58 DEBUG 03:31:58 DEBUG 03:31:58 DEBUG 03:31:58 DEBUG 03:31:58 DEBUG "instance": "10.142.0.43:9100" 03:31:58 DEBUG 03:31:58 DEBUG 03:31:58 DEBUG 03:31:58 DEBUG 03:31:58 DEBUG 03:31:58 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:31:58 DEBUG 03:31:58 DEBUG 03:31:58 DEBUG }, 03:31:58 DEBUG "value": [ 03:31:58 DEBUG 03:31:58 DEBUG 03:31:58 DEBUG 03:31:58 DEBUG "pod": "am-767f6b67b5-vbltz" 03:31:58 DEBUG }, 03:31:58 DEBUG 03:31:58 DEBUG 03:31:58 DEBUG 03:31:58 DEBUG "pod": "am-767f6b67b5-vbltz" 03:31:58 DEBUG 03:31:58 DEBUG }, 03:31:58 DEBUG "value": [ 03:31:58 DEBUG "pod": "ds-cts-2" 03:31:58 DEBUG 1713666718, 03:31:58 DEBUG "0" 03:31:58 DEBUG "instance": "10.142.0.43:9100" 03:31:58 DEBUG "pod": "am-767f6b67b5-vbltz" 03:31:58 DEBUG "instance": "10.142.0.43:9100" 03:31:58 DEBUG }, 03:31:58 DEBUG "value": [ 03:31:58 DEBUG 1713666718, 03:31:58 DEBUG 03:31:58 DEBUG "instance": "10.142.0.43:9100" 03:31:58 DEBUG }, 03:31:58 DEBUG 03:31:58 DEBUG 1713666718, 03:31:58 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:31:58 DEBUG }, 03:31:58 DEBUG ] 03:31:58 DEBUG }, 03:31:58 DEBUG }, 03:31:58 DEBUG }, 03:31:58 DEBUG }, 03:31:58 DEBUG "101270048.56666666" 03:31:58 DEBUG "value": [ 03:31:58 DEBUG "pod": "ds-idrepo-2" 03:31:58 DEBUG 03:31:58 DEBUG }, 03:31:58 DEBUG "value": [ 03:31:58 DEBUG "pod": "ds-idrepo-2" 03:31:58 DEBUG "1280659456" 03:31:58 DEBUG }, 03:31:58 DEBUG "value": [ 03:31:58 DEBUG } 03:31:58 DEBUG "value": [ 03:31:58 DEBUG "value": [ 03:31:58 DEBUG "value": [ 03:31:58 DEBUG "value": [ 03:31:58 DEBUG ] 03:31:58 DEBUG 1713666718, 03:31:58 DEBUG }, 03:31:58 DEBUG "pod": "ds-cts-2" 03:31:58 DEBUG "value": [ 03:31:58 DEBUG 1713666718, 03:31:58 DEBUG }, 03:31:58 DEBUG ] 03:31:58 DEBUG "value": [ 03:31:58 DEBUG 1713666718, 03:31:58 DEBUG ] 03:31:58 DEBUG 1713666718, 03:31:58 DEBUG 1713666718, 03:31:58 DEBUG 1713666718, 03:31:58 DEBUG 1713666718, 03:31:58 DEBUG } 03:31:58 DEBUG "4457700348928" 03:31:58 DEBUG "value": [ 03:31:58 DEBUG }, 03:31:58 DEBUG 1713666718, 03:31:58 DEBUG "4046.2666666666664" 03:31:58 DEBUG "value": [ 03:31:58 DEBUG } 03:31:58 DEBUG 1713666718, 03:31:58 DEBUG "32" 03:31:58 DEBUG } 03:31:58 DEBUG "0" 03:31:58 DEBUG "0.08966666666667038" 03:31:58 DEBUG "2709.6" 03:31:58 DEBUG "177.685" 03:31:58 DEBUG ] 03:31:58 DEBUG ] 03:31:58 DEBUG 1713666718, 03:31:58 DEBUG "0" 03:31:58 DEBUG "101266039.73333333" 03:31:58 DEBUG ] 03:31:58 DEBUG 1713666718, 03:31:58 DEBUG ] 03:31:58 DEBUG "0" 03:31:58 DEBUG ] 03:31:58 DEBUG } 03:31:58 DEBUG ] 03:31:58 DEBUG ] 03:31:58 DEBUG ] 03:31:58 DEBUG ] 03:31:58 DEBUG } 03:31:58 DEBUG } 03:31:58 DEBUG ] 03:31:58 DEBUG "value": [ 03:31:58 DEBUG ] 03:31:58 DEBUG } 03:31:58 DEBUG "0" 03:31:58 DEBUG } 03:31:58 DEBUG ] 03:31:58 DEBUG } 03:31:58 DEBUG } 03:31:58 DEBUG } 03:31:58 DEBUG } 03:31:58 DEBUG } 03:31:58 DEBUG } 03:31:58 DEBUG ] 03:31:58 DEBUG } 03:31:58 DEBUG 1713666718, 03:31:58 DEBUG } 03:31:58 DEBUG ] 03:31:58 DEBUG ] 03:31:58 DEBUG } 03:31:58 DEBUG } 03:31:58 DEBUG ] 03:31:58 DEBUG ] 03:31:58 DEBUG ] 03:31:58 DEBUG ] 03:31:58 DEBUG ] 03:31:58 DEBUG } 03:31:58 DEBUG ] 03:31:58 DEBUG "0" 03:31:58 DEBUG ] 03:31:58 DEBUG } 03:31:58 DEBUG } 03:31:58 DEBUG ] 03:31:58 DEBUG } 03:31:58 DEBUG } 03:31:58 DEBUG } 03:31:58 DEBUG } 03:31:58 DEBUG } 03:31:58 DEBUG } 03:31:58 DEBUG } 03:31:58 DEBUG ] 03:31:58 DEBUG } 03:31:58 DEBUG } 03:31:58 DEBUG ] 03:31:58 DEBUG } 03:31:58 DEBUG } 03:31:58 DEBUG } 03:31:58 DEBUG } 03:31:58 DEBUG } 03:31:58 DEBUG } 03:31:58 DEBUG } 03:31:58 DEBUG } 03:31:58 DEBUG } 03:31:58 DEBUG } 03:31:58 DEBUG } 03:31:58 DEBUG ] 03:31:58 DEBUG } 03:31:58 DEBUG } 03:31:58 DEBUG } 03:32:10 WARNING File /tmp/lodemon_data-20240421_025941/node_disk_read_bytes_total.json does not exist. Skipping average value computation... 127.0.0.1 - - [21/Apr/2024 03:32:10] "GET /monitoring/average?start_time=20240421_020255&stop_time=20240421_023009 HTTP/1.1" 200 - 03:32:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713666774" 03:32:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666774" 03:32:54 ERROR [http_cmd]: ERROR 03:32:54 ERROR --- status code --- 03:32:54 ERROR http status code is 400 (expected [200, 401, 403, 404]) 03:32:54 ERROR --- http response --- 03:32:54 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 03:32:54 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 03:32:54 INFO [http_cmd]: http status code OK 03:32:54 DEBUG --- status code --- 03:32:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:32:54 DEBUG --- http response --- 03:32:54 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 03:32:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666775" 03:32:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666775" 03:32:55 INFO [http_cmd]: http status code OK 03:32:55 DEBUG --- status code --- 03:32:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:32:55 DEBUG --- http response --- 03:32:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:32:55 DEBUG ----- output ----- 03:32:55 DEBUG { 03:32:55 DEBUG "status": "success", 03:32:55 DEBUG "data": { 03:32:55 DEBUG "resultType": "vector", 03:32:55 DEBUG "result": [ 03:32:55 DEBUG { 03:32:55 DEBUG "metric": { 03:32:55 DEBUG "pod": "ds-cts-0" 03:32:55 DEBUG }, 03:32:55 DEBUG "value": [ 03:32:55 DEBUG 03:32:55 DEBUG 03:32:55 DEBUG 03:32:55 DEBUG [...] 03:32:55 DEBUG 03:32:55 DEBUG 03:32:55 DEBUG 03:32:55 DEBUG "pod": "ds-cts-2" 03:32:55 DEBUG }, 03:32:55 DEBUG "value": [ 03:32:55 DEBUG 1713666775, 03:32:55 DEBUG "0.0010333333333316357" 03:32:55 DEBUG ] 03:32:55 DEBUG } 03:32:55 DEBUG ] 03:32:55 DEBUG } 03:32:55 DEBUG } 03:32:55 INFO [http_cmd]: http status code OK 03:32:55 DEBUG --- status code --- 03:32:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:32:55 DEBUG --- http response --- 03:32:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:32:55 DEBUG ----- output ----- 03:32:55 DEBUG { 03:32:55 DEBUG "status": "success", 03:32:55 DEBUG "data": { 03:32:55 DEBUG "resultType": "vector", 03:32:55 DEBUG "result": [ 03:32:55 DEBUG { 03:32:55 DEBUG "metric": { 03:32:55 DEBUG "instance": "10.142.0.107:9100" 03:32:55 DEBUG }, 03:32:55 DEBUG "value": [ 03:32:55 DEBUG 03:32:55 DEBUG 03:32:55 DEBUG 03:32:55 DEBUG [...] 03:32:55 DEBUG 03:32:55 DEBUG 03:32:55 DEBUG 03:32:55 DEBUG "instance": "10.142.0.43:9100" 03:32:55 DEBUG }, 03:32:55 DEBUG "value": [ 03:32:55 DEBUG 1713666775, 03:32:55 DEBUG "0.49443333333332284" 03:32:55 DEBUG ] 03:32:55 DEBUG } 03:32:55 DEBUG ] 03:32:55 DEBUG } 03:32:55 DEBUG } 03:32:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666776" 03:32:56 INFO [http_cmd]: http status code OK 03:32:56 DEBUG --- status code --- 03:32:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:32:56 DEBUG --- http response --- 03:32:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:32:56 DEBUG ----- output ----- 03:32:56 DEBUG { 03:32:56 DEBUG "status": "success", 03:32:56 DEBUG "data": { 03:32:56 DEBUG "resultType": "vector", 03:32:56 DEBUG "result": [ 03:32:56 DEBUG { 03:32:56 DEBUG "metric": { 03:32:56 DEBUG "pod": "am-767f6b67b5-9zw22" 03:32:56 DEBUG }, 03:32:56 DEBUG "value": [ 03:32:56 DEBUG 03:32:56 DEBUG 03:32:56 DEBUG 03:32:56 DEBUG [...] 03:32:56 DEBUG 03:32:56 DEBUG 03:32:56 DEBUG 03:32:56 DEBUG "pod": "am-767f6b67b5-vbltz" 03:32:56 DEBUG }, 03:32:56 DEBUG "value": [ 03:32:56 DEBUG 1713666776, 03:32:56 DEBUG "930.1999999999999" 03:32:56 DEBUG ] 03:32:56 DEBUG } 03:32:56 DEBUG ] 03:32:56 DEBUG } 03:32:56 DEBUG } 03:32:56 INFO 03:32:56 INFO [loop_until]: kubectl --namespace=xlou top pods 03:32:56 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:32:56 INFO [loop_until]: OK (rc = 0) 03:32:56 DEBUG --- stdout --- 03:32:56 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 3955m 8816Mi am-767f6b67b5-qcqz2 3635m 8381Mi am-767f6b67b5-vbltz 3808m 8893Mi ds-cts-0 18446m 12520Mi ds-cts-1 18452m 12481Mi ds-cts-2 19313m 12562Mi ds-idrepo-0 954m 3918Mi ds-idrepo-1 905m 3961Mi ds-idrepo-2 915m 3914Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1179Mi idm-b85f9d978-wnhxb 3m 1239Mi lodemon-54c87f8cbb-jzrjs 15m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-68c5d5b884-p97w5 1090m 581Mi 03:32:56 DEBUG --- stderr --- 03:32:56 DEBUG 03:32:57 INFO 03:32:57 INFO [loop_until]: kubectl --namespace=xlou top node 03:32:57 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:32:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666777" 03:32:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666777" 03:32:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666777" 03:32:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666777" 03:32:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666777" 03:32:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666777" 03:32:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713666777" 03:32:57 INFO [http_cmd]: http status code OK 03:32:57 DEBUG --- status code --- 03:32:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:32:57 DEBUG --- http response --- 03:32:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:32:57 DEBUG ----- output ----- 03:32:57 DEBUG { 03:32:57 INFO [http_cmd]: http status code OK 03:32:57 DEBUG "status": "success", 03:32:57 INFO [http_cmd]: http status code OK 03:32:57 INFO [http_cmd]: http status code OK 03:32:57 INFO [http_cmd]: http status code OK 03:32:57 DEBUG --- status code --- 03:32:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:32:57 INFO [http_cmd]: http status code OK 03:32:57 DEBUG --- status code --- 03:32:57 DEBUG --- status code --- 03:32:57 DEBUG --- status code --- 03:32:57 DEBUG "data": { 03:32:57 DEBUG "resultType": "vector", 03:32:57 DEBUG --- status code --- 03:32:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:32:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:32:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:32:57 DEBUG --- http response --- 03:32:57 DEBUG "result": [ 03:32:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:32:57 DEBUG --- http response --- 03:32:57 DEBUG --- http response --- 03:32:57 DEBUG --- http response --- 03:32:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:32:57 DEBUG { 03:32:57 DEBUG --- http response --- 03:32:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:32:57 INFO [http_cmd]: http status code OK 03:32:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:32:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:32:57 DEBUG ----- output ----- 03:32:57 DEBUG "metric": { 03:32:57 DEBUG ----- output ----- 03:32:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:32:57 DEBUG --- status code --- 03:32:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:32:57 DEBUG ----- output ----- 03:32:57 DEBUG { 03:32:57 DEBUG "pod": "ds-idrepo-0" 03:32:57 DEBUG { 03:32:57 DEBUG "status": "success", 03:32:57 DEBUG --- http response --- 03:32:57 DEBUG ----- output ----- 03:32:57 DEBUG { 03:32:57 DEBUG "status": "success", 03:32:57 DEBUG }, 03:32:57 DEBUG "data": { 03:32:57 DEBUG ----- output ----- 03:32:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:32:57 DEBUG { 03:32:57 DEBUG "status": "success", 03:32:57 DEBUG "data": { 03:32:57 DEBUG "resultType": "vector", 03:32:57 DEBUG "resultType": "vector", 03:32:57 DEBUG { 03:32:57 DEBUG ----- output ----- 03:32:57 DEBUG { 03:32:57 DEBUG "status": "success", 03:32:57 DEBUG "data": { 03:32:57 DEBUG "resultType": "vector", 03:32:57 DEBUG "value": [ 03:32:57 DEBUG 03:32:57 DEBUG "status": "success", 03:32:57 DEBUG "result": [ 03:32:57 DEBUG "result": [ 03:32:57 DEBUG "data": { 03:32:57 DEBUG "result": [ 03:32:57 DEBUG 03:32:57 DEBUG "status": "success", 03:32:57 DEBUG "data": { 03:32:57 DEBUG { 03:32:57 DEBUG { 03:32:57 DEBUG "resultType": "vector", 03:32:57 DEBUG { 03:32:57 DEBUG 03:32:57 DEBUG "resultType": "vector", 03:32:57 DEBUG "data": { 03:32:57 DEBUG "metric": { 03:32:57 DEBUG "metric": { 03:32:57 DEBUG "result": [ 03:32:57 DEBUG "metric": { 03:32:57 DEBUG [...] 03:32:57 DEBUG "result": [ 03:32:57 DEBUG "resultType": "vector", 03:32:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:32:57 DEBUG }, 03:32:57 DEBUG { 03:32:57 DEBUG "pod": "am-767f6b67b5-9zw22" 03:32:57 DEBUG }, 03:32:57 DEBUG { 03:32:57 DEBUG "result": [ 03:32:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:32:57 DEBUG }, 03:32:57 DEBUG "metric": { 03:32:57 DEBUG "value": [ 03:32:57 DEBUG 03:32:57 DEBUG "metric": { 03:32:57 DEBUG { 03:32:57 DEBUG "metric": { 03:32:57 DEBUG "value": [ 03:32:57 DEBUG "value": [ 03:32:57 DEBUG 03:32:57 DEBUG 03:32:57 DEBUG "pod": "am-767f6b67b5-9zw22" 03:32:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:32:57 DEBUG 03:32:57 DEBUG "pod": "am-767f6b67b5-9zw22" 03:32:57 DEBUG }, 03:32:57 DEBUG 03:32:57 DEBUG 03:32:57 DEBUG }, 03:32:57 DEBUG }, 03:32:57 DEBUG 03:32:57 DEBUG "value": [ 03:32:57 DEBUG 03:32:57 DEBUG 03:32:57 DEBUG "pod": "ds-idrepo-2" 03:32:57 DEBUG "value": [ 03:32:57 DEBUG "value": [ 03:32:57 DEBUG 03:32:57 DEBUG 03:32:57 DEBUG 03:32:57 DEBUG 03:32:57 DEBUG }, 03:32:57 DEBUG "value": [ 03:32:57 DEBUG 03:32:57 DEBUG [...] 03:32:57 DEBUG 03:32:57 DEBUG [...] 03:32:57 DEBUG [...] 03:32:57 DEBUG 03:32:57 DEBUG 1713666777, 03:32:57 DEBUG 03:32:57 DEBUG 03:32:57 DEBUG 03:32:57 DEBUG 03:32:57 DEBUG 03:32:57 DEBUG 03:32:57 DEBUG "0" 03:32:57 DEBUG 03:32:57 DEBUG 03:32:57 DEBUG 03:32:57 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 03:32:57 DEBUG }, 03:32:57 DEBUG 03:32:57 DEBUG ] 03:32:57 DEBUG [...] 03:32:57 DEBUG "value": [ 03:32:57 DEBUG 1713666777, 03:32:57 DEBUG 03:32:57 DEBUG 03:32:57 DEBUG [...] 03:32:57 DEBUG } 03:32:57 DEBUG 03:32:57 DEBUG "0" 03:32:57 DEBUG ] 03:32:57 DEBUG 03:32:57 DEBUG 03:32:57 DEBUG "pod": "am-767f6b67b5-vbltz" 03:32:57 DEBUG ] 03:32:57 DEBUG } 03:32:57 DEBUG } 03:32:57 DEBUG ] 03:32:57 DEBUG } 03:32:57 DEBUG } 03:32:57 DEBUG [...] 03:32:57 DEBUG } 03:32:57 DEBUG 03:32:57 DEBUG 03:32:57 DEBUG 03:32:57 DEBUG 03:32:57 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 03:32:57 DEBUG 03:32:57 DEBUG 03:32:57 DEBUG 03:32:57 DEBUG 03:32:57 DEBUG "pod": "am-767f6b67b5-vbltz" 03:32:57 DEBUG }, 03:32:57 DEBUG "value": [ 03:32:57 DEBUG 1713666777, 03:32:57 DEBUG "0" 03:32:57 DEBUG ] 03:32:57 DEBUG "pod": "am-767f6b67b5-vbltz" 03:32:57 DEBUG }, 03:32:57 DEBUG "value": [ 03:32:57 DEBUG }, 03:32:57 DEBUG } 03:32:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:32:57 DEBUG }, 03:32:57 DEBUG 1713666777, 03:32:57 DEBUG "value": [ 03:32:57 DEBUG ] 03:32:57 DEBUG }, 03:32:57 DEBUG "value": [ 03:32:57 DEBUG 1713666777, 03:32:57 DEBUG 1713666777, 03:32:57 DEBUG } 03:32:57 DEBUG "932.4333333333333" 03:32:57 DEBUG "1.5693242160064438" 03:32:57 DEBUG "value": [ 03:32:57 DEBUG 1713666777, 03:32:57 DEBUG } 03:32:57 DEBUG ] 03:32:57 DEBUG ] 03:32:57 DEBUG "445504" 03:32:57 DEBUG "0.00001099231978469501" 03:32:57 DEBUG } 03:32:57 DEBUG } 03:32:57 DEBUG ] 03:32:57 DEBUG } 03:32:57 DEBUG ] 03:32:57 DEBUG ] 03:32:57 DEBUG ] 03:32:57 DEBUG } 03:32:57 DEBUG } 03:32:57 DEBUG } 03:32:57 DEBUG } 03:32:57 DEBUG ] 03:32:57 DEBUG } 03:32:57 DEBUG } 03:32:57 DEBUG } 03:32:57 DEBUG ] 03:32:57 DEBUG } 03:32:57 DEBUG } 03:32:57 INFO [loop_until]: OK (rc = 0) 03:32:57 DEBUG --- stdout --- 03:32:57 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 63m 0% 6280Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 3243m 10% 7769Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 7244m 24% 11635Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 86m 0% 3350Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 19015m 63% 15851Mi 14% gke-xlou-cdm-default-pool-1b0769c8-wpwl 162m 0% 5268Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 11456m 38% 11314Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 3360m 11% 3160Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 8499m 28% 11168Mi 10% gke-xlou-cdm-ds-0f601751-j488 4687m 15% 7354Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 18486m 61% 17619Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 2800m 9% 8010Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 19718m 66% 15279Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 4182m 26% 10961Mi 18% 03:32:57 DEBUG --- stderr --- 03:32:57 DEBUG 03:32:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713666778" 03:32:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666778" 03:32:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666778" 03:32:58 INFO [http_cmd]: http status code OK 03:32:58 DEBUG --- status code --- 03:32:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:32:58 DEBUG --- http response --- 03:32:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:32:58 DEBUG ----- output ----- 03:32:58 DEBUG { 03:32:58 DEBUG "status": "success", 03:32:58 DEBUG "data": { 03:32:58 DEBUG "resultType": "vector", 03:32:58 DEBUG "result": [ 03:32:58 DEBUG { 03:32:58 DEBUG "metric": { 03:32:58 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:32:58 DEBUG }, 03:32:58 DEBUG "value": [ 03:32:58 DEBUG 03:32:58 DEBUG 03:32:58 DEBUG 03:32:58 DEBUG [...] 03:32:58 DEBUG 03:32:58 DEBUG 03:32:58 DEBUG 03:32:58 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:32:58 DEBUG }, 03:32:58 DEBUG "value": [ 03:32:58 DEBUG 1713666778, 03:32:58 DEBUG "783183872" 03:32:58 DEBUG ] 03:32:58 DEBUG } 03:32:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666778" 03:32:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666778" 03:32:58 DEBUG ] 03:32:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666778" 03:32:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666778" 03:32:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713666778" 03:32:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666778" 03:32:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666778" 03:32:58 DEBUG } 03:32:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666778" 03:32:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666778" 03:32:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713666778" 03:32:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666778" 03:32:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666778" 03:32:58 INFO [http_cmd]: http status code OK 03:32:58 INFO [http_cmd]: http status code OK 03:32:58 INFO [http_cmd]: http status code OK 03:32:58 DEBUG } 03:32:58 INFO [http_cmd]: http status code OK 03:32:58 INFO [http_cmd]: http status code OK 03:32:58 DEBUG --- status code --- 03:32:58 INFO [http_cmd]: http status code OK 03:32:58 DEBUG --- status code --- 03:32:58 INFO [http_cmd]: http status code OK 03:32:58 INFO [http_cmd]: http status code OK 03:32:58 DEBUG --- status code --- 03:32:58 INFO [http_cmd]: http status code OK 03:32:58 INFO [http_cmd]: http status code OK 03:32:58 DEBUG --- status code --- 03:32:58 DEBUG --- status code --- 03:32:58 INFO [http_cmd]: http status code OK 03:32:58 INFO [http_cmd]: http status code OK 03:32:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:32:58 INFO [http_cmd]: http status code OK 03:32:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:32:58 DEBUG --- status code --- 03:32:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:32:58 DEBUG --- status code --- 03:32:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:32:58 INFO [http_cmd]: http status code OK 03:32:58 DEBUG --- status code --- 03:32:58 DEBUG --- status code --- 03:32:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:32:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:32:58 DEBUG --- http response --- 03:32:58 DEBUG --- status code --- 03:32:58 DEBUG --- http response --- 03:32:58 DEBUG --- status code --- 03:32:58 DEBUG --- http response --- 03:32:58 DEBUG --- status code --- 03:32:58 DEBUG --- http response --- 03:32:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:32:58 DEBUG --- http response --- 03:32:58 DEBUG --- status code --- 03:32:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:32:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:32:58 DEBUG --- http response --- 03:32:58 DEBUG --- status code --- 03:32:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:32:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:32:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:32:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:32:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:32:58 DEBUG --- http response --- 03:32:58 DEBUG --- http response --- 03:32:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:32:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:32:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:32:58 DEBUG --- http response --- 03:32:58 DEBUG --- http response --- 03:32:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:32:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:32:58 DEBUG ----- output ----- 03:32:58 DEBUG --- http response --- 03:32:58 DEBUG ----- output ----- 03:32:58 DEBUG --- http response --- 03:32:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:32:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:32:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:32:58 DEBUG ----- output ----- 03:32:58 DEBUG --- http response --- 03:32:58 DEBUG ----- output ----- 03:32:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:32:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:32:58 DEBUG ----- output ----- 03:32:58 DEBUG --- http response --- 03:32:58 DEBUG { 03:32:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:32:58 DEBUG { 03:32:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:32:58 DEBUG ----- output ----- 03:32:58 DEBUG ----- output ----- 03:32:58 DEBUG { 03:32:58 DEBUG { 03:32:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:32:58 DEBUG { 03:32:58 DEBUG ----- output ----- 03:32:58 DEBUG ----- output ----- 03:32:58 DEBUG { 03:32:58 DEBUG "status": "success", 03:32:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:32:58 DEBUG ----- output ----- 03:32:58 DEBUG "status": "success", 03:32:58 DEBUG ----- output ----- 03:32:58 DEBUG { 03:32:58 DEBUG "status": "success", 03:32:58 DEBUG ----- output ----- 03:32:58 DEBUG "status": "success", 03:32:58 DEBUG "data": { 03:32:58 DEBUG "status": "success", 03:32:58 DEBUG { 03:32:58 DEBUG { 03:32:58 DEBUG "status": "success", 03:32:58 DEBUG "data": { 03:32:58 DEBUG ----- output ----- 03:32:58 DEBUG { 03:32:58 DEBUG "data": { 03:32:58 DEBUG { 03:32:58 DEBUG "status": "success", 03:32:58 DEBUG "data": { 03:32:58 DEBUG { 03:32:58 DEBUG "resultType": "vector", 03:32:58 DEBUG ----- output ----- 03:32:58 DEBUG "data": { 03:32:58 DEBUG "status": "success", 03:32:58 DEBUG "status": "success", 03:32:58 DEBUG "data": { 03:32:58 DEBUG "resultType": "vector", 03:32:58 DEBUG "status": "success", 03:32:58 DEBUG "resultType": "vector", 03:32:58 DEBUG "status": "success", 03:32:58 DEBUG "data": { 03:32:58 DEBUG "resultType": "vector", 03:32:58 DEBUG "status": "success", 03:32:58 DEBUG "result": [ 03:32:58 DEBUG { 03:32:58 DEBUG "resultType": "vector", 03:32:58 DEBUG "data": { 03:32:58 DEBUG "data": { 03:32:58 DEBUG "resultType": "vector", 03:32:58 DEBUG "result": [ 03:32:58 DEBUG "data": { 03:32:58 DEBUG "result": [ 03:32:58 DEBUG "data": { 03:32:58 DEBUG "resultType": "vector", 03:32:58 DEBUG "result": [ 03:32:58 DEBUG { 03:32:58 DEBUG { 03:32:58 DEBUG "status": "success", 03:32:58 DEBUG "result": [ 03:32:58 DEBUG "resultType": "vector", 03:32:58 DEBUG "resultType": "vector", 03:32:58 DEBUG "result": [ 03:32:58 DEBUG { 03:32:58 DEBUG { 03:32:58 DEBUG "resultType": "vector", 03:32:58 DEBUG { 03:32:58 DEBUG "resultType": "vector", 03:32:58 DEBUG "result": [ 03:32:58 DEBUG "data": { 03:32:58 DEBUG "metric": { 03:32:58 DEBUG "metric": { 03:32:58 DEBUG "data": { 03:32:58 DEBUG { 03:32:58 DEBUG "result": [ 03:32:58 DEBUG "result": [ 03:32:58 DEBUG { 03:32:58 DEBUG "status": "success", 03:32:58 DEBUG "metric": { 03:32:58 DEBUG "result": [ 03:32:58 DEBUG "metric": { 03:32:58 DEBUG "result": [ 03:32:58 DEBUG { 03:32:58 DEBUG "resultType": "vector", 03:32:58 DEBUG "pod": "am-767f6b67b5-9zw22" 03:32:58 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:32:58 DEBUG }, 03:32:58 DEBUG "metric": { 03:32:58 DEBUG "resultType": "vector", 03:32:58 DEBUG { 03:32:58 DEBUG "metric": { 03:32:58 DEBUG "data": { 03:32:58 DEBUG "instance": "10.142.0.107:9100" 03:32:58 DEBUG { 03:32:58 DEBUG "pod": "ds-cts-0" 03:32:58 DEBUG { 03:32:58 DEBUG "metric": { 03:32:58 DEBUG "pod": "ds-cts-0" 03:32:58 DEBUG }, 03:32:58 DEBUG "value": [ 03:32:58 DEBUG "instance": "10.142.0.107:9100" 03:32:58 DEBUG { 03:32:58 DEBUG "result": [ 03:32:58 DEBUG "metric": { 03:32:58 DEBUG "pod": "am-767f6b67b5-9zw22" 03:32:58 DEBUG "resultType": "vector", 03:32:58 DEBUG }, 03:32:58 DEBUG "metric": { 03:32:58 DEBUG }, 03:32:58 DEBUG "result": [ 03:32:58 DEBUG }, 03:32:58 DEBUG "metric": { 03:32:58 DEBUG "value": [ 03:32:58 DEBUG 03:32:58 DEBUG }, 03:32:58 DEBUG "metric": { 03:32:58 DEBUG "instance": "10.142.0.107:9100" 03:32:58 DEBUG "pod": "ds-idrepo-0" 03:32:58 DEBUG }, 03:32:58 DEBUG "result": [ 03:32:58 DEBUG "value": [ 03:32:58 DEBUG "pod": "ds-idrepo-0" 03:32:58 DEBUG "value": [ 03:32:58 DEBUG { 03:32:58 DEBUG "value": [ 03:32:58 DEBUG "instance": "10.142.0.107:9100" 03:32:58 DEBUG 03:32:58 DEBUG 03:32:58 DEBUG "value": [ 03:32:58 DEBUG }, 03:32:58 DEBUG { 03:32:58 DEBUG }, 03:32:58 DEBUG "value": [ 03:32:58 DEBUG { 03:32:58 DEBUG 03:32:58 DEBUG }, 03:32:58 DEBUG 03:32:58 DEBUG 03:32:58 DEBUG 03:32:58 DEBUG }, 03:32:58 DEBUG 03:32:58 DEBUG 03:32:58 DEBUG 03:32:58 DEBUG "value": [ 03:32:58 DEBUG "metric": { 03:32:58 DEBUG "pod": "ds-idrepo-0" 03:32:58 DEBUG }, 03:32:58 DEBUG "metric": { 03:32:58 DEBUG 03:32:58 DEBUG "value": [ 03:32:58 DEBUG 03:32:58 DEBUG 03:32:58 DEBUG "value": [ 03:32:58 DEBUG 03:32:58 DEBUG "metric": { 03:32:58 DEBUG [...] 03:32:58 DEBUG 03:32:58 DEBUG 03:32:58 DEBUG 03:32:58 DEBUG "value": [ 03:32:58 DEBUG 03:32:58 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:32:58 DEBUG 03:32:58 DEBUG 03:32:58 DEBUG [...] 03:32:58 DEBUG 03:32:58 DEBUG 03:32:58 DEBUG [...] 03:32:58 DEBUG "pod": "am-767f6b67b5-9zw22" 03:32:58 DEBUG }, 03:32:58 DEBUG 03:32:58 DEBUG 03:32:58 DEBUG "value": [ 03:32:58 DEBUG 03:32:58 DEBUG 03:32:58 DEBUG }, 03:32:58 DEBUG 03:32:58 DEBUG 03:32:58 DEBUG 03:32:58 DEBUG [...] 03:32:58 DEBUG 03:32:58 DEBUG 03:32:58 DEBUG 03:32:58 DEBUG 03:32:58 DEBUG [...] 03:32:58 DEBUG [...] 03:32:58 DEBUG 03:32:58 DEBUG 03:32:58 DEBUG [...] 03:32:58 DEBUG "value": [ 03:32:58 DEBUG [...] 03:32:58 DEBUG 03:32:58 DEBUG 03:32:58 DEBUG 03:32:58 DEBUG 03:32:58 DEBUG "pod": "am-767f6b67b5-vbltz" 03:32:58 DEBUG 03:32:58 DEBUG "value": [ 03:32:58 DEBUG 03:32:58 DEBUG 03:32:58 DEBUG 03:32:58 DEBUG 03:32:58 DEBUG 03:32:58 DEBUG 03:32:58 DEBUG 03:32:58 DEBUG [...] 03:32:58 DEBUG 03:32:58 DEBUG 03:32:58 DEBUG [...] 03:32:58 DEBUG }, 03:32:58 DEBUG 03:32:58 DEBUG 03:32:58 DEBUG 03:32:58 DEBUG 03:32:58 DEBUG 03:32:58 DEBUG [...] 03:32:58 DEBUG 03:32:58 DEBUG 03:32:58 DEBUG 03:32:58 DEBUG 03:32:58 DEBUG "pod": "ds-cts-2" 03:32:58 DEBUG 03:32:58 DEBUG 03:32:58 DEBUG "value": [ 03:32:58 DEBUG 1713666778, 03:32:58 DEBUG "180.461" 03:32:58 DEBUG 03:32:58 DEBUG "instance": "10.142.0.43:9100" 03:32:58 DEBUG 03:32:58 DEBUG 03:32:58 DEBUG 03:32:58 DEBUG 03:32:58 DEBUG 03:32:58 DEBUG 03:32:58 DEBUG }, 03:32:58 DEBUG "pod": "ds-cts-2" 03:32:58 DEBUG 03:32:58 DEBUG 03:32:58 DEBUG 03:32:58 DEBUG ] 03:32:58 DEBUG } 03:32:58 DEBUG }, 03:32:58 DEBUG [...] 03:32:58 DEBUG 03:32:58 DEBUG 03:32:58 DEBUG "instance": "10.142.0.43:9100" 03:32:58 DEBUG "pod": "am-767f6b67b5-vbltz" 03:32:58 DEBUG 03:32:58 DEBUG "value": [ 03:32:58 DEBUG }, 03:32:58 DEBUG 03:32:58 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 03:32:58 DEBUG 03:32:58 DEBUG ] 03:32:58 DEBUG "instance": "10.142.0.43:9100" 03:32:58 DEBUG }, 03:32:58 DEBUG 03:32:58 DEBUG [...] 03:32:58 DEBUG "pod": "ds-idrepo-2" 03:32:58 DEBUG }, 03:32:58 DEBUG }, 03:32:58 DEBUG "pod": "ds-cts-2" 03:32:58 DEBUG 1713666778, 03:32:58 DEBUG "value": [ 03:32:58 DEBUG "instance": "10.142.0.43:9100" 03:32:58 DEBUG }, 03:32:58 DEBUG [...] 03:32:58 DEBUG } 03:32:58 DEBUG "value": [ 03:32:58 DEBUG "value": [ 03:32:58 DEBUG 03:32:58 DEBUG 03:32:58 DEBUG }, 03:32:58 DEBUG "value": [ 03:32:58 DEBUG "value": [ 03:32:58 DEBUG }, 03:32:58 DEBUG "27.266666666666666" 03:32:58 DEBUG 1713666778, 03:32:58 DEBUG }, 03:32:58 DEBUG "value": [ 03:32:58 DEBUG 03:32:58 DEBUG } 03:32:58 DEBUG 1713666778, 03:32:58 DEBUG 1713666778, 03:32:58 DEBUG "27925466.266666666" 03:32:58 DEBUG 03:32:58 DEBUG "value": [ 03:32:58 DEBUG 1713666778, 03:32:58 DEBUG 1713666778, 03:32:58 DEBUG "4000.7999999999997" 03:32:58 DEBUG "value": [ 03:32:58 DEBUG "0" 03:32:58 DEBUG "value": [ 03:32:58 DEBUG 1713666778, 03:32:58 DEBUG 03:32:58 DEBUG "27917382.133333333" 03:32:58 DEBUG ] 03:32:58 DEBUG } 03:32:58 DEBUG ] 03:32:58 DEBUG } 03:32:58 DEBUG } 03:32:58 DEBUG ] 03:32:58 DEBUG ] 03:32:58 DEBUG } 03:32:58 DEBUG ] 03:32:58 DEBUG 1713666778, 03:32:58 DEBUG "0" 03:32:58 DEBUG 03:32:58 DEBUG "4462294320128" 03:32:58 DEBUG 1713666778, 03:32:58 DEBUG 03:32:58 DEBUG 03:32:58 DEBUG ] 03:32:58 DEBUG } 03:32:58 DEBUG ] 03:32:58 DEBUG 1713666778, 03:32:58 DEBUG } 03:32:58 DEBUG "0.096666666666664" 03:32:58 DEBUG ] 03:32:58 DEBUG "pod": "am-767f6b67b5-vbltz" 03:32:58 DEBUG ] 03:32:58 DEBUG "0" 03:32:58 DEBUG "pod": "ds-idrepo-2" 03:32:58 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 03:32:58 DEBUG } 03:32:58 DEBUG ] 03:32:58 DEBUG } 03:32:58 DEBUG "0" 03:32:58 DEBUG ] 03:32:58 DEBUG ] 03:32:58 DEBUG } 03:32:58 DEBUG }, 03:32:58 DEBUG } 03:32:58 DEBUG ] 03:32:58 DEBUG }, 03:32:58 DEBUG }, 03:32:58 DEBUG ] 03:32:58 DEBUG } 03:32:58 DEBUG } 03:32:58 DEBUG ] 03:32:58 DEBUG } 03:32:58 DEBUG } 03:32:58 DEBUG ] 03:32:58 DEBUG "value": [ 03:32:58 DEBUG ] 03:32:58 DEBUG } 03:32:58 DEBUG "value": [ 03:32:58 DEBUG "value": [ 03:32:58 DEBUG } 03:32:58 DEBUG } 03:32:58 DEBUG } 03:32:58 DEBUG } 03:32:58 DEBUG } 03:32:58 DEBUG ] 03:32:58 DEBUG 1713666778, 03:32:58 DEBUG } 03:32:58 DEBUG ] 03:32:58 DEBUG 1713666778, 03:32:58 DEBUG 1713666778, 03:32:58 DEBUG } 03:32:58 DEBUG ] 03:32:58 DEBUG } 03:32:58 DEBUG } 03:32:58 DEBUG } 03:32:58 DEBUG } 03:32:58 DEBUG } 03:32:58 DEBUG "0" 03:32:58 DEBUG "0" 03:32:58 DEBUG } 03:32:58 DEBUG "928.1333333333333" 03:32:58 DEBUG } 03:32:58 DEBUG ] 03:32:58 DEBUG ] 03:32:58 DEBUG } 03:32:58 DEBUG ] 03:32:58 DEBUG } 03:32:58 DEBUG } 03:32:58 DEBUG } 03:32:58 DEBUG ] 03:32:58 DEBUG ] 03:32:58 DEBUG ] 03:32:58 DEBUG } 03:32:58 DEBUG } 03:32:58 DEBUG } 03:32:58 DEBUG } 03:32:58 DEBUG } 03:32:58 DEBUG } 03:33:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713666834" 03:33:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666834" 03:33:54 INFO [http_cmd]: http status code OK 03:33:54 ERROR [http_cmd]: ERROR 03:33:54 DEBUG --- status code --- 03:33:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:33:54 DEBUG --- http response --- 03:33:54 ERROR --- status code --- 03:33:54 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 03:33:54 ERROR http status code is 400 (expected [200, 401, 403, 404]) 03:33:54 ERROR --- http response --- 03:33:54 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 03:33:54 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 03:33:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666835" 03:33:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666835" 03:33:55 INFO [http_cmd]: http status code OK 03:33:55 DEBUG --- status code --- 03:33:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:33:55 DEBUG --- http response --- 03:33:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:33:55 DEBUG ----- output ----- 03:33:55 DEBUG { 03:33:55 DEBUG "status": "success", 03:33:55 DEBUG "data": { 03:33:55 DEBUG "resultType": "vector", 03:33:55 DEBUG "result": [ 03:33:55 DEBUG { 03:33:55 DEBUG "metric": { 03:33:55 DEBUG "pod": "ds-cts-0" 03:33:55 DEBUG }, 03:33:55 DEBUG "value": [ 03:33:55 DEBUG 03:33:55 DEBUG 03:33:55 DEBUG 03:33:55 DEBUG [...] 03:33:55 DEBUG 03:33:55 DEBUG 03:33:55 DEBUG 03:33:55 DEBUG "pod": "ds-cts-2" 03:33:55 DEBUG }, 03:33:55 DEBUG "value": [ 03:33:55 DEBUG 1713666835, 03:33:55 DEBUG "0.553533333333335" 03:33:55 DEBUG ] 03:33:55 DEBUG } 03:33:55 DEBUG ] 03:33:55 DEBUG } 03:33:55 DEBUG } 03:33:55 INFO [http_cmd]: http status code OK 03:33:55 DEBUG --- status code --- 03:33:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:33:55 DEBUG --- http response --- 03:33:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:33:55 DEBUG ----- output ----- 03:33:55 DEBUG { 03:33:55 DEBUG "status": "success", 03:33:55 DEBUG "data": { 03:33:55 DEBUG "resultType": "vector", 03:33:55 DEBUG "result": [ 03:33:55 DEBUG { 03:33:55 DEBUG "metric": { 03:33:55 DEBUG "instance": "10.142.0.107:9100" 03:33:55 DEBUG }, 03:33:55 DEBUG "value": [ 03:33:55 DEBUG 03:33:55 DEBUG 03:33:55 DEBUG 03:33:55 DEBUG [...] 03:33:55 DEBUG 03:33:55 DEBUG 03:33:55 DEBUG 03:33:55 DEBUG "instance": "10.142.0.43:9100" 03:33:55 DEBUG }, 03:33:55 DEBUG "value": [ 03:33:55 DEBUG 1713666835, 03:33:55 DEBUG "0.36626666666659086" 03:33:55 DEBUG ] 03:33:55 DEBUG } 03:33:55 DEBUG ] 03:33:55 DEBUG } 03:33:55 DEBUG } 03:33:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666836" 03:33:56 INFO [http_cmd]: http status code OK 03:33:56 DEBUG --- status code --- 03:33:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:33:56 DEBUG --- http response --- 03:33:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:33:56 DEBUG ----- output ----- 03:33:56 DEBUG { 03:33:56 DEBUG "status": "success", 03:33:56 DEBUG "data": { 03:33:56 DEBUG "resultType": "vector", 03:33:56 DEBUG "result": [ 03:33:56 DEBUG { 03:33:56 DEBUG "metric": { 03:33:56 DEBUG "pod": "am-767f6b67b5-9zw22" 03:33:56 DEBUG }, 03:33:56 DEBUG "value": [ 03:33:56 DEBUG 03:33:56 DEBUG 03:33:56 DEBUG 03:33:56 DEBUG [...] 03:33:56 DEBUG 03:33:56 DEBUG 03:33:56 DEBUG 03:33:56 DEBUG "pod": "am-767f6b67b5-vbltz" 03:33:56 DEBUG }, 03:33:56 DEBUG "value": [ 03:33:56 DEBUG 1713666836, 03:33:56 DEBUG "3411.6666666666665" 03:33:56 DEBUG ] 03:33:56 DEBUG } 03:33:56 DEBUG ] 03:33:56 DEBUG } 03:33:56 DEBUG } 03:33:56 INFO 03:33:56 INFO [loop_until]: kubectl --namespace=xlou top pods 03:33:56 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:33:56 INFO [loop_until]: OK (rc = 0) 03:33:56 DEBUG --- stdout --- 03:33:56 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 22565m 8801Mi am-767f6b67b5-qcqz2 22429m 8385Mi am-767f6b67b5-vbltz 24557m 8928Mi ds-cts-0 19267m 12526Mi ds-cts-1 19542m 12489Mi ds-cts-2 20214m 12566Mi ds-idrepo-0 9268m 3918Mi ds-idrepo-1 10126m 3963Mi ds-idrepo-2 9116m 3915Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 3m 1179Mi idm-b85f9d978-wnhxb 3m 1239Mi lodemon-54c87f8cbb-jzrjs 15m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-68c5d5b884-p97w5 12149m 1644Mi 03:33:56 DEBUG --- stderr --- 03:33:56 DEBUG 03:33:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666837" 03:33:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666837" 03:33:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666837" 03:33:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666837" 03:33:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666837" 03:33:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713666837" 03:33:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666837" 03:33:57 INFO [http_cmd]: http status code OK 03:33:57 INFO [http_cmd]: http status code OK 03:33:57 INFO [http_cmd]: http status code OK 03:33:57 DEBUG --- status code --- 03:33:57 INFO [http_cmd]: http status code OK 03:33:57 DEBUG --- status code --- 03:33:57 DEBUG --- status code --- 03:33:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:33:57 DEBUG --- status code --- 03:33:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:33:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:33:57 DEBUG --- http response --- 03:33:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:33:57 DEBUG --- http response --- 03:33:57 DEBUG --- http response --- 03:33:57 INFO [http_cmd]: http status code OK 03:33:57 INFO [http_cmd]: http status code OK 03:33:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:33:57 DEBUG --- http response --- 03:33:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:33:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:33:57 DEBUG --- status code --- 03:33:57 DEBUG --- status code --- 03:33:57 DEBUG ----- output ----- 03:33:57 DEBUG ----- output ----- 03:33:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 03:33:57 DEBUG ----- output ----- 03:33:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:33:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:33:57 DEBUG { 03:33:57 DEBUG { 03:33:57 DEBUG ----- output ----- 03:33:57 DEBUG { 03:33:57 DEBUG --- http response --- 03:33:57 DEBUG --- http response --- 03:33:57 DEBUG "status": "success", 03:33:57 DEBUG "status": "success", 03:33:57 DEBUG { 03:33:57 DEBUG "status": "success", 03:33:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:33:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 03:33:57 DEBUG "data": { 03:33:57 INFO [http_cmd]: http status code OK 03:33:57 DEBUG "data": { 03:33:57 DEBUG "status": "success", 03:33:57 DEBUG "data": { 03:33:57 DEBUG ----- output ----- 03:33:57 DEBUG ----- output ----- 03:33:57 DEBUG "resultType": "vector", 03:33:57 DEBUG --- status code --- 03:33:57 DEBUG "resultType": "vector", 03:33:57 DEBUG "data": { 03:33:57 DEBUG "resultType": "vector", 03:33:57 DEBUG { 03:33:57 DEBUG { 03:33:57 DEBUG "result": [ 03:33:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:33:57 DEBUG "result": [ 03:33:57 DEBUG "resultType": "vector", 03:33:57 DEBUG "result": [ 03:33:57 DEBUG "status": "success", 03:33:57 DEBUG "status": "success", 03:33:57 DEBUG { 03:33:57 DEBUG --- http response --- 03:33:57 DEBUG { 03:33:57 DEBUG "result": [ 03:33:57 DEBUG { 03:33:57 DEBUG "data": { 03:33:57 DEBUG "data": { 03:33:57 DEBUG "metric": { 03:33:57 DEBUG "metric": { 03:33:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:33:57 DEBUG { 03:33:57 DEBUG "metric": { 03:33:57 DEBUG "resultType": "vector", 03:33:57 DEBUG "resultType": "vector", 03:33:57 DEBUG "pod": "am-767f6b67b5-9zw22" 03:33:57 DEBUG }, 03:33:57 DEBUG ----- output ----- 03:33:57 DEBUG "metric": { 03:33:57 DEBUG "pod": "am-767f6b67b5-9zw22" 03:33:57 DEBUG "result": [ 03:33:57 DEBUG "result": [ 03:33:57 DEBUG "pod": "ds-idrepo-0" 03:33:57 DEBUG "value": [ 03:33:57 DEBUG { 03:33:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:33:57 DEBUG }, 03:33:57 DEBUG { 03:33:57 DEBUG { 03:33:57 DEBUG }, 03:33:57 DEBUG 03:33:57 DEBUG "status": "success", 03:33:57 DEBUG "data": { 03:33:57 DEBUG "value": [ 03:33:57 DEBUG "metric": { 03:33:57 DEBUG "metric": { 03:33:57 DEBUG "value": [ 03:33:57 DEBUG 03:33:57 DEBUG }, 03:33:57 DEBUG "resultType": "vector", 03:33:57 DEBUG 03:33:57 DEBUG "pod": "am-767f6b67b5-9zw22" 03:33:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:33:57 DEBUG 03:33:57 DEBUG 03:33:57 DEBUG "value": [ 03:33:57 DEBUG "result": [ 03:33:57 DEBUG 03:33:57 DEBUG }, 03:33:57 DEBUG }, 03:33:57 DEBUG 03:33:57 DEBUG [...] 03:33:57 DEBUG 03:33:57 DEBUG { 03:33:57 DEBUG 03:33:57 DEBUG "value": [ 03:33:57 DEBUG "value": [ 03:33:57 DEBUG 03:33:57 DEBUG 03:33:57 DEBUG 03:33:57 DEBUG "metric": { 03:33:57 DEBUG [...] 03:33:57 DEBUG 03:33:57 DEBUG 03:33:57 DEBUG [...] 03:33:57 DEBUG 03:33:57 DEBUG 03:33:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:33:57 DEBUG 03:33:57 DEBUG 03:33:57 DEBUG 03:33:57 DEBUG 03:33:57 DEBUG 03:33:57 DEBUG [...] 03:33:57 DEBUG }, 03:33:57 DEBUG 03:33:57 DEBUG 03:33:57 DEBUG 03:33:57 DEBUG 03:33:57 DEBUG "pod": "am-767f6b67b5-vbltz" 03:33:57 DEBUG 03:33:57 DEBUG "value": [ 03:33:57 DEBUG 03:33:57 DEBUG [...] 03:33:57 DEBUG [...] 03:33:57 DEBUG 03:33:57 DEBUG }, 03:33:57 DEBUG 03:33:57 DEBUG 03:33:57 DEBUG "pod": "am-767f6b67b5-vbltz" 03:33:57 DEBUG 03:33:57 DEBUG 03:33:57 DEBUG "pod": "ds-idrepo-2" 03:33:57 DEBUG "value": [ 03:33:57 DEBUG 03:33:57 DEBUG 03:33:57 DEBUG }, 03:33:57 DEBUG 03:33:57 DEBUG 03:33:57 DEBUG }, 03:33:57 DEBUG 1713666837, 03:33:57 DEBUG "pod": "ds-cts-0" 03:33:57 DEBUG 03:33:57 DEBUG "value": [ 03:33:57 DEBUG 03:33:57 DEBUG 03:33:57 DEBUG "value": [ 03:33:57 DEBUG "0" 03:33:57 DEBUG }, 03:33:57 DEBUG [...] 03:33:57 DEBUG 1713666837, 03:33:57 DEBUG "pod": "am-767f6b67b5-vbltz" 03:33:57 DEBUG "pod": "ds-cts-0" 03:33:57 DEBUG 1713666837, 03:33:57 DEBUG ] 03:33:57 DEBUG "value": [ 03:33:57 DEBUG 03:33:57 DEBUG "412761.5" 03:33:57 DEBUG }, 03:33:57 DEBUG }, 03:33:57 DEBUG "0" 03:33:57 DEBUG } 03:33:57 DEBUG 1713666837, 03:33:57 DEBUG 03:33:57 DEBUG ] 03:33:57 DEBUG "value": [ 03:33:57 DEBUG "value": [ 03:33:57 DEBUG ] 03:33:57 DEBUG ] 03:33:57 DEBUG "377471023.63855517" 03:33:57 DEBUG 03:33:57 DEBUG } 03:33:57 DEBUG 1713666837, 03:33:57 DEBUG 1713666837, 03:33:57 DEBUG } 03:33:57 DEBUG } 03:33:57 DEBUG ] 03:33:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:33:57 DEBUG ] 03:33:57 DEBUG "3411.4333333333334" 03:33:57 DEBUG "9.910366715997492" 03:33:57 DEBUG ] 03:33:57 DEBUG } 03:33:57 DEBUG } 03:33:57 DEBUG }, 03:33:57 DEBUG } 03:33:57 DEBUG ] 03:33:57 DEBUG ] 03:33:57 DEBUG } 03:33:57 DEBUG ] 03:33:57 DEBUG "value": [ 03:33:57 DEBUG } 03:33:57 INFO 03:33:57 DEBUG } 03:33:57 DEBUG } 03:33:57 DEBUG } 03:33:57 DEBUG } 03:33:57 DEBUG 1713666837, 03:33:57 INFO [loop_until]: kubectl --namespace=xlou top node 03:33:57 DEBUG ] 03:33:57 DEBUG ] 03:33:57 DEBUG } 03:33:57 DEBUG "13.205984576836627" 03:33:57 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:33:57 DEBUG } 03:33:57 DEBUG } 03:33:57 DEBUG ] 03:33:57 DEBUG } 03:33:57 DEBUG } 03:33:57 DEBUG } 03:33:57 DEBUG ] 03:33:57 DEBUG } 03:33:57 DEBUG } 03:33:57 INFO [loop_until]: OK (rc = 0) 03:33:57 DEBUG --- stdout --- 03:33:57 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 65m 0% 6289Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 10627m 35% 7751Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 24449m 81% 11639Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 83m 0% 3338Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 20018m 67% 15844Mi 14% gke-xlou-cdm-default-pool-1b0769c8-wpwl 154m 0% 5268Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 25081m 84% 12125Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 23673m 79% 3408Mi 3% gke-xlou-cdm-ds-0f601751-hmm7 24879m 83% 11266Mi 10% gke-xlou-cdm-ds-0f601751-j488 10859m 36% 7357Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 19759m 66% 17619Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 10805m 36% 8016Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20362m 68% 15284Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 13494m 84% 12448Mi 21% 03:33:57 DEBUG --- stderr --- 03:33:57 DEBUG 03:33:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713666838" 03:33:58 INFO [http_cmd]: http status code OK 03:33:58 DEBUG --- status code --- 03:33:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:33:58 DEBUG --- http response --- 03:33:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:33:58 DEBUG ----- output ----- 03:33:58 DEBUG { 03:33:58 DEBUG "status": "success", 03:33:58 DEBUG "data": { 03:33:58 DEBUG "resultType": "vector", 03:33:58 DEBUG "result": [ 03:33:58 DEBUG { 03:33:58 DEBUG "metric": { 03:33:58 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:33:58 DEBUG }, 03:33:58 DEBUG "value": [ 03:33:58 DEBUG 03:33:58 DEBUG 03:33:58 DEBUG 03:33:58 DEBUG [...] 03:33:58 DEBUG 03:33:58 DEBUG 03:33:58 DEBUG 03:33:58 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:33:58 DEBUG }, 03:33:58 DEBUG "value": [ 03:33:58 DEBUG 1713666838, 03:33:58 DEBUG "1823604736" 03:33:58 DEBUG ] 03:33:58 DEBUG } 03:33:58 DEBUG ] 03:33:58 DEBUG } 03:33:58 DEBUG } 03:33:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713666838" 03:33:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666838" 03:33:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666838" 03:33:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666838" 03:33:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666838" 03:33:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666838" 03:33:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666838" 03:33:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666838" 03:33:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713666838" 03:33:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666838" 03:33:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666838" 03:33:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666838" 03:33:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666838" 03:33:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666838" 03:33:58 INFO [http_cmd]: http status code OK 03:33:58 INFO [http_cmd]: http status code OK 03:33:58 INFO [http_cmd]: http status code OK 03:33:58 INFO [http_cmd]: http status code OK 03:33:58 INFO [http_cmd]: http status code OK 03:33:58 INFO [http_cmd]: http status code OK 03:33:58 INFO [http_cmd]: http status code OK 03:33:58 INFO [http_cmd]: http status code OK 03:33:58 INFO [http_cmd]: http status code OK 03:33:58 INFO [http_cmd]: http status code OK 03:33:58 INFO [http_cmd]: http status code OK 03:33:58 DEBUG --- status code --- 03:33:58 DEBUG --- status code --- 03:33:58 DEBUG --- status code --- 03:33:58 DEBUG --- status code --- 03:33:58 DEBUG --- status code --- 03:33:58 DEBUG --- status code --- 03:33:58 INFO [http_cmd]: http status code OK 03:33:58 DEBUG --- status code --- 03:33:58 INFO [http_cmd]: http status code OK 03:33:58 DEBUG --- status code --- 03:33:58 DEBUG --- status code --- 03:33:58 DEBUG --- status code --- 03:33:58 DEBUG --- status code --- 03:33:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:33:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:33:58 INFO [http_cmd]: http status code OK 03:33:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:33:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:33:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:33:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:33:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:33:58 DEBUG --- status code --- 03:33:58 DEBUG --- status code --- 03:33:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:33:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:33:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:33:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:33:58 DEBUG --- http response --- 03:33:58 DEBUG --- http response --- 03:33:58 DEBUG --- status code --- 03:33:58 DEBUG --- http response --- 03:33:58 DEBUG --- http response --- 03:33:58 DEBUG --- http response --- 03:33:58 DEBUG --- http response --- 03:33:58 DEBUG --- http response --- 03:33:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:33:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:33:58 DEBUG --- http response --- 03:33:58 DEBUG --- http response --- 03:33:58 DEBUG --- http response --- 03:33:58 DEBUG --- http response --- 03:33:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:33:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:33:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:33:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 03:33:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:33:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:33:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:33:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:33:58 DEBUG ----- output ----- 03:33:58 DEBUG --- http response --- 03:33:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:33:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:33:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:33:58 DEBUG ----- output ----- 03:33:58 DEBUG ----- output ----- 03:33:58 DEBUG --- http response --- 03:33:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:33:58 DEBUG ----- output ----- 03:33:58 DEBUG ----- output ----- 03:33:58 DEBUG ----- output ----- 03:33:58 DEBUG ----- output ----- 03:33:58 DEBUG --- http response --- 03:33:58 DEBUG { 03:33:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 03:33:58 DEBUG ----- output ----- 03:33:58 DEBUG ----- output ----- 03:33:58 DEBUG ----- output ----- 03:33:58 DEBUG { 03:33:58 DEBUG { 03:33:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:33:58 DEBUG ----- output ----- 03:33:58 DEBUG { 03:33:58 DEBUG { 03:33:58 DEBUG { 03:33:58 DEBUG { 03:33:58 DEBUG "status": "success", 03:33:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:33:58 DEBUG ----- output ----- 03:33:58 DEBUG { 03:33:58 DEBUG { 03:33:58 DEBUG { 03:33:58 DEBUG "status": "success", 03:33:58 DEBUG "data": { 03:33:58 DEBUG "resultType": "vector", 03:33:58 DEBUG { 03:33:58 DEBUG "status": "success", 03:33:58 DEBUG "status": "success", 03:33:58 DEBUG "status": "success", 03:33:58 DEBUG "status": "success", 03:33:58 DEBUG "data": { 03:33:58 DEBUG { 03:33:58 DEBUG ----- output ----- 03:33:58 DEBUG "status": "success", 03:33:58 DEBUG "status": "success", 03:33:58 DEBUG "status": "success", 03:33:58 DEBUG "result": [ 03:33:58 DEBUG "status": "success", 03:33:58 DEBUG ----- output ----- 03:33:58 DEBUG "status": "success", 03:33:58 DEBUG "data": { 03:33:58 DEBUG "data": { 03:33:58 DEBUG "data": { 03:33:58 DEBUG "resultType": "vector", 03:33:58 DEBUG "resultType": "vector", 03:33:58 DEBUG "status": "success", 03:33:58 DEBUG { 03:33:58 DEBUG "data": { 03:33:58 DEBUG "data": { 03:33:58 DEBUG "resultType": "vector", 03:33:58 DEBUG { 03:33:58 DEBUG "data": { 03:33:58 DEBUG { 03:33:58 DEBUG "data": { 03:33:58 DEBUG "resultType": "vector", 03:33:58 DEBUG "resultType": "vector", 03:33:58 DEBUG "result": [ 03:33:58 DEBUG "data": { 03:33:58 DEBUG "result": [ 03:33:58 DEBUG "data": { 03:33:58 DEBUG "status": "success", 03:33:58 DEBUG "resultType": "vector", 03:33:58 DEBUG "data": { 03:33:58 DEBUG "result": [ 03:33:58 DEBUG "metric": { 03:33:58 DEBUG "resultType": "vector", 03:33:58 DEBUG "status": "success", 03:33:58 DEBUG "resultType": "vector", 03:33:58 DEBUG "result": [ 03:33:58 DEBUG "result": [ 03:33:58 DEBUG { 03:33:58 DEBUG "resultType": "vector", 03:33:58 DEBUG { 03:33:58 DEBUG "resultType": "vector", 03:33:58 DEBUG "data": { 03:33:58 DEBUG "result": [ 03:33:58 DEBUG "resultType": "vector", 03:33:58 DEBUG { 03:33:58 DEBUG "pod": "am-767f6b67b5-9zw22" 03:33:58 DEBUG "result": [ 03:33:58 DEBUG "data": { 03:33:58 DEBUG "result": [ 03:33:58 DEBUG { 03:33:58 DEBUG { 03:33:58 DEBUG "metric": { 03:33:58 DEBUG "result": [ 03:33:58 DEBUG "metric": { 03:33:58 DEBUG "result": [ 03:33:58 DEBUG "resultType": "vector", 03:33:58 DEBUG { 03:33:58 DEBUG "result": [ 03:33:58 DEBUG "metric": { 03:33:58 DEBUG }, 03:33:58 DEBUG { 03:33:58 DEBUG "resultType": "vector", 03:33:58 DEBUG { 03:33:58 DEBUG "metric": { 03:33:58 DEBUG "metric": { 03:33:58 DEBUG "instance": "10.142.0.107:9100" 03:33:58 DEBUG { 03:33:58 DEBUG "instance": "10.142.0.107:9100" 03:33:58 DEBUG { 03:33:58 DEBUG "result": [ 03:33:58 DEBUG "metric": { 03:33:58 DEBUG "instance": "10.142.0.107:9100" 03:33:58 DEBUG "pod": "ds-idrepo-0" 03:33:58 DEBUG "value": [ 03:33:58 DEBUG "metric": { 03:33:58 DEBUG "result": [ 03:33:58 DEBUG "metric": { 03:33:58 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:33:58 DEBUG "pod": "ds-cts-0" 03:33:58 DEBUG }, 03:33:58 DEBUG "metric": { 03:33:58 DEBUG }, 03:33:58 DEBUG "metric": { 03:33:58 DEBUG { 03:33:58 DEBUG { 03:33:58 DEBUG }, 03:33:58 DEBUG }, 03:33:58 DEBUG 03:33:58 DEBUG "instance": "10.142.0.107:9100" 03:33:58 DEBUG { 03:33:58 DEBUG "pod": "am-767f6b67b5-9zw22" 03:33:58 DEBUG }, 03:33:58 DEBUG "value": [ 03:33:58 DEBUG "value": [ 03:33:58 DEBUG 03:33:58 DEBUG "value": [ 03:33:58 DEBUG "pod": "am-767f6b67b5-9zw22" 03:33:58 DEBUG "metric": { 03:33:58 DEBUG "metric": { 03:33:58 DEBUG "value": [ 03:33:58 DEBUG "value": [ 03:33:58 DEBUG 03:33:58 DEBUG }, 03:33:58 DEBUG "metric": { 03:33:58 DEBUG }, 03:33:58 DEBUG 03:33:58 DEBUG }, 03:33:58 DEBUG "value": [ 03:33:58 DEBUG 03:33:58 DEBUG "pod": "ds-cts-0" 03:33:58 DEBUG }, 03:33:58 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:33:58 DEBUG "pod": "ds-idrepo-0" 03:33:58 DEBUG 03:33:58 DEBUG 03:33:58 DEBUG 03:33:58 DEBUG "value": [ 03:33:58 DEBUG "pod": "ds-idrepo-0" 03:33:58 DEBUG "value": [ 03:33:58 DEBUG 03:33:58 DEBUG 03:33:58 DEBUG 03:33:58 DEBUG 03:33:58 DEBUG }, 03:33:58 DEBUG "value": [ 03:33:58 DEBUG }, 03:33:58 DEBUG }, 03:33:58 DEBUG 03:33:58 DEBUG 03:33:58 DEBUG [...] 03:33:58 DEBUG 03:33:58 DEBUG }, 03:33:58 DEBUG 03:33:58 DEBUG 03:33:58 DEBUG 03:33:58 DEBUG 03:33:58 DEBUG 03:33:58 DEBUG "value": [ 03:33:58 DEBUG 03:33:58 DEBUG "value": [ 03:33:58 DEBUG "value": [ 03:33:58 DEBUG 03:33:58 DEBUG 03:33:58 DEBUG 03:33:58 DEBUG 03:33:58 DEBUG "value": [ 03:33:58 DEBUG 03:33:58 DEBUG [...] 03:33:58 DEBUG 03:33:58 DEBUG [...] 03:33:58 DEBUG [...] 03:33:58 DEBUG 03:33:58 DEBUG 03:33:58 DEBUG 03:33:58 DEBUG 03:33:58 DEBUG [...] 03:33:58 DEBUG [...] 03:33:58 DEBUG 03:33:58 DEBUG 03:33:58 DEBUG 03:33:58 DEBUG 03:33:58 DEBUG [...] 03:33:58 DEBUG [...] 03:33:58 DEBUG 03:33:58 DEBUG 03:33:58 DEBUG 03:33:58 DEBUG 03:33:58 DEBUG 03:33:58 DEBUG 03:33:58 DEBUG 03:33:58 DEBUG 03:33:58 DEBUG 03:33:58 DEBUG [...] 03:33:58 DEBUG 03:33:58 DEBUG 03:33:58 DEBUG 03:33:58 DEBUG 03:33:58 DEBUG 03:33:58 DEBUG 03:33:58 DEBUG 03:33:58 DEBUG [...] 03:33:58 DEBUG 03:33:58 DEBUG 03:33:58 DEBUG 03:33:58 DEBUG 03:33:58 DEBUG "pod": "am-767f6b67b5-vbltz" 03:33:58 DEBUG 03:33:58 DEBUG 03:33:58 DEBUG 03:33:58 DEBUG 03:33:58 DEBUG 03:33:58 DEBUG 03:33:58 DEBUG 03:33:58 DEBUG [...] 03:33:58 DEBUG 03:33:58 DEBUG [...] 03:33:58 DEBUG [...] 03:33:58 DEBUG 03:33:58 DEBUG "instance": "10.142.0.43:9100" 03:33:58 DEBUG }, 03:33:58 DEBUG 03:33:58 DEBUG [...] 03:33:58 DEBUG 03:33:58 DEBUG 03:33:58 DEBUG 03:33:58 DEBUG "pod": "ds-cts-2" 03:33:58 DEBUG "instance": "10.142.0.43:9100" 03:33:58 DEBUG }, 03:33:58 DEBUG 03:33:58 DEBUG 03:33:58 DEBUG 03:33:58 DEBUG 03:33:58 DEBUG }, 03:33:58 DEBUG "value": [ 03:33:58 DEBUG 03:33:58 DEBUG 03:33:58 DEBUG "pod": "am-767f6b67b5-vbltz" 03:33:58 DEBUG "pod": "ds-cts-0" 03:33:58 DEBUG }, 03:33:58 DEBUG "value": [ 03:33:58 DEBUG 03:33:58 DEBUG 03:33:58 DEBUG "instance": "10.142.0.43:9100" 03:33:58 DEBUG 03:33:58 DEBUG 03:33:58 DEBUG "pod": "ds-cts-2" 03:33:58 DEBUG 1713666838, 03:33:58 DEBUG "instance": "10.142.0.43:9100" 03:33:58 DEBUG "value": [ 03:33:58 DEBUG 03:33:58 DEBUG }, 03:33:58 DEBUG }, 03:33:58 DEBUG "value": [ 03:33:58 DEBUG 1713666838, 03:33:58 DEBUG 03:33:58 DEBUG "pod": "am-767f6b67b5-vbltz" 03:33:58 DEBUG }, 03:33:58 DEBUG 03:33:58 DEBUG 03:33:58 DEBUG }, 03:33:58 DEBUG "4467048854528" 03:33:58 DEBUG }, 03:33:58 DEBUG 1713666838, 03:33:58 DEBUG 03:33:58 DEBUG "value": [ 03:33:58 DEBUG 1713666838, 03:33:58 DEBUG 1713666838, 03:33:58 DEBUG "107567878.66666666" 03:33:58 DEBUG 03:33:58 DEBUG }, 03:33:58 DEBUG "value": [ 03:33:58 DEBUG "pod": "ds-cts-0" 03:33:58 DEBUG "pod": "ds-idrepo-2" 03:33:58 DEBUG "value": [ 03:33:58 DEBUG ] 03:33:58 DEBUG "value": [ 03:33:58 DEBUG "182.5775" 03:33:58 DEBUG "pod": "ds-idrepo-2" 03:33:58 DEBUG "3411.4333333333334" 03:33:58 DEBUG "0" 03:33:58 DEBUG ] 03:33:58 DEBUG } 03:33:58 DEBUG "pod": "ds-cts-2" 03:33:58 DEBUG "value": [ 03:33:58 DEBUG 1713666838, 03:33:58 DEBUG }, 03:33:58 DEBUG }, 03:33:58 DEBUG 1713666838, 03:33:58 DEBUG } 03:33:58 DEBUG 1713666838, 03:33:58 DEBUG ] 03:33:58 DEBUG }, 03:33:58 DEBUG ] 03:33:58 DEBUG "value": [ 03:33:58 DEBUG ] 03:33:58 DEBUG }, 03:33:58 DEBUG 1713666838, 03:33:58 DEBUG ] 03:33:58 DEBUG "0.0676666666666752" 03:33:58 DEBUG "value": [ 03:33:58 DEBUG "value": [ 03:33:58 DEBUG 1713666838, 03:33:58 DEBUG ] 03:33:58 DEBUG "107563065.83333333" 03:33:58 DEBUG } 03:33:58 DEBUG "value": [ 03:33:58 DEBUG } 03:33:58 DEBUG 1713666838, 03:33:58 DEBUG } 03:33:58 DEBUG "value": [ 03:33:58 DEBUG "3843.8999999999996" 03:33:58 DEBUG } 03:33:58 DEBUG ] 03:33:58 DEBUG 1713666838, 03:33:58 DEBUG "0" 03:33:58 DEBUG "0" 03:33:58 DEBUG } 03:33:58 DEBUG ] 03:33:58 DEBUG ] 03:33:58 DEBUG 1713666838, 03:33:58 DEBUG ] 03:33:58 DEBUG "39333869.36661612" 03:33:58 DEBUG } 03:33:58 DEBUG 1713666838, 03:33:58 DEBUG ] 03:33:58 DEBUG } 03:33:58 DEBUG } 03:33:58 DEBUG "38029938.308643416" 03:33:58 DEBUG ] 03:33:58 DEBUG ] 03:33:58 DEBUG } 03:33:58 DEBUG } 03:33:58 DEBUG } 03:33:58 DEBUG } 03:33:58 DEBUG } 03:33:58 DEBUG "0" 03:33:58 DEBUG "30.866666666666667" 03:33:58 DEBUG ] 03:33:58 DEBUG ] 03:33:58 DEBUG ] 03:33:58 DEBUG ] 03:33:58 DEBUG } 03:33:58 DEBUG } 03:33:58 DEBUG ] 03:33:58 DEBUG ] 03:33:58 DEBUG } 03:33:58 DEBUG ] 03:33:58 DEBUG ] 03:33:58 DEBUG } 03:33:58 DEBUG } 03:33:58 DEBUG } 03:33:58 DEBUG } 03:33:58 DEBUG ] 03:33:58 DEBUG ] 03:33:58 DEBUG } 03:33:58 DEBUG } 03:33:58 DEBUG } 03:33:58 DEBUG } 03:33:58 DEBUG } 03:33:58 DEBUG } 03:33:58 DEBUG ] 03:33:58 DEBUG } 03:33:58 DEBUG } 03:33:58 DEBUG } 03:33:58 DEBUG ] 03:33:58 DEBUG ] 03:33:58 DEBUG ] 03:33:58 DEBUG } 03:33:58 DEBUG } 03:33:58 DEBUG } 03:33:58 DEBUG } 03:33:58 DEBUG } 03:33:58 DEBUG } 03:33:58 DEBUG } 03:33:58 DEBUG } 03:33:58 DEBUG } 03:33:58 DEBUG } 03:33:58 DEBUG } 03:34:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713666894" 03:34:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666894" 03:34:54 INFO [http_cmd]: http status code OK 03:34:54 ERROR [http_cmd]: ERROR 03:34:54 DEBUG --- status code --- 03:34:54 ERROR --- status code --- 03:34:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:34:54 ERROR http status code is 400 (expected [200, 401, 403, 404]) 03:34:54 DEBUG --- http response --- 03:34:54 ERROR --- http response --- 03:34:54 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 03:34:54 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 03:34:54 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 03:34:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666895" 03:34:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666895" 03:34:55 INFO [http_cmd]: http status code OK 03:34:55 DEBUG --- status code --- 03:34:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:34:55 DEBUG --- http response --- 03:34:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:34:55 DEBUG ----- output ----- 03:34:55 DEBUG { 03:34:55 DEBUG "status": "success", 03:34:55 DEBUG "data": { 03:34:55 DEBUG "resultType": "vector", 03:34:55 DEBUG "result": [ 03:34:55 DEBUG { 03:34:55 DEBUG "metric": { 03:34:55 DEBUG "pod": "ds-cts-0" 03:34:55 DEBUG }, 03:34:55 DEBUG "value": [ 03:34:55 DEBUG 03:34:55 DEBUG 03:34:55 DEBUG 03:34:55 DEBUG [...] 03:34:55 DEBUG 03:34:55 DEBUG 03:34:55 DEBUG 03:34:55 DEBUG "pod": "ds-cts-2" 03:34:55 DEBUG }, 03:34:55 DEBUG "value": [ 03:34:55 DEBUG 1713666895, 03:34:55 DEBUG "0.05160000000000006" 03:34:55 DEBUG ] 03:34:55 DEBUG } 03:34:55 DEBUG ] 03:34:55 DEBUG } 03:34:55 DEBUG } 03:34:55 INFO [http_cmd]: http status code OK 03:34:55 DEBUG --- status code --- 03:34:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:34:55 DEBUG --- http response --- 03:34:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:34:55 DEBUG ----- output ----- 03:34:55 DEBUG { 03:34:55 DEBUG "status": "success", 03:34:55 DEBUG "data": { 03:34:55 DEBUG "resultType": "vector", 03:34:55 DEBUG "result": [ 03:34:55 DEBUG { 03:34:55 DEBUG "metric": { 03:34:55 DEBUG "instance": "10.142.0.107:9100" 03:34:55 DEBUG }, 03:34:55 DEBUG "value": [ 03:34:55 DEBUG 03:34:55 DEBUG 03:34:55 DEBUG 03:34:55 DEBUG [...] 03:34:55 DEBUG 03:34:55 DEBUG 03:34:55 DEBUG 03:34:55 DEBUG "instance": "10.142.0.43:9100" 03:34:55 DEBUG }, 03:34:55 DEBUG "value": [ 03:34:55 DEBUG 1713666895, 03:34:55 DEBUG "0.3960000000000207" 03:34:55 DEBUG ] 03:34:55 DEBUG } 03:34:55 DEBUG ] 03:34:55 DEBUG } 03:34:55 DEBUG } 03:34:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666896" 03:34:56 INFO [http_cmd]: http status code OK 03:34:56 DEBUG --- status code --- 03:34:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:34:56 DEBUG --- http response --- 03:34:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:34:56 DEBUG ----- output ----- 03:34:56 DEBUG { 03:34:56 DEBUG "status": "success", 03:34:56 DEBUG "data": { 03:34:56 DEBUG "resultType": "vector", 03:34:56 DEBUG "result": [ 03:34:56 DEBUG { 03:34:56 DEBUG "metric": { 03:34:56 DEBUG "pod": "am-767f6b67b5-9zw22" 03:34:56 DEBUG }, 03:34:56 DEBUG "value": [ 03:34:56 DEBUG 03:34:56 DEBUG 03:34:56 DEBUG 03:34:56 DEBUG [...] 03:34:56 DEBUG 03:34:56 DEBUG 03:34:56 DEBUG 03:34:56 DEBUG "pod": "am-767f6b67b5-vbltz" 03:34:56 DEBUG }, 03:34:56 DEBUG "value": [ 03:34:56 DEBUG 1713666896, 03:34:56 DEBUG "2191.1666666666665" 03:34:56 DEBUG ] 03:34:56 DEBUG } 03:34:56 DEBUG ] 03:34:56 DEBUG } 03:34:56 DEBUG } 03:34:56 INFO 03:34:56 INFO [loop_until]: kubectl --namespace=xlou top pods 03:34:56 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:34:56 INFO [loop_until]: OK (rc = 0) 03:34:56 DEBUG --- stdout --- 03:34:56 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 13410m 8828Mi am-767f6b67b5-qcqz2 13393m 9161Mi am-767f6b67b5-vbltz 15155m 8927Mi ds-cts-0 20138m 12517Mi ds-cts-1 18907m 12482Mi ds-cts-2 20094m 12560Mi ds-idrepo-0 7117m 3917Mi ds-idrepo-1 6410m 3962Mi ds-idrepo-2 7022m 3915Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1179Mi idm-b85f9d978-wnhxb 3m 1239Mi lodemon-54c87f8cbb-jzrjs 27m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-68c5d5b884-p97w5 15170m 3811Mi 03:34:56 DEBUG --- stderr --- 03:34:56 DEBUG 03:34:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666897" 03:34:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666897" 03:34:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666897" 03:34:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666897" 03:34:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666897" 03:34:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666897" 03:34:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713666897" 03:34:57 INFO [http_cmd]: http status code OK 03:34:57 INFO [http_cmd]: http status code OK 03:34:57 INFO [http_cmd]: http status code OK 03:34:57 DEBUG --- status code --- 03:34:57 DEBUG --- status code --- 03:34:57 DEBUG --- status code --- 03:34:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:34:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:34:57 INFO [http_cmd]: http status code OK 03:34:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:34:57 DEBUG --- http response --- 03:34:57 DEBUG --- http response --- 03:34:57 INFO [http_cmd]: http status code OK 03:34:57 INFO [http_cmd]: http status code OK 03:34:57 DEBUG --- http response --- 03:34:57 DEBUG --- status code --- 03:34:57 INFO [http_cmd]: http status code OK 03:34:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:34:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 03:34:57 DEBUG --- status code --- 03:34:57 DEBUG --- status code --- 03:34:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:34:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:34:57 DEBUG --- status code --- 03:34:57 DEBUG ----- output ----- 03:34:57 DEBUG ----- output ----- 03:34:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:34:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:34:57 DEBUG ----- output ----- 03:34:57 DEBUG --- http response --- 03:34:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:34:57 DEBUG { 03:34:57 DEBUG { 03:34:57 DEBUG --- http response --- 03:34:57 DEBUG --- http response --- 03:34:57 DEBUG { 03:34:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:34:57 DEBUG --- http response --- 03:34:57 DEBUG "status": "success", 03:34:57 DEBUG "status": "success", 03:34:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:34:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:34:57 DEBUG "status": "success", 03:34:57 DEBUG ----- output ----- 03:34:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 03:34:57 DEBUG "data": { 03:34:57 DEBUG "data": { 03:34:57 DEBUG ----- output ----- 03:34:57 DEBUG ----- output ----- 03:34:57 DEBUG "data": { 03:34:57 DEBUG "resultType": "vector", 03:34:57 DEBUG ----- output ----- 03:34:57 DEBUG "resultType": "vector", 03:34:57 DEBUG "resultType": "vector", 03:34:57 DEBUG { 03:34:57 DEBUG { 03:34:57 DEBUG "result": [ 03:34:57 DEBUG { 03:34:57 DEBUG { 03:34:57 DEBUG "result": [ 03:34:57 DEBUG "result": [ 03:34:57 DEBUG "status": "success", 03:34:57 DEBUG "status": "success", 03:34:57 DEBUG { 03:34:57 DEBUG "status": "success", 03:34:57 DEBUG "status": "success", 03:34:57 DEBUG { 03:34:57 DEBUG { 03:34:57 DEBUG "data": { 03:34:57 DEBUG "data": { 03:34:57 DEBUG "metric": { 03:34:57 DEBUG "data": { 03:34:57 DEBUG "data": { 03:34:57 DEBUG "resultType": "vector", 03:34:57 DEBUG "metric": { 03:34:57 DEBUG "resultType": "vector", 03:34:57 DEBUG "resultType": "vector", 03:34:57 DEBUG "pod": "am-767f6b67b5-9zw22" 03:34:57 DEBUG "resultType": "vector", 03:34:57 DEBUG "metric": { 03:34:57 DEBUG "result": [ 03:34:57 DEBUG "pod": "ds-idrepo-1" 03:34:57 DEBUG "result": [ 03:34:57 DEBUG "result": [ 03:34:57 DEBUG }, 03:34:57 DEBUG "result": [ 03:34:57 DEBUG "pod": "am-767f6b67b5-9zw22" 03:34:57 DEBUG { 03:34:57 DEBUG }, 03:34:57 DEBUG { 03:34:57 DEBUG { 03:34:57 DEBUG "value": [ 03:34:57 DEBUG { 03:34:57 DEBUG }, 03:34:57 DEBUG "metric": { 03:34:57 DEBUG "value": [ 03:34:57 DEBUG "metric": { 03:34:57 DEBUG "metric": { 03:34:57 DEBUG 03:34:57 DEBUG "metric": { 03:34:57 DEBUG "value": [ 03:34:57 DEBUG "pod": "ds-idrepo-0" 03:34:57 DEBUG 03:34:57 DEBUG "pod": "am-767f6b67b5-9zw22" 03:34:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:34:57 DEBUG 03:34:57 DEBUG "pod": "ds-idrepo-1" 03:34:57 DEBUG 03:34:57 DEBUG }, 03:34:57 DEBUG 03:34:57 DEBUG }, 03:34:57 DEBUG }, 03:34:57 DEBUG 03:34:57 DEBUG }, 03:34:57 DEBUG 03:34:57 DEBUG "value": [ 03:34:57 DEBUG 03:34:57 DEBUG "value": [ 03:34:57 DEBUG "value": [ 03:34:57 DEBUG [...] 03:34:57 DEBUG "value": [ 03:34:57 DEBUG 03:34:57 DEBUG 03:34:57 DEBUG [...] 03:34:57 DEBUG 03:34:57 DEBUG 03:34:57 DEBUG 03:34:57 DEBUG 03:34:57 DEBUG [...] 03:34:57 DEBUG 03:34:57 DEBUG 03:34:57 DEBUG 03:34:57 DEBUG 03:34:57 DEBUG 03:34:57 DEBUG 03:34:57 DEBUG 03:34:57 DEBUG 03:34:57 DEBUG 03:34:57 DEBUG 03:34:57 DEBUG 03:34:57 DEBUG 03:34:57 DEBUG 03:34:57 DEBUG 03:34:57 DEBUG [...] 03:34:57 DEBUG 03:34:57 DEBUG [...] 03:34:57 DEBUG [...] 03:34:57 DEBUG "pod": "am-767f6b67b5-vbltz" 03:34:57 DEBUG [...] 03:34:57 DEBUG 03:34:57 DEBUG 03:34:57 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 03:34:57 DEBUG 03:34:57 DEBUG 03:34:57 DEBUG }, 03:34:57 DEBUG 03:34:57 DEBUG "pod": "am-767f6b67b5-vbltz" 03:34:57 DEBUG 03:34:57 DEBUG }, 03:34:57 DEBUG 03:34:57 DEBUG 03:34:57 DEBUG "value": [ 03:34:57 DEBUG 03:34:57 DEBUG }, 03:34:57 DEBUG 03:34:57 DEBUG "value": [ 03:34:57 DEBUG 03:34:57 DEBUG 03:34:57 DEBUG 1713666897, 03:34:57 DEBUG 03:34:57 DEBUG "value": [ 03:34:57 DEBUG "pod": "ds-idrepo-2" 03:34:57 DEBUG 1713666897, 03:34:57 DEBUG "pod": "am-767f6b67b5-vbltz" 03:34:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:34:57 DEBUG "368462.5" 03:34:57 DEBUG ] 03:34:57 DEBUG 1713666897, 03:34:57 DEBUG }, 03:34:57 DEBUG "0" 03:34:57 DEBUG }, 03:34:57 DEBUG }, 03:34:57 DEBUG } 03:34:57 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 03:34:57 DEBUG "0" 03:34:57 DEBUG "value": [ 03:34:57 DEBUG ] 03:34:57 DEBUG "value": [ 03:34:57 DEBUG "value": [ 03:34:57 DEBUG ] 03:34:57 DEBUG }, 03:34:57 DEBUG ] 03:34:57 DEBUG 1713666897, 03:34:57 DEBUG } 03:34:57 DEBUG 1713666897, 03:34:57 DEBUG 1713666897, 03:34:57 DEBUG } 03:34:57 DEBUG "value": [ 03:34:57 DEBUG } 03:34:57 DEBUG "0" 03:34:57 DEBUG ] 03:34:57 DEBUG "2187.6" 03:34:57 DEBUG "14.613126470138445" 03:34:57 DEBUG } 03:34:57 DEBUG 1713666897, 03:34:57 DEBUG ] 03:34:57 DEBUG ] 03:34:57 DEBUG } 03:34:57 DEBUG } 03:34:57 DEBUG ] 03:34:57 DEBUG "0.00002067565347844163" 03:34:57 DEBUG } 03:34:57 DEBUG } 03:34:57 DEBUG ] 03:34:57 DEBUG } 03:34:57 DEBUG ] 03:34:57 DEBUG } 03:34:57 DEBUG ] 03:34:57 DEBUG } 03:34:57 DEBUG ] 03:34:57 DEBUG } 03:34:57 DEBUG } 03:34:57 DEBUG ] 03:34:57 DEBUG } 03:34:57 DEBUG ] 03:34:57 DEBUG } 03:34:57 DEBUG } 03:34:57 DEBUG } 03:34:57 DEBUG } 03:34:57 DEBUG } 03:34:57 DEBUG } 03:34:57 INFO 03:34:57 INFO [loop_until]: kubectl --namespace=xlou top node 03:34:57 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:34:57 INFO [loop_until]: OK (rc = 0) 03:34:57 DEBUG --- stdout --- 03:34:57 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 65m 0% 6290Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 6463m 21% 7767Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 15423m 51% 11632Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 80m 0% 3344Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 19118m 64% 15833Mi 14% gke-xlou-cdm-default-pool-1b0769c8-wpwl 148m 0% 5268Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 13527m 45% 12108Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 13384m 44% 4010Mi 3% gke-xlou-cdm-ds-0f601751-hmm7 17336m 58% 11188Mi 10% gke-xlou-cdm-ds-0f601751-j488 5924m 19% 7358Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 18690m 62% 17607Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 6810m 22% 8022Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 20203m 67% 15271Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 14977m 94% 18003Mi 30% 03:34:57 DEBUG --- stderr --- 03:34:57 DEBUG 03:34:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713666898" 03:34:58 INFO [http_cmd]: http status code OK 03:34:58 DEBUG --- status code --- 03:34:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:34:58 DEBUG --- http response --- 03:34:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:34:58 DEBUG ----- output ----- 03:34:58 DEBUG { 03:34:58 DEBUG "status": "success", 03:34:58 DEBUG "data": { 03:34:58 DEBUG "resultType": "vector", 03:34:58 DEBUG "result": [ 03:34:58 DEBUG { 03:34:58 DEBUG "metric": { 03:34:58 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:34:58 DEBUG }, 03:34:58 DEBUG "value": [ 03:34:58 DEBUG 03:34:58 DEBUG 03:34:58 DEBUG 03:34:58 DEBUG [...] 03:34:58 DEBUG 03:34:58 DEBUG 03:34:58 DEBUG 03:34:58 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:34:58 DEBUG }, 03:34:58 DEBUG "value": [ 03:34:58 DEBUG 1713666898, 03:34:58 DEBUG "7205658624" 03:34:58 DEBUG ] 03:34:58 DEBUG } 03:34:58 DEBUG ] 03:34:58 DEBUG } 03:34:58 DEBUG } 03:34:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666898" 03:34:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666898" 03:34:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666898" 03:34:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713666898" 03:34:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666898" 03:34:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666898" 03:34:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666898" 03:34:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666898" 03:34:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666898" 03:34:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713666898" 03:34:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666898" 03:34:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666898" 03:34:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666898" 03:34:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666898" 03:34:58 INFO [http_cmd]: http status code OK 03:34:58 INFO [http_cmd]: http status code OK 03:34:58 INFO [http_cmd]: http status code OK 03:34:58 INFO [http_cmd]: http status code OK 03:34:58 INFO [http_cmd]: http status code OK 03:34:58 DEBUG --- status code --- 03:34:58 INFO [http_cmd]: http status code OK 03:34:58 DEBUG --- status code --- 03:34:58 INFO [http_cmd]: http status code OK 03:34:58 INFO [http_cmd]: http status code OK 03:34:58 DEBUG --- status code --- 03:34:58 INFO [http_cmd]: http status code OK 03:34:58 INFO [http_cmd]: http status code OK 03:34:58 INFO [http_cmd]: http status code OK 03:34:58 DEBUG --- status code --- 03:34:58 DEBUG --- status code --- 03:34:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:34:58 DEBUG --- status code --- 03:34:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:34:58 DEBUG --- status code --- 03:34:58 INFO [http_cmd]: http status code OK 03:34:58 INFO [http_cmd]: http status code OK 03:34:58 DEBUG --- status code --- 03:34:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:34:58 DEBUG --- status code --- 03:34:58 DEBUG --- status code --- 03:34:58 DEBUG --- status code --- 03:34:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:34:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:34:58 DEBUG --- http response --- 03:34:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:34:58 DEBUG --- http response --- 03:34:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:34:58 DEBUG --- status code --- 03:34:58 DEBUG --- status code --- 03:34:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:34:58 DEBUG --- http response --- 03:34:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:34:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:34:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:34:58 DEBUG --- http response --- 03:34:58 DEBUG --- http response --- 03:34:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:34:58 DEBUG --- http response --- 03:34:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:34:58 DEBUG --- http response --- 03:34:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:34:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:34:58 DEBUG --- http response --- 03:34:58 INFO [http_cmd]: http status code OK 03:34:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:34:58 DEBUG --- http response --- 03:34:58 DEBUG --- http response --- 03:34:58 DEBUG --- http response --- 03:34:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:34:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:34:58 DEBUG ----- output ----- 03:34:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:34:58 DEBUG ----- output ----- 03:34:58 DEBUG --- http response --- 03:34:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:34:58 DEBUG --- http response --- 03:34:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:34:58 DEBUG --- status code --- 03:34:58 DEBUG ----- output ----- 03:34:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:34:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:34:58 DEBUG ----- output ----- 03:34:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:34:58 DEBUG ----- output ----- 03:34:58 DEBUG { 03:34:58 DEBUG { 03:34:58 DEBUG ----- output ----- 03:34:58 DEBUG ----- output ----- 03:34:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:34:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:34:58 DEBUG ----- output ----- 03:34:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:34:58 DEBUG { 03:34:58 DEBUG ----- output ----- 03:34:58 DEBUG ----- output ----- 03:34:58 DEBUG { 03:34:58 DEBUG ----- output ----- 03:34:58 DEBUG { 03:34:58 DEBUG "status": "success", 03:34:58 DEBUG "status": "success", 03:34:58 DEBUG { 03:34:58 DEBUG { 03:34:58 DEBUG ----- output ----- 03:34:58 DEBUG ----- output ----- 03:34:58 DEBUG { 03:34:58 DEBUG --- http response --- 03:34:58 DEBUG "status": "success", 03:34:58 DEBUG { 03:34:58 DEBUG { 03:34:58 DEBUG "status": "success", 03:34:58 DEBUG { 03:34:58 DEBUG "status": "success", 03:34:58 DEBUG "data": { 03:34:58 DEBUG "data": { 03:34:58 DEBUG "status": "success", 03:34:58 DEBUG "status": "success", 03:34:58 DEBUG { 03:34:58 DEBUG { 03:34:58 DEBUG "status": "success", 03:34:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:34:58 DEBUG "data": { 03:34:58 DEBUG "status": "success", 03:34:58 DEBUG "status": "success", 03:34:58 DEBUG "data": { 03:34:58 DEBUG "status": "success", 03:34:58 DEBUG "data": { 03:34:58 DEBUG "resultType": "vector", 03:34:58 DEBUG "resultType": "vector", 03:34:58 DEBUG "data": { 03:34:58 DEBUG "data": { 03:34:58 DEBUG "status": "success", 03:34:58 DEBUG "status": "success", 03:34:58 DEBUG "data": { 03:34:58 DEBUG ----- output ----- 03:34:58 DEBUG "resultType": "vector", 03:34:58 DEBUG "data": { 03:34:58 DEBUG "data": { 03:34:58 DEBUG "resultType": "vector", 03:34:58 DEBUG "data": { 03:34:58 DEBUG "resultType": "vector", 03:34:58 DEBUG "result": [ 03:34:58 DEBUG "result": [ 03:34:58 DEBUG "resultType": "vector", 03:34:58 DEBUG "resultType": "vector", 03:34:58 DEBUG "data": { 03:34:58 DEBUG "data": { 03:34:58 DEBUG "resultType": "vector", 03:34:58 DEBUG { 03:34:58 DEBUG "result": [ 03:34:58 DEBUG "resultType": "vector", 03:34:58 DEBUG "resultType": "vector", 03:34:58 DEBUG "result": [ 03:34:58 DEBUG "resultType": "vector", 03:34:58 DEBUG "result": [ 03:34:58 DEBUG { 03:34:58 DEBUG { 03:34:58 DEBUG "result": [ 03:34:58 DEBUG "result": [ 03:34:58 DEBUG "resultType": "vector", 03:34:58 DEBUG "resultType": "vector", 03:34:58 DEBUG "result": [ 03:34:58 DEBUG "status": "success", 03:34:58 DEBUG { 03:34:58 DEBUG "result": [ 03:34:58 DEBUG "result": [ 03:34:58 DEBUG { 03:34:58 DEBUG "result": [ 03:34:58 DEBUG { 03:34:58 DEBUG "metric": { 03:34:58 DEBUG "metric": { 03:34:58 DEBUG { 03:34:58 DEBUG { 03:34:58 DEBUG "result": [ 03:34:58 DEBUG "result": [ 03:34:58 DEBUG { 03:34:58 DEBUG "data": { 03:34:58 DEBUG "metric": { 03:34:58 DEBUG { 03:34:58 DEBUG { 03:34:58 DEBUG "metric": { 03:34:58 DEBUG { 03:34:58 DEBUG "metric": { 03:34:58 DEBUG "pod": "ds-cts-0" 03:34:58 DEBUG "pod": "am-767f6b67b5-9zw22" 03:34:58 DEBUG "metric": { 03:34:58 DEBUG "metric": { 03:34:58 DEBUG { 03:34:58 DEBUG { 03:34:58 DEBUG "metric": { 03:34:58 DEBUG "resultType": "vector", 03:34:58 DEBUG "pod": "ds-idrepo-0" 03:34:58 DEBUG "metric": { 03:34:58 DEBUG "metric": { 03:34:58 DEBUG "pod": "am-767f6b67b5-9zw22" 03:34:58 DEBUG "metric": { 03:34:58 DEBUG "instance": "10.142.0.107:9100" 03:34:58 DEBUG }, 03:34:58 DEBUG }, 03:34:58 DEBUG "pod": "ds-idrepo-1" 03:34:58 DEBUG "instance": "10.142.0.107:9100" 03:34:58 DEBUG "metric": { 03:34:58 DEBUG "metric": { 03:34:58 DEBUG "instance": "10.142.0.107:9100" 03:34:58 DEBUG "result": [ 03:34:58 DEBUG }, 03:34:58 DEBUG "pod": "am-767f6b67b5-9zw22" 03:34:58 DEBUG "pod": "ds-idrepo-1" 03:34:58 DEBUG }, 03:34:58 DEBUG "pod": "ds-idrepo-0" 03:34:58 DEBUG }, 03:34:58 DEBUG "value": [ 03:34:58 DEBUG "value": [ 03:34:58 DEBUG }, 03:34:58 DEBUG }, 03:34:58 DEBUG "pod": "ds-cts-0" 03:34:58 DEBUG "instance": "10.142.0.107:9100" 03:34:58 DEBUG }, 03:34:58 DEBUG "value": [ 03:34:58 DEBUG "value": [ 03:34:58 DEBUG }, 03:34:58 DEBUG }, 03:34:58 DEBUG "value": [ 03:34:58 DEBUG }, 03:34:58 DEBUG "value": [ 03:34:58 DEBUG 03:34:58 DEBUG 03:34:58 DEBUG "value": [ 03:34:58 DEBUG "value": [ 03:34:58 DEBUG }, 03:34:58 DEBUG }, 03:34:58 DEBUG 03:34:58 DEBUG { 03:34:58 DEBUG 03:34:58 DEBUG "value": [ 03:34:58 DEBUG "value": [ 03:34:58 DEBUG 03:34:58 DEBUG "value": [ 03:34:58 DEBUG 03:34:58 DEBUG 03:34:58 DEBUG 03:34:58 DEBUG 03:34:58 DEBUG 03:34:58 DEBUG "value": [ 03:34:58 DEBUG "value": [ 03:34:58 DEBUG 03:34:58 DEBUG "metric": { 03:34:58 DEBUG 03:34:58 DEBUG 03:34:58 DEBUG 03:34:58 DEBUG 03:34:58 DEBUG 03:34:58 DEBUG 03:34:58 DEBUG 03:34:58 DEBUG 03:34:58 DEBUG 03:34:58 DEBUG 03:34:58 DEBUG 03:34:58 DEBUG 03:34:58 DEBUG 03:34:58 DEBUG "pod": "ds-idrepo-0" 03:34:58 DEBUG 03:34:58 DEBUG 03:34:58 DEBUG 03:34:58 DEBUG 03:34:58 DEBUG 03:34:58 DEBUG 03:34:58 DEBUG [...] 03:34:58 DEBUG [...] 03:34:58 DEBUG 03:34:58 DEBUG 03:34:58 DEBUG 03:34:58 DEBUG 03:34:58 DEBUG [...] 03:34:58 DEBUG }, 03:34:58 DEBUG [...] 03:34:58 DEBUG 03:34:58 DEBUG 03:34:58 DEBUG [...] 03:34:58 DEBUG 03:34:58 DEBUG [...] 03:34:58 DEBUG 03:34:58 DEBUG 03:34:58 DEBUG [...] 03:34:58 DEBUG [...] 03:34:58 DEBUG 03:34:58 DEBUG 03:34:58 DEBUG 03:34:58 DEBUG "value": [ 03:34:58 DEBUG 03:34:58 DEBUG [...] 03:34:58 DEBUG [...] 03:34:58 DEBUG 03:34:58 DEBUG [...] 03:34:58 DEBUG 03:34:58 DEBUG 03:34:58 DEBUG 03:34:58 DEBUG 03:34:58 DEBUG 03:34:58 DEBUG [...] 03:34:58 DEBUG [...] 03:34:58 DEBUG 03:34:58 DEBUG 03:34:58 DEBUG 03:34:58 DEBUG 03:34:58 DEBUG 03:34:58 DEBUG 03:34:58 DEBUG 03:34:58 DEBUG 03:34:58 DEBUG 03:34:58 DEBUG 03:34:58 DEBUG 03:34:58 DEBUG 03:34:58 DEBUG 03:34:58 DEBUG 03:34:58 DEBUG 03:34:58 DEBUG 03:34:58 DEBUG 03:34:58 DEBUG 03:34:58 DEBUG 03:34:58 DEBUG 03:34:58 DEBUG 03:34:58 DEBUG 03:34:58 DEBUG "pod": "ds-cts-2" 03:34:58 DEBUG "pod": "am-767f6b67b5-vbltz" 03:34:58 DEBUG 03:34:58 DEBUG 03:34:58 DEBUG 03:34:58 DEBUG 03:34:58 DEBUG "instance": "10.142.0.43:9100" 03:34:58 DEBUG 03:34:58 DEBUG "pod": "ds-idrepo-2" 03:34:58 DEBUG 03:34:58 DEBUG "pod": "am-767f6b67b5-vbltz" 03:34:58 DEBUG "pod": "am-767f6b67b5-vbltz" 03:34:58 DEBUG 03:34:58 DEBUG "pod": "ds-cts-2" 03:34:58 DEBUG }, 03:34:58 DEBUG }, 03:34:58 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 03:34:58 DEBUG "instance": "10.142.0.43:9100" 03:34:58 DEBUG 03:34:58 DEBUG 03:34:58 DEBUG }, 03:34:58 DEBUG [...] 03:34:58 DEBUG }, 03:34:58 DEBUG 03:34:58 DEBUG }, 03:34:58 DEBUG }, 03:34:58 DEBUG "value": [ 03:34:58 DEBUG }, 03:34:58 DEBUG "value": [ 03:34:58 DEBUG "value": [ 03:34:58 DEBUG }, 03:34:58 DEBUG }, 03:34:58 DEBUG "pod": "ds-cts-2" 03:34:58 DEBUG "instance": "10.142.0.43:9100" 03:34:58 DEBUG "value": [ 03:34:58 DEBUG 03:34:58 DEBUG "value": [ 03:34:58 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 03:34:58 DEBUG "value": [ 03:34:58 DEBUG "instance": "10.142.0.43:9100" 03:34:58 DEBUG 1713666898, 03:34:58 DEBUG "value": [ 03:34:58 DEBUG 1713666898, 03:34:58 DEBUG 1713666898, 03:34:58 DEBUG "value": [ 03:34:58 DEBUG "value": [ 03:34:58 DEBUG }, 03:34:58 DEBUG }, 03:34:58 DEBUG 1713666898, 03:34:58 DEBUG 03:34:58 DEBUG 1713666898, 03:34:58 DEBUG }, 03:34:58 DEBUG 1713666898, 03:34:58 DEBUG }, 03:34:58 DEBUG "185.0165" 03:34:58 DEBUG 1713666898, 03:34:58 DEBUG "0" 03:34:58 DEBUG "3040.9666666666667" 03:34:58 DEBUG 1713666898, 03:34:58 DEBUG 1713666898, 03:34:58 DEBUG "value": [ 03:34:58 DEBUG "value": [ 03:34:58 DEBUG "65220076.06666666" 03:34:58 DEBUG 03:34:58 DEBUG "0" 03:34:58 DEBUG "value": [ 03:34:58 DEBUG "2192.1" 03:34:58 DEBUG "value": [ 03:34:58 DEBUG ] 03:34:58 DEBUG "0" 03:34:58 DEBUG ] 03:34:58 DEBUG ] 03:34:58 DEBUG "0" 03:34:58 DEBUG "65226467.666666664" 03:34:58 DEBUG 1713666898, 03:34:58 DEBUG 1713666898, 03:34:58 DEBUG ] 03:34:58 DEBUG "pod": "ds-idrepo-2" 03:34:58 DEBUG ] 03:34:58 DEBUG 1713666898, 03:34:58 DEBUG ] 03:34:58 DEBUG 1713666898, 03:34:58 DEBUG } 03:34:58 DEBUG ] 03:34:58 DEBUG } 03:34:58 DEBUG } 03:34:58 DEBUG ] 03:34:58 DEBUG ] 03:34:58 DEBUG "31.633333333333333" 03:34:58 DEBUG "0.06900000000002061" 03:34:58 DEBUG } 03:34:58 DEBUG }, 03:34:58 DEBUG } 03:34:58 DEBUG "0" 03:34:58 DEBUG } 03:34:58 DEBUG "4471643427840" 03:34:58 DEBUG ] 03:34:58 DEBUG } 03:34:58 DEBUG ] 03:34:58 DEBUG ] 03:34:58 DEBUG } 03:34:58 DEBUG } 03:34:58 DEBUG ] 03:34:58 DEBUG ] 03:34:58 DEBUG ] 03:34:58 DEBUG "value": [ 03:34:58 DEBUG ] 03:34:58 DEBUG ] 03:34:58 DEBUG } 03:34:58 DEBUG ] 03:34:58 DEBUG } 03:34:58 DEBUG ] 03:34:58 DEBUG } 03:34:58 DEBUG } 03:34:58 DEBUG ] 03:34:58 DEBUG ] 03:34:58 DEBUG } 03:34:58 DEBUG } 03:34:58 DEBUG } 03:34:58 DEBUG 1713666898, 03:34:58 DEBUG } 03:34:58 DEBUG ] 03:34:58 DEBUG ] 03:34:58 DEBUG } 03:34:58 DEBUG } 03:34:58 DEBUG } 03:34:58 DEBUG } 03:34:58 DEBUG } 03:34:58 DEBUG } 03:34:58 DEBUG } 03:34:58 DEBUG ] 03:34:58 DEBUG ] 03:34:58 DEBUG } 03:34:58 DEBUG "0" 03:34:58 DEBUG } 03:34:58 DEBUG } 03:34:58 DEBUG } 03:34:58 DEBUG ] 03:34:58 DEBUG } 03:34:58 DEBUG } 03:34:58 DEBUG } 03:34:58 DEBUG } 03:34:58 DEBUG } 03:34:58 DEBUG ] 03:34:58 DEBUG } 03:34:58 DEBUG } 03:34:58 DEBUG } 03:34:58 DEBUG } 03:34:58 DEBUG } 03:34:58 DEBUG } 03:34:58 DEBUG } 03:34:58 DEBUG ] 03:34:58 DEBUG } 03:34:58 DEBUG } 03:35:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713666954" 03:35:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666954" 03:35:54 INFO [http_cmd]: http status code OK 03:35:54 ERROR [http_cmd]: ERROR 03:35:54 DEBUG --- status code --- 03:35:54 ERROR --- status code --- 03:35:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:35:54 ERROR http status code is 400 (expected [200, 401, 403, 404]) 03:35:54 DEBUG --- http response --- 03:35:54 ERROR --- http response --- 03:35:54 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 03:35:54 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 03:35:54 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 03:35:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666955" 03:35:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666955" 03:35:55 INFO [http_cmd]: http status code OK 03:35:55 DEBUG --- status code --- 03:35:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:35:55 DEBUG --- http response --- 03:35:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:35:55 DEBUG ----- output ----- 03:35:55 DEBUG { 03:35:55 DEBUG "status": "success", 03:35:55 DEBUG "data": { 03:35:55 DEBUG "resultType": "vector", 03:35:55 DEBUG "result": [ 03:35:55 DEBUG { 03:35:55 DEBUG "metric": { 03:35:55 DEBUG "instance": "10.142.0.107:9100" 03:35:55 DEBUG }, 03:35:55 DEBUG "value": [ 03:35:55 DEBUG 03:35:55 DEBUG 03:35:55 DEBUG 03:35:55 DEBUG [...] 03:35:55 DEBUG 03:35:55 DEBUG 03:35:55 DEBUG 03:35:55 DEBUG "instance": "10.142.0.43:9100" 03:35:55 DEBUG }, 03:35:55 DEBUG "value": [ 03:35:55 DEBUG 1713666955, 03:35:55 DEBUG "0.42863333333337245" 03:35:55 DEBUG ] 03:35:55 DEBUG } 03:35:55 DEBUG ] 03:35:55 DEBUG } 03:35:55 DEBUG } 03:35:55 INFO [http_cmd]: http status code OK 03:35:55 DEBUG --- status code --- 03:35:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:35:55 DEBUG --- http response --- 03:35:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:35:55 DEBUG ----- output ----- 03:35:55 DEBUG { 03:35:55 DEBUG "status": "success", 03:35:55 DEBUG "data": { 03:35:55 DEBUG "resultType": "vector", 03:35:55 DEBUG "result": [ 03:35:55 DEBUG { 03:35:55 DEBUG "metric": { 03:35:55 DEBUG "pod": "ds-cts-0" 03:35:55 DEBUG }, 03:35:55 DEBUG "value": [ 03:35:55 DEBUG 03:35:55 DEBUG 03:35:55 DEBUG 03:35:55 DEBUG [...] 03:35:55 DEBUG 03:35:55 DEBUG 03:35:55 DEBUG 03:35:55 DEBUG "pod": "ds-cts-2" 03:35:55 DEBUG }, 03:35:55 DEBUG "value": [ 03:35:55 DEBUG 1713666955, 03:35:55 DEBUG "29.73186666666667" 03:35:55 DEBUG ] 03:35:55 DEBUG } 03:35:55 DEBUG ] 03:35:55 DEBUG } 03:35:55 DEBUG } 03:35:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666956" 03:35:56 INFO [http_cmd]: http status code OK 03:35:56 DEBUG --- status code --- 03:35:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:35:56 DEBUG --- http response --- 03:35:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:35:56 DEBUG ----- output ----- 03:35:56 DEBUG { 03:35:56 DEBUG "status": "success", 03:35:56 DEBUG "data": { 03:35:56 DEBUG "resultType": "vector", 03:35:56 DEBUG "result": [ 03:35:56 DEBUG { 03:35:56 DEBUG "metric": { 03:35:56 DEBUG "pod": "am-767f6b67b5-9zw22" 03:35:56 DEBUG }, 03:35:56 DEBUG "value": [ 03:35:56 DEBUG 03:35:56 DEBUG 03:35:56 DEBUG 03:35:56 DEBUG [...] 03:35:56 DEBUG 03:35:56 DEBUG 03:35:56 DEBUG 03:35:56 DEBUG "pod": "am-767f6b67b5-vbltz" 03:35:56 DEBUG }, 03:35:56 DEBUG "value": [ 03:35:56 DEBUG 1713666956, 03:35:56 DEBUG "2091.7666666666664" 03:35:56 DEBUG ] 03:35:56 DEBUG } 03:35:56 DEBUG ] 03:35:56 DEBUG } 03:35:56 DEBUG } 03:35:56 INFO 03:35:56 INFO [loop_until]: kubectl --namespace=xlou top pods 03:35:56 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:35:56 INFO [loop_until]: OK (rc = 0) 03:35:56 DEBUG --- stdout --- 03:35:56 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 11267m 8803Mi am-767f6b67b5-qcqz2 10763m 9142Mi am-767f6b67b5-vbltz 10587m 8889Mi ds-cts-0 18773m 12516Mi ds-cts-1 17874m 12491Mi ds-cts-2 19443m 12560Mi ds-idrepo-0 4278m 3917Mi ds-idrepo-1 4250m 3963Mi ds-idrepo-2 4357m 3915Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1179Mi idm-b85f9d978-wnhxb 3m 1239Mi lodemon-54c87f8cbb-jzrjs 17m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-68c5d5b884-p97w5 15048m 10522Mi 03:35:56 DEBUG --- stderr --- 03:35:56 DEBUG 03:35:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666957" 03:35:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666957" 03:35:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666957" 03:35:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666957" 03:35:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666957" 03:35:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666957" 03:35:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713666957" 03:35:57 INFO [http_cmd]: http status code OK 03:35:57 DEBUG --- status code --- 03:35:57 INFO [http_cmd]: http status code OK 03:35:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:35:57 INFO [http_cmd]: http status code OK 03:35:57 INFO [http_cmd]: http status code OK 03:35:57 DEBUG --- status code --- 03:35:57 DEBUG --- http response --- 03:35:57 INFO [http_cmd]: http status code OK 03:35:57 DEBUG --- status code --- 03:35:57 DEBUG --- status code --- 03:35:57 INFO [http_cmd]: http status code OK 03:35:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:35:57 DEBUG --- status code --- 03:35:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:35:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:35:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:35:57 DEBUG --- http response --- 03:35:57 DEBUG --- status code --- 03:35:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:35:57 INFO [http_cmd]: http status code OK 03:35:57 DEBUG --- status code --- 03:35:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:35:57 DEBUG --- http response --- 03:35:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:35:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:35:57 DEBUG --- http response --- 03:35:57 DEBUG --- http response --- 03:35:57 DEBUG --- http response --- 03:35:57 DEBUG ----- output ----- 03:35:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 03:35:57 DEBUG --- http response --- 03:35:57 DEBUG ----- output ----- 03:35:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:35:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:35:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:35:57 DEBUG { 03:35:57 DEBUG ----- output ----- 03:35:57 DEBUG { 03:35:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 03:35:57 DEBUG ----- output ----- 03:35:57 DEBUG ----- output ----- 03:35:57 DEBUG ----- output ----- 03:35:57 DEBUG { 03:35:57 DEBUG "status": "success", 03:35:57 DEBUG "status": "success", 03:35:57 DEBUG ----- output ----- 03:35:57 DEBUG { 03:35:57 DEBUG { 03:35:57 DEBUG "status": "success", 03:35:57 DEBUG { 03:35:57 DEBUG "data": { 03:35:57 DEBUG "data": { 03:35:57 DEBUG { 03:35:57 DEBUG "status": "success", 03:35:57 DEBUG "data": { 03:35:57 DEBUG "status": "success", 03:35:57 DEBUG "status": "success", 03:35:57 DEBUG "resultType": "vector", 03:35:57 DEBUG "resultType": "vector", 03:35:57 DEBUG "resultType": "vector", 03:35:57 DEBUG "data": { 03:35:57 DEBUG "data": { 03:35:57 DEBUG "data": { 03:35:57 DEBUG "status": "success", 03:35:57 DEBUG "result": [ 03:35:57 DEBUG "result": [ 03:35:57 DEBUG "result": [ 03:35:57 DEBUG "resultType": "vector", 03:35:57 DEBUG "resultType": "vector", 03:35:57 DEBUG "resultType": "vector", 03:35:57 DEBUG "data": { 03:35:57 DEBUG { 03:35:57 DEBUG "metric": { 03:35:57 DEBUG "pod": "am-767f6b67b5-9zw22" 03:35:57 DEBUG }, 03:35:57 DEBUG "value": [ 03:35:57 DEBUG "result": [ 03:35:57 DEBUG { 03:35:57 DEBUG { 03:35:57 DEBUG 03:35:57 DEBUG "result": [ 03:35:57 DEBUG { 03:35:57 DEBUG "metric": { 03:35:57 DEBUG "metric": { 03:35:57 DEBUG "resultType": "vector", 03:35:57 DEBUG "metric": { 03:35:57 DEBUG 03:35:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:35:57 DEBUG "result": [ 03:35:57 DEBUG { 03:35:57 DEBUG "pod": "ds-idrepo-1" 03:35:57 DEBUG }, 03:35:57 DEBUG "pod": "ds-idrepo-1" 03:35:57 DEBUG 03:35:57 DEBUG }, 03:35:57 DEBUG "value": [ 03:35:57 DEBUG "metric": { 03:35:57 DEBUG "value": [ 03:35:57 DEBUG "result": [ 03:35:57 DEBUG }, 03:35:57 DEBUG [...] 03:35:57 DEBUG 03:35:57 DEBUG { 03:35:57 DEBUG "pod": "ds-idrepo-0" 03:35:57 DEBUG 03:35:57 DEBUG 03:35:57 DEBUG 03:35:57 DEBUG "value": [ 03:35:57 DEBUG 03:35:57 DEBUG "metric": { 03:35:57 DEBUG }, 03:35:57 DEBUG { 03:35:57 DEBUG [...] 03:35:57 DEBUG 03:35:57 DEBUG 03:35:57 DEBUG 03:35:57 DEBUG "pod": "am-767f6b67b5-9zw22" 03:35:57 DEBUG }, 03:35:57 DEBUG "metric": { 03:35:57 DEBUG "value": [ 03:35:57 DEBUG 03:35:57 DEBUG 03:35:57 DEBUG "pod": "am-767f6b67b5-vbltz" 03:35:57 DEBUG "value": [ 03:35:57 DEBUG 03:35:57 DEBUG 03:35:57 DEBUG 03:35:57 DEBUG [...] 03:35:57 DEBUG 03:35:57 DEBUG }, 03:35:57 DEBUG "pod": "am-767f6b67b5-9zw22" 03:35:57 DEBUG 03:35:57 DEBUG 03:35:57 DEBUG 03:35:57 DEBUG [...] 03:35:57 DEBUG 03:35:57 DEBUG "value": [ 03:35:57 DEBUG }, 03:35:57 DEBUG 03:35:57 DEBUG 03:35:57 DEBUG 03:35:57 DEBUG 03:35:57 DEBUG 1713666957, 03:35:57 DEBUG "value": [ 03:35:57 DEBUG [...] 03:35:57 DEBUG 03:35:57 DEBUG [...] 03:35:57 DEBUG 03:35:57 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:35:57 DEBUG "2092.1666666666665" 03:35:57 DEBUG 03:35:57 DEBUG 03:35:57 DEBUG 03:35:57 DEBUG 03:35:57 DEBUG 03:35:57 DEBUG }, 03:35:57 DEBUG ] 03:35:57 DEBUG 03:35:57 DEBUG 03:35:57 DEBUG 03:35:57 DEBUG 03:35:57 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:35:57 DEBUG "value": [ 03:35:57 DEBUG } 03:35:57 DEBUG 03:35:57 DEBUG 03:35:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:35:57 DEBUG 03:35:57 DEBUG }, 03:35:57 DEBUG 1713666957, 03:35:57 DEBUG ] 03:35:57 DEBUG [...] 03:35:57 DEBUG "pod": "am-767f6b67b5-vbltz" 03:35:57 DEBUG }, 03:35:57 DEBUG "pod": "ds-idrepo-2" 03:35:57 DEBUG "value": [ 03:35:57 DEBUG "45223544.5233814" 03:35:57 DEBUG } 03:35:57 DEBUG 03:35:57 DEBUG }, 03:35:57 DEBUG "value": [ 03:35:57 DEBUG }, 03:35:57 DEBUG 1713666957, 03:35:57 DEBUG "10.399111946934779" 03:35:57 DEBUG ] 03:35:57 DEBUG 03:35:57 DEBUG "value": [ 03:35:57 DEBUG 1713666957, 03:35:57 DEBUG "14.755697099114128" 03:35:57 DEBUG } 03:35:57 DEBUG ] 03:35:57 DEBUG } 03:35:57 DEBUG 03:35:57 DEBUG "pod": "am-767f6b67b5-vbltz" 03:35:57 DEBUG ] 03:35:57 DEBUG "value": [ 03:35:57 DEBUG 1713666957, 03:35:57 DEBUG ] 03:35:57 DEBUG } 03:35:57 DEBUG 1713666957, 03:35:57 DEBUG "432015.5" 03:35:57 DEBUG } 03:35:57 DEBUG "0" 03:35:57 DEBUG ] 03:35:57 DEBUG } 03:35:57 DEBUG }, 03:35:57 DEBUG ] 03:35:57 DEBUG ] 03:35:57 DEBUG ] 03:35:57 DEBUG } 03:35:57 DEBUG } 03:35:57 DEBUG "value": [ 03:35:57 DEBUG 1713666957, 03:35:57 DEBUG "0" 03:35:57 DEBUG } 03:35:57 DEBUG } 03:35:57 DEBUG } 03:35:57 DEBUG ] 03:35:57 DEBUG ] 03:35:57 DEBUG } 03:35:57 DEBUG ] 03:35:57 DEBUG } 03:35:57 DEBUG } 03:35:57 DEBUG } 03:35:57 DEBUG } 03:35:57 DEBUG ] 03:35:57 DEBUG } 03:35:57 DEBUG } 03:35:57 DEBUG } 03:35:57 DEBUG } 03:35:57 INFO 03:35:57 INFO [loop_until]: kubectl --namespace=xlou top node 03:35:57 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:35:58 INFO [loop_until]: OK (rc = 0) 03:35:58 DEBUG --- stdout --- 03:35:58 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 65m 0% 6292Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5532m 18% 7759Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 12438m 41% 11613Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 79m 0% 3346Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 18942m 63% 15838Mi 14% gke-xlou-cdm-default-pool-1b0769c8-wpwl 158m 0% 5239Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 14029m 46% 12076Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 12785m 42% 3999Mi 3% gke-xlou-cdm-ds-0f601751-hmm7 14170m 47% 11147Mi 9% gke-xlou-cdm-ds-0f601751-j488 6263m 20% 7353Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 18225m 61% 17612Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 5656m 18% 8031Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 19506m 65% 15254Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 15318m 96% 21958Mi 37% 03:35:58 DEBUG --- stderr --- 03:35:58 DEBUG 03:35:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713666958" 03:35:58 INFO [http_cmd]: http status code OK 03:35:58 DEBUG --- status code --- 03:35:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:35:58 DEBUG --- http response --- 03:35:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:35:58 DEBUG ----- output ----- 03:35:58 DEBUG { 03:35:58 DEBUG "status": "success", 03:35:58 DEBUG "data": { 03:35:58 DEBUG "resultType": "vector", 03:35:58 DEBUG "result": [ 03:35:58 DEBUG { 03:35:58 DEBUG "metric": { 03:35:58 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:35:58 DEBUG }, 03:35:58 DEBUG "value": [ 03:35:58 DEBUG 03:35:58 DEBUG 03:35:58 DEBUG 03:35:58 DEBUG [...] 03:35:58 DEBUG 03:35:58 DEBUG 03:35:58 DEBUG 03:35:58 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:35:58 DEBUG }, 03:35:58 DEBUG "value": [ 03:35:58 DEBUG 1713666958, 03:35:58 DEBUG "11132801024" 03:35:58 DEBUG ] 03:35:58 DEBUG } 03:35:58 DEBUG ] 03:35:58 DEBUG } 03:35:58 DEBUG } 03:35:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713666958" 03:35:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666958" 03:35:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666958" 03:35:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666958" 03:35:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666958" 03:35:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666958" 03:35:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713666958" 03:35:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666958" 03:35:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666958" 03:35:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666958" 03:35:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666958" 03:35:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713666958" 03:35:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666958" 03:35:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713666958" 03:35:58 INFO [http_cmd]: http status code OK 03:35:58 INFO [http_cmd]: http status code OK 03:35:58 INFO [http_cmd]: http status code OK 03:35:58 INFO [http_cmd]: http status code OK 03:35:58 INFO [http_cmd]: http status code OK 03:35:58 INFO [http_cmd]: http status code OK 03:35:58 INFO [http_cmd]: http status code OK 03:35:58 INFO [http_cmd]: http status code OK 03:35:58 INFO [http_cmd]: http status code OK 03:35:58 INFO [http_cmd]: http status code OK 03:35:58 DEBUG --- status code --- 03:35:58 DEBUG --- status code --- 03:35:58 DEBUG --- status code --- 03:35:58 DEBUG --- status code --- 03:35:58 DEBUG --- status code --- 03:35:58 DEBUG --- status code --- 03:35:58 DEBUG --- status code --- 03:35:58 INFO [http_cmd]: http status code OK 03:35:58 DEBUG --- status code --- 03:35:58 DEBUG --- status code --- 03:35:58 DEBUG --- status code --- 03:35:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:35:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:35:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:35:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:35:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:35:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:35:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:35:58 INFO [http_cmd]: http status code OK 03:35:58 DEBUG --- status code --- 03:35:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:35:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:35:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:35:58 DEBUG --- http response --- 03:35:58 DEBUG --- http response --- 03:35:58 DEBUG --- http response --- 03:35:58 DEBUG --- http response --- 03:35:58 INFO [http_cmd]: http status code OK 03:35:58 DEBUG --- http response --- 03:35:58 DEBUG --- http response --- 03:35:58 DEBUG --- http response --- 03:35:58 DEBUG --- status code --- 03:35:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:35:58 DEBUG --- http response --- 03:35:58 INFO [http_cmd]: http status code OK 03:35:58 DEBUG --- status code --- 03:35:58 DEBUG --- http response --- 03:35:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:35:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:35:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:35:58 DEBUG --- status code --- 03:35:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:35:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:35:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:35:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:35:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:35:58 DEBUG --- http response --- 03:35:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:35:58 DEBUG --- http response --- 03:35:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:35:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:35:58 DEBUG ----- output ----- 03:35:58 DEBUG ----- output ----- 03:35:58 DEBUG ----- output ----- 03:35:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:35:58 DEBUG ----- output ----- 03:35:58 DEBUG ----- output ----- 03:35:58 DEBUG ----- output ----- 03:35:58 DEBUG --- http response --- 03:35:58 DEBUG ----- output ----- 03:35:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:35:58 DEBUG --- http response --- 03:35:58 DEBUG ----- output ----- 03:35:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:35:58 DEBUG ----- output ----- 03:35:58 DEBUG { 03:35:58 DEBUG { 03:35:58 DEBUG { 03:35:58 DEBUG --- http response --- 03:35:58 DEBUG { 03:35:58 DEBUG { 03:35:58 DEBUG { 03:35:58 DEBUG { 03:35:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:35:58 DEBUG ----- output ----- 03:35:58 DEBUG { 03:35:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:35:58 DEBUG ----- output ----- 03:35:58 DEBUG { 03:35:58 DEBUG "status": "success", 03:35:58 DEBUG "status": "success", 03:35:58 DEBUG "status": "success", 03:35:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:35:58 DEBUG "status": "success", 03:35:58 DEBUG "status": "success", 03:35:58 DEBUG "data": { 03:35:58 DEBUG "status": "success", 03:35:58 DEBUG ----- output ----- 03:35:58 DEBUG { 03:35:58 DEBUG "status": "success", 03:35:58 DEBUG ----- output ----- 03:35:58 DEBUG { 03:35:58 DEBUG "status": "success", 03:35:58 DEBUG "data": { 03:35:58 DEBUG "data": { 03:35:58 DEBUG "data": { 03:35:58 DEBUG ----- output ----- 03:35:58 DEBUG { 03:35:58 DEBUG "resultType": "vector", 03:35:58 DEBUG "status": "success", 03:35:58 DEBUG "data": { 03:35:58 DEBUG "resultType": "vector", 03:35:58 DEBUG { 03:35:58 DEBUG "data": { 03:35:58 DEBUG { 03:35:58 DEBUG "status": "success", 03:35:58 DEBUG "data": { 03:35:58 DEBUG "resultType": "vector", 03:35:58 DEBUG "resultType": "vector", 03:35:58 DEBUG "result": [ 03:35:58 DEBUG "resultType": "vector", 03:35:58 DEBUG "status": "success", 03:35:58 DEBUG "result": [ 03:35:58 DEBUG "data": { 03:35:58 DEBUG "result": [ 03:35:58 DEBUG { 03:35:58 DEBUG "status": "success", 03:35:58 DEBUG "data": { 03:35:58 DEBUG "status": "success", 03:35:58 DEBUG "data": { 03:35:58 DEBUG "resultType": "vector", 03:35:58 DEBUG "result": [ 03:35:58 DEBUG "data": { 03:35:58 DEBUG { 03:35:58 DEBUG "result": [ 03:35:58 DEBUG "data": { 03:35:58 DEBUG { 03:35:58 DEBUG "resultType": "vector", 03:35:58 DEBUG "result": [ 03:35:58 DEBUG "resultType": "vector", 03:35:58 DEBUG "metric": { 03:35:58 DEBUG "data": { 03:35:58 DEBUG "status": "success", 03:35:58 DEBUG "resultType": "vector", 03:35:58 DEBUG "result": [ 03:35:58 DEBUG { 03:35:58 DEBUG "resultType": "vector", 03:35:58 DEBUG "metric": { 03:35:58 DEBUG { 03:35:58 DEBUG "resultType": "vector", 03:35:58 DEBUG "metric": { 03:35:58 DEBUG { 03:35:58 DEBUG "result": [ 03:35:58 DEBUG "resultType": "vector", 03:35:58 DEBUG "pod": "ds-idrepo-0" 03:35:58 DEBUG "resultType": "vector", 03:35:58 DEBUG "data": { 03:35:58 DEBUG "resultType": "vector", 03:35:58 DEBUG "result": [ 03:35:58 DEBUG { 03:35:58 DEBUG "metric": { 03:35:58 DEBUG "instance": "10.142.0.107:9100" 03:35:58 DEBUG "metric": { 03:35:58 DEBUG "result": [ 03:35:58 DEBUG "pod": "ds-idrepo-1" 03:35:58 DEBUG "metric": { 03:35:58 DEBUG { 03:35:58 DEBUG "result": [ 03:35:58 DEBUG }, 03:35:58 DEBUG "result": [ 03:35:58 DEBUG "instance": "10.142.0.107:9100" 03:35:58 DEBUG { 03:35:58 DEBUG "result": [ 03:35:58 DEBUG "metric": { 03:35:58 DEBUG }, 03:35:58 DEBUG "pod": "am-767f6b67b5-9zw22" 03:35:58 DEBUG "result": [ 03:35:58 DEBUG { 03:35:58 DEBUG }, 03:35:58 DEBUG "pod": "ds-idrepo-0" 03:35:58 DEBUG "metric": { 03:35:58 DEBUG { 03:35:58 DEBUG "value": [ 03:35:58 DEBUG { 03:35:58 DEBUG }, 03:35:58 DEBUG "metric": { 03:35:58 DEBUG { 03:35:58 DEBUG "pod": "ds-cts-0" 03:35:58 DEBUG "value": [ 03:35:58 DEBUG }, 03:35:58 DEBUG { 03:35:58 DEBUG "metric": { 03:35:58 DEBUG "value": [ 03:35:58 DEBUG }, 03:35:58 DEBUG "instance": "10.142.0.107:9100" 03:35:58 DEBUG "metric": { 03:35:58 DEBUG 03:35:58 DEBUG "metric": { 03:35:58 DEBUG "value": [ 03:35:58 DEBUG "pod": "am-767f6b67b5-9zw22" 03:35:58 DEBUG "metric": { 03:35:58 DEBUG }, 03:35:58 DEBUG 03:35:58 DEBUG "value": [ 03:35:58 DEBUG "metric": { 03:35:58 DEBUG "pod": "am-767f6b67b5-9zw22" 03:35:58 DEBUG 03:35:58 DEBUG "value": [ 03:35:58 DEBUG }, 03:35:58 DEBUG "value": [ 03:35:58 DEBUG 03:35:58 DEBUG "pod": "ds-idrepo-0" 03:35:58 DEBUG 03:35:58 DEBUG }, 03:35:58 DEBUG "pod": "ds-cts-0" 03:35:58 DEBUG "value": [ 03:35:58 DEBUG 03:35:58 DEBUG 03:35:58 DEBUG 03:35:58 DEBUG "pod": "ds-idrepo-1" 03:35:58 DEBUG 03:35:58 DEBUG 03:35:58 DEBUG 03:35:58 DEBUG "instance": "10.142.0.107:9100" 03:35:58 DEBUG 03:35:58 DEBUG }, 03:35:58 DEBUG 03:35:58 DEBUG "value": [ 03:35:58 DEBUG }, 03:35:58 DEBUG 03:35:58 DEBUG 03:35:58 DEBUG }, 03:35:58 DEBUG 03:35:58 DEBUG }, 03:35:58 DEBUG 03:35:58 DEBUG [...] 03:35:58 DEBUG 03:35:58 DEBUG }, 03:35:58 DEBUG [...] 03:35:58 DEBUG "value": [ 03:35:58 DEBUG 03:35:58 DEBUG 03:35:58 DEBUG "value": [ 03:35:58 DEBUG 03:35:58 DEBUG [...] 03:35:58 DEBUG "value": [ 03:35:58 DEBUG [...] 03:35:58 DEBUG "value": [ 03:35:58 DEBUG 03:35:58 DEBUG 03:35:58 DEBUG 03:35:58 DEBUG [...] 03:35:58 DEBUG 03:35:58 DEBUG 03:35:58 DEBUG [...] 03:35:58 DEBUG 03:35:58 DEBUG 03:35:58 DEBUG 03:35:58 DEBUG 03:35:58 DEBUG 03:35:58 DEBUG 03:35:58 DEBUG 03:35:58 DEBUG 03:35:58 DEBUG 03:35:58 DEBUG "value": [ 03:35:58 DEBUG 03:35:58 DEBUG 03:35:58 DEBUG [...] 03:35:58 DEBUG 03:35:58 DEBUG 03:35:58 DEBUG 03:35:58 DEBUG [...] 03:35:58 DEBUG 03:35:58 DEBUG 03:35:58 DEBUG 03:35:58 DEBUG 03:35:58 DEBUG 03:35:58 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:35:58 DEBUG }, 03:35:58 DEBUG 03:35:58 DEBUG 03:35:58 DEBUG 03:35:58 DEBUG 03:35:58 DEBUG [...] 03:35:58 DEBUG 03:35:58 DEBUG 03:35:58 DEBUG 03:35:58 DEBUG "instance": "10.142.0.43:9100" 03:35:58 DEBUG "pod": "am-767f6b67b5-vbltz" 03:35:58 DEBUG 03:35:58 DEBUG "value": [ 03:35:58 DEBUG 1713666958, 03:35:58 DEBUG 03:35:58 DEBUG 03:35:58 DEBUG "pod": "ds-idrepo-2" 03:35:58 DEBUG 03:35:58 DEBUG 03:35:58 DEBUG 03:35:58 DEBUG [...] 03:35:58 DEBUG 03:35:58 DEBUG }, 03:35:58 DEBUG 03:35:58 DEBUG }, 03:35:58 DEBUG 03:35:58 DEBUG "0" 03:35:58 DEBUG [...] 03:35:58 DEBUG 03:35:58 DEBUG 03:35:58 DEBUG }, 03:35:58 DEBUG "instance": "10.142.0.43:9100" 03:35:58 DEBUG 03:35:58 DEBUG 03:35:58 DEBUG 03:35:58 DEBUG 03:35:58 DEBUG "value": [ 03:35:58 DEBUG [...] 03:35:58 DEBUG "value": [ 03:35:58 DEBUG [...] 03:35:58 DEBUG ] 03:35:58 DEBUG 03:35:58 DEBUG 03:35:58 DEBUG 03:35:58 DEBUG "value": [ 03:35:58 DEBUG }, 03:35:58 DEBUG "instance": "10.142.0.43:9100" 03:35:58 DEBUG }, 03:35:58 DEBUG 03:35:58 DEBUG "pod": "ds-cts-2" 03:35:58 DEBUG 1713666958, 03:35:58 DEBUG 03:35:58 DEBUG 1713666958, 03:35:58 DEBUG 03:35:58 DEBUG } 03:35:58 DEBUG 03:35:58 DEBUG [...] 03:35:58 DEBUG "pod": "ds-cts-2" 03:35:58 DEBUG 1713666958, 03:35:58 DEBUG "value": [ 03:35:58 DEBUG "value": [ 03:35:58 DEBUG 03:35:58 DEBUG 03:35:58 DEBUG }, 03:35:58 DEBUG "52930833.1" 03:35:58 DEBUG 03:35:58 DEBUG 03:35:58 DEBUG 03:35:58 DEBUG ] 03:35:58 DEBUG 03:35:58 DEBUG 03:35:58 DEBUG }, 03:35:58 DEBUG "0" 03:35:58 DEBUG 1713666958, 03:35:58 DEBUG 1713666958, 03:35:58 DEBUG "pod": "ds-cts-2" 03:35:58 DEBUG "pod": "am-767f6b67b5-vbltz" 03:35:58 DEBUG "value": [ 03:35:58 DEBUG ] 03:35:58 DEBUG "pod": "am-767f6b67b5-vbltz" 03:35:58 DEBUG 03:35:58 DEBUG "187.1825" 03:35:58 DEBUG } 03:35:58 DEBUG "pod": "ds-idrepo-2" 03:35:58 DEBUG 03:35:58 DEBUG "value": [ 03:35:58 DEBUG ] 03:35:58 DEBUG "4475756276736" 03:35:58 DEBUG "0.06766666666669419" 03:35:58 DEBUG }, 03:35:58 DEBUG "value": [ 03:35:58 DEBUG 1713666958, 03:35:58 DEBUG } 03:35:58 DEBUG }, 03:35:58 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:35:58 DEBUG ] 03:35:58 DEBUG } 03:35:58 DEBUG }, 03:35:58 DEBUG 03:35:58 DEBUG 1713666958, 03:35:58 DEBUG "0" 03:35:58 DEBUG ] 03:35:58 DEBUG ] 03:35:58 DEBUG 1713666958, 03:35:58 DEBUG }, 03:35:58 DEBUG "value": [ 03:35:58 DEBUG ] 03:35:58 DEBUG "value": [ 03:35:58 DEBUG }, 03:35:58 DEBUG } 03:35:58 DEBUG "value": [ 03:35:58 DEBUG "instance": "10.142.0.43:9100" 03:35:58 DEBUG ] 03:35:58 DEBUG } 03:35:58 DEBUG } 03:35:58 DEBUG } 03:35:58 DEBUG "26.099999999999998" 03:35:58 DEBUG ] 03:35:58 DEBUG 1713666958, 03:35:58 DEBUG } 03:35:58 DEBUG 1713666958, 03:35:58 DEBUG "value": [ 03:35:58 DEBUG ] 03:35:58 DEBUG 1713666958, 03:35:58 DEBUG }, 03:35:58 DEBUG } 03:35:58 DEBUG ] 03:35:58 DEBUG ] 03:35:58 DEBUG ] 03:35:58 DEBUG "0" 03:35:58 DEBUG } 03:35:58 DEBUG "2092.3" 03:35:58 DEBUG } 03:35:58 DEBUG "1927.2666666666667" 03:35:58 DEBUG 1713666958, 03:35:58 DEBUG } 03:35:58 DEBUG "0" 03:35:58 DEBUG "value": [ 03:35:58 DEBUG ] 03:35:58 DEBUG } 03:35:58 DEBUG } 03:35:58 DEBUG } 03:35:58 DEBUG ] 03:35:58 DEBUG ] 03:35:58 DEBUG ] 03:35:58 DEBUG ] 03:35:58 DEBUG "0" 03:35:58 DEBUG } 03:35:58 DEBUG ] 03:35:58 DEBUG 1713666958, 03:35:58 DEBUG } 03:35:58 DEBUG } 03:35:58 DEBUG } 03:35:58 DEBUG } 03:35:58 DEBUG } 03:35:58 DEBUG } 03:35:58 DEBUG } 03:35:58 DEBUG } 03:35:58 DEBUG ] 03:35:58 DEBUG } 03:35:58 DEBUG "52931701.93333334" 03:35:58 DEBUG } 03:35:58 DEBUG ] 03:35:58 DEBUG } 03:35:58 DEBUG ] 03:35:58 DEBUG ] 03:35:58 DEBUG } 03:35:58 DEBUG ] 03:35:58 DEBUG ] 03:35:58 DEBUG } 03:35:58 DEBUG } 03:35:58 DEBUG } 03:35:58 DEBUG ] 03:35:58 DEBUG } 03:35:58 DEBUG } 03:35:58 DEBUG } 03:35:58 DEBUG } 03:35:58 DEBUG } 03:35:58 DEBUG } 03:35:58 DEBUG } 03:35:58 DEBUG ] 03:35:58 DEBUG } 03:35:58 DEBUG } 03:35:58 DEBUG } 03:36:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713667014" 03:36:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667014" 03:36:54 ERROR [http_cmd]: ERROR 03:36:54 INFO [http_cmd]: http status code OK 03:36:54 ERROR --- status code --- 03:36:54 DEBUG --- status code --- 03:36:54 ERROR http status code is 400 (expected [200, 401, 403, 404]) 03:36:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:36:54 ERROR --- http response --- 03:36:54 DEBUG --- http response --- 03:36:54 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 03:36:54 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 03:36:54 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 03:36:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713667016" 03:36:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667016" 03:36:56 INFO [http_cmd]: http status code OK 03:36:56 INFO [http_cmd]: http status code OK 03:36:56 DEBUG --- status code --- 03:36:56 DEBUG --- status code --- 03:36:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:36:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:36:56 DEBUG --- http response --- 03:36:56 DEBUG --- http response --- 03:36:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:36:56 DEBUG ----- output ----- 03:36:56 DEBUG { 03:36:56 DEBUG "status": "success", 03:36:56 DEBUG "data": { 03:36:56 DEBUG "resultType": "vector", 03:36:56 DEBUG "result": [ 03:36:56 DEBUG { 03:36:56 DEBUG "metric": { 03:36:56 DEBUG "pod": "ds-cts-0" 03:36:56 DEBUG }, 03:36:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:36:56 DEBUG "value": [ 03:36:56 DEBUG ----- output ----- 03:36:56 DEBUG 03:36:56 DEBUG { 03:36:56 DEBUG 03:36:56 DEBUG "status": "success", 03:36:56 DEBUG 03:36:56 DEBUG "data": { 03:36:56 DEBUG [...] 03:36:56 DEBUG "resultType": "vector", 03:36:56 DEBUG 03:36:56 DEBUG "result": [ 03:36:56 DEBUG 03:36:56 DEBUG { 03:36:56 DEBUG 03:36:56 DEBUG "metric": { 03:36:56 DEBUG "pod": "ds-cts-2" 03:36:56 DEBUG "instance": "10.142.0.107:9100" 03:36:56 DEBUG }, 03:36:56 DEBUG }, 03:36:56 DEBUG "value": [ 03:36:56 DEBUG "value": [ 03:36:56 DEBUG 1713667016, 03:36:56 DEBUG 03:36:56 DEBUG "29.954033333333335" 03:36:56 DEBUG 03:36:56 DEBUG ] 03:36:56 DEBUG 03:36:56 DEBUG } 03:36:56 DEBUG [...] 03:36:56 DEBUG ] 03:36:56 DEBUG 03:36:56 DEBUG } 03:36:56 DEBUG 03:36:56 DEBUG } 03:36:56 DEBUG 03:36:56 DEBUG "instance": "10.142.0.43:9100" 03:36:56 DEBUG }, 03:36:56 DEBUG "value": [ 03:36:56 DEBUG 1713667016, 03:36:56 DEBUG "0.41216666666672" 03:36:56 DEBUG ] 03:36:56 DEBUG } 03:36:56 DEBUG ] 03:36:56 DEBUG } 03:36:56 DEBUG } 03:36:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667016" 03:36:56 INFO [http_cmd]: http status code OK 03:36:56 DEBUG --- status code --- 03:36:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:36:56 DEBUG --- http response --- 03:36:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:36:56 DEBUG ----- output ----- 03:36:56 DEBUG { 03:36:56 DEBUG "status": "success", 03:36:56 DEBUG "data": { 03:36:56 DEBUG "resultType": "vector", 03:36:56 DEBUG "result": [ 03:36:56 DEBUG { 03:36:56 DEBUG "metric": { 03:36:56 DEBUG "pod": "am-767f6b67b5-9zw22" 03:36:56 DEBUG }, 03:36:56 DEBUG "value": [ 03:36:56 DEBUG 03:36:56 DEBUG 03:36:56 DEBUG 03:36:56 DEBUG [...] 03:36:56 DEBUG 03:36:56 DEBUG 03:36:56 DEBUG 03:36:56 DEBUG "pod": "am-767f6b67b5-vbltz" 03:36:56 DEBUG }, 03:36:56 DEBUG "value": [ 03:36:56 DEBUG 1713667016, 03:36:56 DEBUG "395.76666666666665" 03:36:56 DEBUG ] 03:36:56 DEBUG } 03:36:56 DEBUG ] 03:36:56 DEBUG } 03:36:56 DEBUG } 03:36:56 INFO 03:36:56 INFO [loop_until]: kubectl --namespace=xlou top pods 03:36:56 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:36:57 INFO [loop_until]: OK (rc = 0) 03:36:57 DEBUG --- stdout --- 03:36:57 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 15173m 8796Mi am-767f6b67b5-qcqz2 15885m 9143Mi am-767f6b67b5-vbltz 16673m 8893Mi ds-cts-0 19892m 12516Mi ds-cts-1 19093m 12482Mi ds-cts-2 19873m 12559Mi ds-idrepo-0 6382m 3918Mi ds-idrepo-1 6272m 3962Mi ds-idrepo-2 6518m 3915Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1179Mi idm-b85f9d978-wnhxb 3m 1239Mi lodemon-54c87f8cbb-jzrjs 27m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-68c5d5b884-p97w5 15160m 13846Mi 03:36:57 DEBUG --- stderr --- 03:36:57 DEBUG 03:36:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667017" 03:36:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667017" 03:36:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667017" 03:36:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713667017" 03:36:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667017" 03:36:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667017" 03:36:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667017" 03:36:57 INFO [http_cmd]: http status code OK 03:36:57 INFO [http_cmd]: http status code OK 03:36:57 DEBUG --- status code --- 03:36:57 DEBUG --- status code --- 03:36:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:36:57 DEBUG --- http response --- 03:36:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:36:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 03:36:57 DEBUG --- http response --- 03:36:57 DEBUG ----- output ----- 03:36:57 DEBUG { 03:36:57 DEBUG "status": "success", 03:36:57 DEBUG "data": { 03:36:57 DEBUG "resultType": "vector", 03:36:57 DEBUG "result": [ 03:36:57 DEBUG { 03:36:57 DEBUG "metric": { 03:36:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:36:57 DEBUG }, 03:36:57 DEBUG "value": [ 03:36:57 DEBUG 03:36:57 DEBUG 03:36:57 DEBUG 03:36:57 DEBUG [...] 03:36:57 DEBUG 03:36:57 DEBUG 03:36:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:36:57 DEBUG 03:36:57 INFO [http_cmd]: http status code OK 03:36:57 DEBUG ----- output ----- 03:36:57 DEBUG "pod": "am-767f6b67b5-qcqz2" 03:36:57 DEBUG --- status code --- 03:36:57 DEBUG { 03:36:57 DEBUG }, 03:36:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:36:57 DEBUG "status": "success", 03:36:57 INFO [http_cmd]: http status code OK 03:36:57 INFO [http_cmd]: http status code OK 03:36:57 INFO [http_cmd]: http status code OK 03:36:57 INFO [http_cmd]: http status code OK 03:36:57 DEBUG "value": [ 03:36:57 DEBUG --- http response --- 03:36:57 DEBUG "data": { 03:36:57 DEBUG --- status code --- 03:36:57 DEBUG --- status code --- 03:36:57 DEBUG --- status code --- 03:36:57 DEBUG --- status code --- 03:36:57 DEBUG 1713667017, 03:36:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:36:57 DEBUG "resultType": "vector", 03:36:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:36:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:36:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:36:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:36:57 DEBUG "173038.5653831027" 03:36:57 DEBUG ----- output ----- 03:36:57 DEBUG "result": [ 03:36:57 DEBUG --- http response --- 03:36:57 DEBUG --- http response --- 03:36:57 DEBUG --- http response --- 03:36:57 DEBUG --- http response --- 03:36:57 DEBUG ] 03:36:57 DEBUG { 03:36:57 DEBUG { 03:36:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:36:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 03:36:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:36:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:36:57 DEBUG } 03:36:57 DEBUG "status": "success", 03:36:57 DEBUG "metric": { 03:36:57 DEBUG ----- output ----- 03:36:57 DEBUG ----- output ----- 03:36:57 DEBUG ----- output ----- 03:36:57 DEBUG ----- output ----- 03:36:57 DEBUG ] 03:36:57 DEBUG "data": { 03:36:57 DEBUG "pod": "am-767f6b67b5-9zw22" 03:36:57 DEBUG { 03:36:57 DEBUG { 03:36:57 DEBUG { 03:36:57 DEBUG { 03:36:57 DEBUG } 03:36:57 DEBUG "resultType": "vector", 03:36:57 DEBUG }, 03:36:57 DEBUG "status": "success", 03:36:57 DEBUG "status": "success", 03:36:57 DEBUG "status": "success", 03:36:57 DEBUG "status": "success", 03:36:57 DEBUG } 03:36:57 DEBUG "result": [ 03:36:57 DEBUG { 03:36:57 DEBUG "data": { 03:36:57 DEBUG "data": { 03:36:57 DEBUG "data": { 03:36:57 DEBUG "data": { 03:36:57 DEBUG "value": [ 03:36:57 DEBUG "metric": { 03:36:57 DEBUG "resultType": "vector", 03:36:57 DEBUG "resultType": "vector", 03:36:57 DEBUG "resultType": "vector", 03:36:57 DEBUG "resultType": "vector", 03:36:57 DEBUG 03:36:57 DEBUG "pod": "am-767f6b67b5-9zw22" 03:36:57 DEBUG }, 03:36:57 DEBUG "result": [ 03:36:57 DEBUG "result": [ 03:36:57 DEBUG "result": [ 03:36:57 DEBUG 03:36:57 DEBUG "result": [ 03:36:57 DEBUG "value": [ 03:36:57 DEBUG { 03:36:57 DEBUG { 03:36:57 DEBUG { 03:36:57 DEBUG 03:36:57 DEBUG { 03:36:57 DEBUG 03:36:57 DEBUG "metric": { 03:36:57 DEBUG "metric": { 03:36:57 DEBUG "metric": { 03:36:57 DEBUG [...] 03:36:57 DEBUG "metric": { 03:36:57 DEBUG 03:36:57 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:36:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:36:57 DEBUG "pod": "am-767f6b67b5-9zw22" 03:36:57 DEBUG 03:36:57 DEBUG "pod": "ds-idrepo-0" 03:36:57 DEBUG 03:36:57 DEBUG }, 03:36:57 DEBUG }, 03:36:57 DEBUG }, 03:36:57 DEBUG 03:36:57 DEBUG }, 03:36:57 DEBUG [...] 03:36:57 DEBUG "value": [ 03:36:57 DEBUG "value": [ 03:36:57 DEBUG "value": [ 03:36:57 DEBUG 03:36:57 DEBUG "value": [ 03:36:57 DEBUG 03:36:57 DEBUG 03:36:57 DEBUG 03:36:57 DEBUG 03:36:57 DEBUG "pod": "am-767f6b67b5-vbltz" 03:36:57 DEBUG 03:36:57 DEBUG 03:36:57 DEBUG 03:36:57 DEBUG 03:36:57 DEBUG 03:36:57 DEBUG }, 03:36:57 DEBUG 03:36:57 DEBUG 03:36:57 DEBUG "pod": "am-767f6b67b5-vbltz" 03:36:57 DEBUG 03:36:57 DEBUG [...] 03:36:57 DEBUG "value": [ 03:36:57 DEBUG 03:36:57 DEBUG 03:36:57 DEBUG 03:36:57 DEBUG 03:36:57 DEBUG 1713667017, 03:36:57 DEBUG [...] 03:36:57 DEBUG }, 03:36:57 DEBUG [...] 03:36:57 DEBUG [...] 03:36:57 DEBUG 03:36:57 DEBUG "395.6333333333333" 03:36:57 DEBUG 03:36:57 DEBUG "value": [ 03:36:57 DEBUG 03:36:57 DEBUG 03:36:57 DEBUG 03:36:57 DEBUG ] 03:36:57 DEBUG 03:36:57 DEBUG 1713667017, 03:36:57 DEBUG 03:36:57 DEBUG 03:36:57 DEBUG "pod": "am-767f6b67b5-qcqz2" 03:36:57 DEBUG } 03:36:57 DEBUG 03:36:57 DEBUG "364256" 03:36:57 DEBUG 03:36:57 DEBUG 03:36:57 DEBUG "pod": "am-767f6b67b5-vbltz" 03:36:57 DEBUG }, 03:36:57 DEBUG ] 03:36:57 DEBUG ] 03:36:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:36:57 DEBUG "pod": "ds-idrepo-2" 03:36:57 DEBUG "value": [ 03:36:57 DEBUG }, 03:36:57 DEBUG } 03:36:57 DEBUG } 03:36:57 DEBUG }, 03:36:57 DEBUG }, 03:36:57 DEBUG 1713667017, 03:36:57 DEBUG "value": [ 03:36:57 DEBUG } 03:36:57 DEBUG ] 03:36:57 DEBUG "value": [ 03:36:57 DEBUG "value": [ 03:36:57 DEBUG "0" 03:36:57 DEBUG 1713667017, 03:36:57 DEBUG } 03:36:57 DEBUG 1713667017, 03:36:57 DEBUG 1713667017, 03:36:57 DEBUG ] 03:36:57 DEBUG "8.407843229660356" 03:36:57 DEBUG } 03:36:57 DEBUG "14.845000471023553" 03:36:57 DEBUG "0" 03:36:57 DEBUG } 03:36:57 DEBUG ] 03:36:57 DEBUG ] 03:36:57 DEBUG ] 03:36:57 DEBUG ] 03:36:57 DEBUG } 03:36:57 DEBUG } 03:36:57 DEBUG ] 03:36:57 DEBUG } 03:36:57 DEBUG ] 03:36:57 DEBUG } 03:36:57 DEBUG } 03:36:57 DEBUG } 03:36:57 DEBUG } 03:36:57 DEBUG } 03:36:57 DEBUG ] 03:36:57 DEBUG } 03:36:57 DEBUG } 03:36:57 DEBUG } 03:36:58 INFO 03:36:58 INFO [loop_until]: kubectl --namespace=xlou top node 03:36:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:36:58 INFO [loop_until]: OK (rc = 0) 03:36:58 DEBUG --- stdout --- 03:36:58 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 62m 0% 6284Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 1985m 6% 7767Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 6426m 21% 11662Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 84m 0% 3350Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 19461m 65% 15828Mi 14% gke-xlou-cdm-default-pool-1b0769c8-wpwl 152m 0% 5256Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 4965m 16% 12130Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 2951m 9% 3984Mi 3% gke-xlou-cdm-ds-0f601751-hmm7 7029m 23% 11207Mi 10% gke-xlou-cdm-ds-0f601751-j488 1624m 5% 7346Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 17611m 58% 17608Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 2342m 7% 8031Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 19385m 64% 15280Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 6830m 42% 10315Mi 17% 03:36:58 DEBUG --- stderr --- 03:36:58 DEBUG 03:36:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713667018" 03:36:58 INFO [http_cmd]: http status code OK 03:36:58 DEBUG --- status code --- 03:36:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:36:58 DEBUG --- http response --- 03:36:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:36:58 DEBUG ----- output ----- 03:36:58 DEBUG { 03:36:58 DEBUG "status": "success", 03:36:58 DEBUG "data": { 03:36:58 DEBUG "resultType": "vector", 03:36:58 DEBUG "result": [ 03:36:58 DEBUG { 03:36:58 DEBUG "metric": { 03:36:58 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:36:58 DEBUG }, 03:36:58 DEBUG "value": [ 03:36:58 DEBUG 03:36:58 DEBUG 03:36:58 DEBUG 03:36:58 DEBUG [...] 03:36:58 DEBUG 03:36:58 DEBUG 03:36:58 DEBUG 03:36:58 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:36:58 DEBUG }, 03:36:58 DEBUG "value": [ 03:36:58 DEBUG 1713667018, 03:36:58 DEBUG "14618464256" 03:36:58 DEBUG ] 03:36:58 DEBUG } 03:36:58 DEBUG ] 03:36:58 DEBUG } 03:36:58 DEBUG } 03:36:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667018" 03:36:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713667018" 03:36:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713667018" 03:36:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713667018" 03:36:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713667018" 03:36:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667018" 03:36:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667018" 03:36:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713667018" 03:36:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667018" 03:36:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667018" 03:36:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667018" 03:36:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713667018" 03:36:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667018" 03:36:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667018" 03:36:58 INFO [http_cmd]: http status code OK 03:36:58 INFO [http_cmd]: http status code OK 03:36:58 INFO [http_cmd]: http status code OK 03:36:58 INFO [http_cmd]: http status code OK 03:36:58 INFO [http_cmd]: http status code OK 03:36:59 DEBUG --- status code --- 03:36:59 INFO [http_cmd]: http status code OK 03:36:59 INFO [http_cmd]: http status code OK 03:36:59 INFO [http_cmd]: http status code OK 03:36:59 DEBUG --- status code --- 03:36:59 DEBUG --- status code --- 03:36:59 DEBUG --- status code --- 03:36:59 INFO [http_cmd]: http status code OK 03:36:59 INFO [http_cmd]: http status code OK 03:36:59 INFO [http_cmd]: http status code OK 03:36:59 DEBUG --- status code --- 03:36:59 INFO [http_cmd]: http status code OK 03:36:59 DEBUG --- status code --- 03:36:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:36:59 DEBUG --- status code --- 03:36:59 DEBUG --- status code --- 03:36:59 DEBUG --- status code --- 03:36:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:36:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:36:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:36:59 DEBUG --- status code --- 03:36:59 DEBUG --- status code --- 03:36:59 DEBUG --- status code --- 03:36:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:36:59 INFO [http_cmd]: http status code OK 03:36:58 INFO [http_cmd]: http status code OK 03:36:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:36:59 DEBUG --- http response --- 03:36:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:36:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:36:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:36:59 DEBUG --- http response --- 03:36:59 DEBUG --- http response --- 03:36:59 DEBUG --- http response --- 03:36:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:36:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:36:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:36:59 DEBUG --- http response --- 03:36:59 DEBUG --- status code --- 03:36:59 DEBUG --- status code --- 03:36:59 DEBUG --- http response --- 03:36:59 DEBUG --- http response --- 03:36:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:36:59 DEBUG --- http response --- 03:36:59 DEBUG --- http response --- 03:36:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:36:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:36:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:36:59 DEBUG --- http response --- 03:36:59 DEBUG --- http response --- 03:36:59 DEBUG --- http response --- 03:36:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:36:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:36:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:36:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:36:59 DEBUG ----- output ----- 03:36:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:36:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:36:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 03:36:59 DEBUG ----- output ----- 03:36:59 DEBUG ----- output ----- 03:36:59 DEBUG ----- output ----- 03:36:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:36:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:36:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:36:59 DEBUG --- http response --- 03:36:59 DEBUG ----- output ----- 03:36:59 DEBUG --- http response --- 03:36:59 DEBUG ----- output ----- 03:36:59 DEBUG { 03:36:59 DEBUG ----- output ----- 03:36:59 DEBUG ----- output ----- 03:36:59 DEBUG ----- output ----- 03:36:59 DEBUG { 03:36:59 DEBUG { 03:36:59 DEBUG { 03:36:59 DEBUG ----- output ----- 03:36:59 DEBUG ----- output ----- 03:36:59 DEBUG ----- output ----- 03:36:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 03:36:59 DEBUG { 03:36:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:36:59 DEBUG { 03:36:59 DEBUG "status": "success", 03:36:59 DEBUG { 03:36:59 DEBUG { 03:36:59 DEBUG { 03:36:59 DEBUG "status": "success", 03:36:59 DEBUG "status": "success", 03:36:59 DEBUG "status": "success", 03:36:59 DEBUG { 03:36:59 DEBUG { 03:36:59 DEBUG { 03:36:59 DEBUG ----- output ----- 03:36:59 DEBUG "status": "success", 03:36:59 DEBUG ----- output ----- 03:36:59 DEBUG { 03:36:59 DEBUG "data": { 03:36:59 DEBUG "status": "success", 03:36:59 DEBUG "status": "success", 03:36:59 DEBUG "status": "success", 03:36:59 DEBUG "data": { 03:36:59 DEBUG "data": { 03:36:59 DEBUG "data": { 03:36:59 DEBUG "status": "success", 03:36:59 DEBUG "status": "success", 03:36:59 DEBUG "status": "success", 03:36:59 DEBUG { 03:36:59 DEBUG "data": { 03:36:59 DEBUG "status": "success", 03:36:59 DEBUG "status": "success", 03:36:59 DEBUG "resultType": "vector", 03:36:59 DEBUG "data": { 03:36:59 DEBUG "data": { 03:36:59 DEBUG "data": { 03:36:59 DEBUG "resultType": "vector", 03:36:59 DEBUG "resultType": "vector", 03:36:59 DEBUG "resultType": "vector", 03:36:59 DEBUG "data": { 03:36:59 DEBUG "data": { 03:36:59 DEBUG "data": { 03:36:59 DEBUG "status": "success", 03:36:59 DEBUG "resultType": "vector", 03:36:59 DEBUG "data": { 03:36:59 DEBUG "data": { 03:36:59 DEBUG "result": [ 03:36:59 DEBUG "resultType": "vector", 03:36:59 DEBUG "resultType": "vector", 03:36:59 DEBUG "resultType": "vector", 03:36:59 DEBUG "result": [ 03:36:59 DEBUG "result": [ 03:36:59 DEBUG "result": [ 03:36:59 DEBUG "resultType": "vector", 03:36:59 DEBUG "resultType": "vector", 03:36:59 DEBUG "resultType": "vector", 03:36:59 DEBUG "data": { 03:36:59 DEBUG "result": [ 03:36:59 DEBUG { 03:36:59 DEBUG "resultType": "vector", 03:36:59 DEBUG { 03:36:59 DEBUG "result": [ 03:36:59 DEBUG { 03:36:59 DEBUG "result": [ 03:36:59 DEBUG { 03:36:59 DEBUG { 03:36:59 DEBUG { 03:36:59 DEBUG "result": [ 03:36:59 DEBUG "result": [ 03:36:59 DEBUG "result": [ 03:36:59 DEBUG "resultType": "vector", 03:36:59 DEBUG "resultType": "vector", 03:36:59 DEBUG "metric": { 03:36:59 DEBUG "result": [ 03:36:59 DEBUG "metric": { 03:36:59 DEBUG "result": [ 03:36:59 DEBUG "metric": { 03:36:59 DEBUG { 03:36:59 DEBUG "metric": { 03:36:59 DEBUG "metric": { 03:36:59 DEBUG "metric": { 03:36:59 DEBUG { 03:36:59 DEBUG { 03:36:59 DEBUG { 03:36:59 DEBUG "result": [ 03:36:59 DEBUG "result": [ 03:36:59 DEBUG "pod": "am-767f6b67b5-9zw22" 03:36:59 DEBUG { 03:36:59 DEBUG "instance": "10.142.0.107:9100" 03:36:59 DEBUG { 03:36:59 DEBUG "pod": "am-767f6b67b5-9zw22" 03:36:59 DEBUG "metric": { 03:36:59 DEBUG "pod": "ds-cts-0" 03:36:59 DEBUG "pod": "am-767f6b67b5-9zw22" 03:36:59 DEBUG "instance": "10.142.0.107:9100" 03:36:59 DEBUG "metric": { 03:36:59 DEBUG "metric": { 03:36:59 DEBUG "metric": { 03:36:59 DEBUG { 03:36:59 DEBUG { 03:36:59 DEBUG }, 03:36:59 DEBUG "metric": { 03:36:59 DEBUG }, 03:36:59 DEBUG "metric": { 03:36:59 DEBUG }, 03:36:59 DEBUG }, 03:36:59 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:36:59 DEBUG }, 03:36:59 DEBUG }, 03:36:59 DEBUG "pod": "ds-idrepo-0" 03:36:59 DEBUG "pod": "ds-idrepo-0" 03:36:59 DEBUG "instance": "10.142.0.107:9100" 03:36:59 DEBUG "metric": { 03:36:59 DEBUG "metric": { 03:36:59 DEBUG "value": [ 03:36:59 DEBUG "instance": "10.142.0.107:9100" 03:36:59 DEBUG "value": [ 03:36:59 DEBUG "pod": "ds-cts-0" 03:36:59 DEBUG "value": [ 03:36:59 DEBUG "value": [ 03:36:59 DEBUG 03:36:59 DEBUG "value": [ 03:36:59 DEBUG "value": [ 03:36:59 DEBUG }, 03:36:59 DEBUG }, 03:36:59 DEBUG }, 03:36:59 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:36:59 DEBUG "pod": "ds-idrepo-0" 03:36:59 DEBUG 03:36:59 DEBUG }, 03:36:59 DEBUG 03:36:59 DEBUG }, 03:36:59 DEBUG 03:36:59 DEBUG 03:36:59 DEBUG 03:36:59 DEBUG [...] 03:36:59 DEBUG 03:36:59 DEBUG 03:36:59 DEBUG 03:36:59 DEBUG "value": [ 03:36:59 DEBUG }, 03:36:59 DEBUG }, 03:36:59 DEBUG 03:36:59 DEBUG "value": [ 03:36:59 DEBUG 03:36:59 DEBUG "value": [ 03:36:59 DEBUG 03:36:59 DEBUG 03:36:59 DEBUG }, 03:36:59 DEBUG "value": [ 03:36:59 DEBUG "value": [ 03:36:59 DEBUG [...] 03:36:59 DEBUG 03:36:59 DEBUG "value": [ 03:36:59 DEBUG "value": [ 03:36:59 DEBUG "value": [ 03:36:59 DEBUG 03:36:59 DEBUG 03:36:59 DEBUG 03:36:59 DEBUG 03:36:59 DEBUG 03:36:59 DEBUG 03:36:59 DEBUG 03:36:59 DEBUG 03:36:59 DEBUG 03:36:59 DEBUG 03:36:59 DEBUG 03:36:59 DEBUG 03:36:59 DEBUG 03:36:59 DEBUG 03:36:59 DEBUG [...] 03:36:59 DEBUG 03:36:59 DEBUG 03:36:59 DEBUG 03:36:59 DEBUG [...] 03:36:59 DEBUG [...] 03:36:59 DEBUG 03:36:59 DEBUG 03:36:59 DEBUG 03:36:59 DEBUG 03:36:59 DEBUG 03:36:59 DEBUG 03:36:59 DEBUG 03:36:59 DEBUG 03:36:59 DEBUG 03:36:59 DEBUG [...] 03:36:59 DEBUG 03:36:59 DEBUG 03:36:59 DEBUG 03:36:59 DEBUG 03:36:59 DEBUG 03:36:59 DEBUG 03:36:59 DEBUG 03:36:59 DEBUG "instance": "10.142.0.43:9100" 03:36:59 DEBUG 03:36:59 DEBUG 03:36:59 DEBUG 03:36:59 DEBUG 03:36:59 DEBUG 03:36:59 DEBUG 03:36:59 DEBUG 03:36:59 DEBUG [...] 03:36:59 DEBUG 03:36:59 DEBUG "pod": "am-767f6b67b5-vbltz" 03:36:59 DEBUG [...] 03:36:59 DEBUG "pod": "ds-cts-2" 03:36:59 DEBUG [...] 03:36:59 DEBUG [...] 03:36:59 DEBUG }, 03:36:59 DEBUG [...] 03:36:59 DEBUG [...] 03:36:59 DEBUG [...] 03:36:59 DEBUG 03:36:59 DEBUG [...] 03:36:59 DEBUG 03:36:59 DEBUG 03:36:59 DEBUG }, 03:36:59 DEBUG 03:36:59 DEBUG 03:36:59 DEBUG }, 03:36:59 DEBUG 03:36:59 DEBUG 03:36:59 DEBUG "value": [ 03:36:59 DEBUG 03:36:59 DEBUG 03:36:59 DEBUG 03:36:59 DEBUG "pod": "am-767f6b67b5-vbltz" 03:36:59 DEBUG 03:36:59 DEBUG "instance": "10.142.0.43:9100" 03:36:59 DEBUG }, 03:36:59 DEBUG "value": [ 03:36:59 DEBUG 03:36:59 DEBUG 03:36:59 DEBUG "value": [ 03:36:59 DEBUG 03:36:59 DEBUG 03:36:59 DEBUG 1713667018, 03:36:59 DEBUG 03:36:59 DEBUG 03:36:59 DEBUG 03:36:59 DEBUG }, 03:36:59 DEBUG 03:36:59 DEBUG "value": [ 03:36:59 DEBUG 1713667018, 03:36:59 DEBUG "instance": "10.142.0.43:9100" 03:36:59 DEBUG 03:36:59 DEBUG 03:36:59 DEBUG 1713667018, 03:36:59 DEBUG 03:36:59 DEBUG 03:36:59 DEBUG "55244496.96666667" 03:36:59 DEBUG 03:36:59 DEBUG 03:36:59 DEBUG 03:36:59 DEBUG "value": [ 03:36:59 DEBUG 03:36:59 DEBUG 1713667018, 03:36:59 DEBUG "4261.766666666666" 03:36:59 DEBUG }, 03:36:59 DEBUG 03:36:59 DEBUG 03:36:59 DEBUG "0" 03:36:59 DEBUG "pod": "ds-idrepo-2" 03:36:59 DEBUG "instance": "10.142.0.43:9100" 03:36:59 DEBUG ] 03:36:59 DEBUG "pod": "ds-cts-2" 03:36:59 DEBUG "pod": "am-767f6b67b5-qcqz2" 03:36:59 DEBUG "pod": "ds-idrepo-2" 03:36:59 DEBUG 1713667018, 03:36:59 DEBUG "pod": "ds-cts-2" 03:36:59 DEBUG "4480196633600" 03:36:59 DEBUG ] 03:36:59 DEBUG ] 03:36:59 DEBUG "pod": "am-767f6b67b5-qcqz2" 03:36:59 DEBUG "pod": "am-767f6b67b5-vbltz" 03:36:59 DEBUG ] 03:36:59 DEBUG }, 03:36:59 DEBUG }, 03:36:59 DEBUG } 03:36:59 DEBUG }, 03:36:59 DEBUG }, 03:36:59 DEBUG }, 03:36:59 DEBUG "395.9" 03:36:59 DEBUG }, 03:36:59 DEBUG "value": [ 03:36:59 DEBUG } 03:36:59 DEBUG } 03:36:59 DEBUG }, 03:36:59 DEBUG }, 03:36:59 DEBUG } 03:36:59 DEBUG "value": [ 03:36:59 DEBUG "value": [ 03:36:59 DEBUG ] 03:36:59 DEBUG } 03:36:59 DEBUG "value": [ 03:36:59 DEBUG "value": [ 03:36:59 DEBUG ] 03:36:59 DEBUG "value": [ 03:36:59 DEBUG 1713667018, 03:36:59 DEBUG "55247714.56666666" 03:36:59 DEBUG ] 03:36:59 DEBUG "value": [ 03:36:59 DEBUG "value": [ 03:36:59 DEBUG ] 03:36:59 DEBUG 1713667018, 03:36:59 DEBUG 1713667018, 03:36:59 DEBUG "value": [ 03:36:59 DEBUG } 03:36:59 DEBUG 1713667018, 03:36:59 DEBUG 1713667018, 03:36:59 DEBUG } 03:36:59 DEBUG 1713667018, 03:36:59 DEBUG ] 03:36:59 DEBUG ] 03:36:59 DEBUG } 03:36:59 DEBUG 1713667018, 03:36:59 DEBUG 1713667018, 03:36:59 DEBUG } 03:36:59 DEBUG "0" 03:36:59 DEBUG "0.09366666666667431" 03:36:59 DEBUG 1713667018, 03:36:59 DEBUG "610.0431763738424" 03:36:59 DEBUG "0" 03:36:59 DEBUG ] 03:36:59 DEBUG "30.266666666666666" 03:36:59 DEBUG } 03:36:59 DEBUG } 03:36:59 DEBUG } 03:36:59 DEBUG "0" 03:36:59 DEBUG "190.91649999999998" 03:36:59 DEBUG } 03:36:59 DEBUG ] 03:36:59 DEBUG ] 03:36:59 DEBUG "0" 03:36:59 DEBUG ] 03:36:59 DEBUG ] 03:36:59 DEBUG } 03:36:59 DEBUG ] 03:36:59 DEBUG ] 03:36:59 DEBUG } 03:36:59 DEBUG ] 03:36:59 DEBUG ] 03:36:59 DEBUG } 03:36:59 DEBUG } 03:36:59 DEBUG ] 03:36:59 DEBUG } 03:36:59 DEBUG } 03:36:59 DEBUG } 03:36:59 DEBUG } 03:36:59 DEBUG } 03:36:59 DEBUG } 03:36:59 DEBUG } 03:36:59 DEBUG ] 03:36:59 DEBUG ] 03:36:59 DEBUG } 03:36:59 DEBUG ] 03:36:59 DEBUG ] 03:36:59 DEBUG ] 03:36:59 DEBUG } 03:36:59 DEBUG ] 03:36:59 DEBUG ] 03:36:59 DEBUG } 03:36:59 DEBUG } 03:36:59 DEBUG ] 03:36:59 DEBUG } 03:36:59 DEBUG } 03:36:59 DEBUG } 03:36:59 DEBUG } 03:36:59 DEBUG } 03:36:59 DEBUG } 03:36:59 DEBUG } 03:36:59 DEBUG } 03:36:59 DEBUG } 03:36:59 DEBUG } 03:36:59 DEBUG } 03:36:59 DEBUG } 03:36:59 DEBUG } 03:36:59 DEBUG } 03:37:01 WARNING File /tmp/lodemon_data-20240421_025941/node_disk_read_bytes_total.json does not exist. Skipping average value computation... 127.0.0.1 - - [21/Apr/2024 03:37:01] "GET /monitoring/average?start_time=20240421_023410&stop_time=20240421_023500 HTTP/1.1" 200 - 03:37:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667074" 03:37:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713667074" 03:37:54 INFO [http_cmd]: http status code OK 03:37:54 DEBUG --- status code --- 03:37:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:37:54 DEBUG --- http response --- 03:37:54 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 03:37:54 ERROR [http_cmd]: ERROR 03:37:54 ERROR --- status code --- 03:37:54 ERROR http status code is 400 (expected [200, 401, 403, 404]) 03:37:54 ERROR --- http response --- 03:37:54 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 03:37:54 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 03:37:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713667076" 03:37:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667076" 03:37:56 INFO [http_cmd]: http status code OK 03:37:56 DEBUG --- status code --- 03:37:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:37:56 DEBUG --- http response --- 03:37:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:37:56 DEBUG ----- output ----- 03:37:56 DEBUG { 03:37:56 DEBUG "status": "success", 03:37:56 DEBUG "data": { 03:37:56 DEBUG "resultType": "vector", 03:37:56 DEBUG "result": [ 03:37:56 DEBUG { 03:37:56 DEBUG "metric": { 03:37:56 DEBUG "pod": "ds-cts-0" 03:37:56 DEBUG }, 03:37:56 DEBUG "value": [ 03:37:56 DEBUG 03:37:56 DEBUG 03:37:56 DEBUG 03:37:56 DEBUG [...] 03:37:56 DEBUG 03:37:56 DEBUG 03:37:56 DEBUG 03:37:56 DEBUG "pod": "ds-cts-2" 03:37:56 DEBUG }, 03:37:56 DEBUG "value": [ 03:37:56 DEBUG 1713667076, 03:37:56 DEBUG "28.904233333333334" 03:37:56 DEBUG ] 03:37:56 DEBUG } 03:37:56 DEBUG ] 03:37:56 DEBUG } 03:37:56 DEBUG } 03:37:56 INFO [http_cmd]: http status code OK 03:37:56 DEBUG --- status code --- 03:37:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:37:56 DEBUG --- http response --- 03:37:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:37:56 DEBUG ----- output ----- 03:37:56 DEBUG { 03:37:56 DEBUG "status": "success", 03:37:56 DEBUG "data": { 03:37:56 DEBUG "resultType": "vector", 03:37:56 DEBUG "result": [ 03:37:56 DEBUG { 03:37:56 DEBUG "metric": { 03:37:56 DEBUG "instance": "10.142.0.107:9100" 03:37:56 DEBUG }, 03:37:56 DEBUG "value": [ 03:37:56 DEBUG 03:37:56 DEBUG 03:37:56 DEBUG 03:37:56 DEBUG [...] 03:37:56 DEBUG 03:37:56 DEBUG 03:37:56 DEBUG 03:37:56 DEBUG "instance": "10.142.0.43:9100" 03:37:56 DEBUG }, 03:37:56 DEBUG "value": [ 03:37:56 DEBUG 1713667076, 03:37:56 DEBUG "0.45063333333336764" 03:37:56 DEBUG ] 03:37:56 DEBUG } 03:37:56 DEBUG ] 03:37:56 DEBUG } 03:37:56 DEBUG } 03:37:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667076" 03:37:56 INFO [http_cmd]: http status code OK 03:37:56 DEBUG --- status code --- 03:37:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:37:56 DEBUG --- http response --- 03:37:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:37:56 DEBUG ----- output ----- 03:37:56 DEBUG { 03:37:56 DEBUG "status": "success", 03:37:56 DEBUG "data": { 03:37:56 DEBUG "resultType": "vector", 03:37:56 DEBUG "result": [ 03:37:56 DEBUG { 03:37:56 DEBUG "metric": { 03:37:56 DEBUG "pod": "am-767f6b67b5-9zw22" 03:37:56 DEBUG }, 03:37:56 DEBUG "value": [ 03:37:56 DEBUG 03:37:56 DEBUG 03:37:56 DEBUG 03:37:56 DEBUG [...] 03:37:56 DEBUG 03:37:56 DEBUG 03:37:56 DEBUG 03:37:56 DEBUG "pod": "am-767f6b67b5-vbltz" 03:37:56 DEBUG }, 03:37:56 DEBUG "value": [ 03:37:56 DEBUG 1713667076, 03:37:56 DEBUG "0.06666666666666667" 03:37:56 DEBUG ] 03:37:56 DEBUG } 03:37:56 DEBUG ] 03:37:56 DEBUG } 03:37:56 DEBUG } 03:37:57 INFO 03:37:57 INFO [loop_until]: kubectl --namespace=xlou top pods 03:37:57 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:37:57 INFO [loop_until]: OK (rc = 0) 03:37:57 DEBUG --- stdout --- 03:37:57 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 2560m 8727Mi am-767f6b67b5-qcqz2 2583m 9070Mi am-767f6b67b5-vbltz 2877m 8787Mi ds-cts-0 18152m 12504Mi ds-cts-1 17820m 12462Mi ds-cts-2 19701m 12635Mi ds-idrepo-0 747m 3924Mi ds-idrepo-1 1026m 3959Mi ds-idrepo-2 962m 3923Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 8m 1180Mi idm-b85f9d978-wnhxb 7m 1239Mi lodemon-54c87f8cbb-jzrjs 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-68c5d5b884-p97w5 372m 443Mi 03:37:57 DEBUG --- stderr --- 03:37:57 DEBUG 03:37:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667077" 03:37:57 INFO [http_cmd]: http status code OK 03:37:57 DEBUG --- status code --- 03:37:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:37:57 DEBUG --- http response --- 03:37:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:37:57 DEBUG ----- output ----- 03:37:57 DEBUG { 03:37:57 DEBUG "status": "success", 03:37:57 DEBUG "data": { 03:37:57 DEBUG "resultType": "vector", 03:37:57 DEBUG "result": [ 03:37:57 DEBUG { 03:37:57 DEBUG "metric": { 03:37:57 DEBUG "pod": "am-767f6b67b5-9zw22" 03:37:57 DEBUG }, 03:37:57 DEBUG "value": [ 03:37:57 DEBUG 03:37:57 DEBUG 03:37:57 DEBUG 03:37:57 DEBUG [...] 03:37:57 DEBUG 03:37:57 DEBUG 03:37:57 DEBUG 03:37:57 DEBUG "pod": "am-767f6b67b5-vbltz" 03:37:57 DEBUG }, 03:37:57 DEBUG "value": [ 03:37:57 DEBUG 1713667077, 03:37:57 DEBUG "0.06666666666666667" 03:37:57 DEBUG ] 03:37:57 DEBUG } 03:37:57 DEBUG ] 03:37:57 DEBUG } 03:37:57 DEBUG } 03:37:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667077" 03:37:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667077" 03:37:57 INFO [http_cmd]: http status code OK 03:37:57 DEBUG --- status code --- 03:37:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:37:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667077" 03:37:57 DEBUG --- http response --- 03:37:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713667077" 03:37:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667077" 03:37:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667077" 03:37:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:37:58 DEBUG ----- output ----- 03:37:58 DEBUG { 03:37:58 DEBUG "status": "success", 03:37:58 DEBUG "data": { 03:37:58 DEBUG "resultType": "vector", 03:37:58 DEBUG "result": [ 03:37:58 DEBUG { 03:37:58 DEBUG "metric": { 03:37:58 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:37:58 DEBUG }, 03:37:58 DEBUG "value": [ 03:37:58 DEBUG 03:37:58 DEBUG 03:37:58 DEBUG 03:37:58 DEBUG [...] 03:37:58 DEBUG 03:37:58 DEBUG 03:37:58 DEBUG 03:37:58 DEBUG "pod": "idm-b85f9d978-txpwv" 03:37:58 DEBUG }, 03:37:58 DEBUG "value": [ 03:37:58 DEBUG 1713667077, 03:37:58 DEBUG "424.3808502963153" 03:37:58 DEBUG ] 03:37:58 DEBUG } 03:37:58 DEBUG ] 03:37:58 DEBUG } 03:37:58 DEBUG } 03:37:58 INFO [http_cmd]: http status code OK 03:37:58 DEBUG --- status code --- 03:37:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:37:58 DEBUG --- http response --- 03:37:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:37:58 DEBUG ----- output ----- 03:37:58 DEBUG { 03:37:58 DEBUG "status": "success", 03:37:58 DEBUG "data": { 03:37:58 DEBUG "resultType": "vector", 03:37:58 DEBUG "result": [ 03:37:58 DEBUG { 03:37:58 DEBUG "metric": { 03:37:58 DEBUG "pod": "am-767f6b67b5-9zw22" 03:37:58 DEBUG }, 03:37:58 DEBUG "value": [ 03:37:58 DEBUG 03:37:58 DEBUG 03:37:58 DEBUG 03:37:58 DEBUG [...] 03:37:58 DEBUG 03:37:58 INFO [http_cmd]: http status code OK 03:37:58 DEBUG 03:37:58 INFO [http_cmd]: http status code OK 03:37:58 DEBUG --- status code --- 03:37:58 DEBUG 03:37:58 DEBUG --- status code --- 03:37:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:37:58 DEBUG "pod": "am-767f6b67b5-vbltz" 03:37:58 DEBUG --- http response --- 03:37:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:37:58 DEBUG }, 03:37:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:37:58 DEBUG --- http response --- 03:37:58 DEBUG "value": [ 03:37:58 INFO [http_cmd]: http status code OK 03:37:58 DEBUG ----- output ----- 03:37:58 DEBUG 1713667077, 03:37:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:37:58 INFO [http_cmd]: http status code OK 03:37:58 DEBUG --- status code --- 03:37:58 DEBUG { 03:37:58 DEBUG "222871.5" 03:37:58 DEBUG ----- output ----- 03:37:58 DEBUG --- status code --- 03:37:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:37:58 DEBUG "status": "success", 03:37:58 DEBUG ] 03:37:58 DEBUG { 03:37:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:37:58 DEBUG --- http response --- 03:37:58 DEBUG "data": { 03:37:58 DEBUG } 03:37:58 DEBUG "status": "success", 03:37:58 DEBUG --- http response --- 03:37:58 DEBUG "resultType": "vector", 03:37:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:37:58 DEBUG ] 03:37:58 DEBUG "data": { 03:37:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:37:58 DEBUG "result": [ 03:37:58 DEBUG ----- output ----- 03:37:58 DEBUG } 03:37:58 DEBUG "resultType": "vector", 03:37:58 DEBUG ----- output ----- 03:37:58 DEBUG { 03:37:58 DEBUG { 03:37:58 DEBUG } 03:37:58 DEBUG "result": [ 03:37:58 DEBUG { 03:37:58 DEBUG "metric": { 03:37:58 DEBUG "status": "success", 03:37:58 DEBUG { 03:37:58 DEBUG "status": "success", 03:37:58 DEBUG "pod": "ds-idrepo-0" 03:37:58 DEBUG "data": { 03:37:58 DEBUG "metric": { 03:37:58 DEBUG "data": { 03:37:58 DEBUG }, 03:37:58 DEBUG "resultType": "vector", 03:37:58 DEBUG "pod": "am-767f6b67b5-9zw22" 03:37:58 DEBUG "resultType": "vector", 03:37:58 DEBUG "value": [ 03:37:58 DEBUG "result": [ 03:37:58 DEBUG }, 03:37:58 DEBUG "result": [ 03:37:58 DEBUG 03:37:58 DEBUG { 03:37:58 DEBUG "value": [ 03:37:58 DEBUG { 03:37:58 DEBUG 03:37:58 DEBUG "metric": { 03:37:58 DEBUG 03:37:58 DEBUG "metric": { 03:37:58 DEBUG 03:37:58 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:37:58 DEBUG 03:37:58 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:37:58 DEBUG [...] 03:37:58 DEBUG }, 03:37:58 DEBUG 03:37:58 DEBUG }, 03:37:58 DEBUG 03:37:58 DEBUG "value": [ 03:37:58 DEBUG [...] 03:37:58 DEBUG "value": [ 03:37:58 DEBUG 03:37:58 DEBUG 03:37:58 DEBUG 03:37:58 DEBUG 03:37:58 DEBUG 03:37:58 DEBUG 03:37:58 DEBUG 03:37:58 DEBUG 03:37:58 DEBUG "pod": "ds-idrepo-2" 03:37:58 DEBUG 03:37:58 DEBUG 03:37:58 DEBUG 03:37:58 DEBUG }, 03:37:58 DEBUG [...] 03:37:58 DEBUG "pod": "am-767f6b67b5-vbltz" 03:37:58 DEBUG [...] 03:37:58 DEBUG "value": [ 03:37:58 DEBUG 03:37:58 DEBUG }, 03:37:58 DEBUG 03:37:58 DEBUG 03:37:58 DEBUG 03:37:58 DEBUG "value": [ 03:37:58 DEBUG 03:37:58 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:37:58 DEBUG 1713667077, 03:37:58 DEBUG 03:37:58 DEBUG 1713667077, 03:37:58 DEBUG }, 03:37:58 DEBUG "0" 03:37:58 DEBUG "pod": "idm-b85f9d978-txpwv" 03:37:58 DEBUG "0.06666666666666667" 03:37:58 DEBUG "value": [ 03:37:58 DEBUG ] 03:37:58 DEBUG }, 03:37:58 DEBUG ] 03:37:58 DEBUG 1713667077, 03:37:58 DEBUG } 03:37:58 DEBUG "value": [ 03:37:58 DEBUG } 03:37:58 DEBUG "0.02828723474154651" 03:37:58 DEBUG ] 03:37:58 DEBUG 1713667077, 03:37:58 DEBUG ] 03:37:58 DEBUG ] 03:37:58 DEBUG } 03:37:58 DEBUG "0.0023220070059218603" 03:37:58 DEBUG } 03:37:58 DEBUG } 03:37:58 DEBUG } 03:37:58 DEBUG ] 03:37:58 DEBUG } 03:37:58 DEBUG ] 03:37:58 DEBUG } 03:37:58 DEBUG } 03:37:58 DEBUG ] 03:37:58 DEBUG } 03:37:58 DEBUG } 03:37:58 DEBUG } 03:37:58 INFO 03:37:58 INFO [loop_until]: kubectl --namespace=xlou top node 03:37:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:37:58 INFO [loop_until]: OK (rc = 0) 03:37:58 DEBUG --- stdout --- 03:37:58 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 66m 0% 6292Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 1022m 3% 7838Mi 7% gke-xlou-cdm-default-pool-1b0769c8-nhlf 2669m 8% 11570Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 79m 0% 3368Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 18448m 61% 15827Mi 14% gke-xlou-cdm-default-pool-1b0769c8-wpwl 194m 0% 5255Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 2667m 8% 12039Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 136m 0% 3874Mi 3% gke-xlou-cdm-ds-0f601751-hmm7 3246m 10% 11075Mi 9% gke-xlou-cdm-ds-0f601751-j488 965m 3% 7348Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 17720m 59% 17599Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 965m 3% 8000Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 18389m 61% 15276Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 901m 5% 10650Mi 18% 03:37:58 DEBUG --- stderr --- 03:37:58 DEBUG 03:37:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713667078" 03:37:58 INFO [http_cmd]: http status code OK 03:37:58 DEBUG --- status code --- 03:37:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:37:58 DEBUG --- http response --- 03:37:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:37:58 DEBUG ----- output ----- 03:37:58 DEBUG { 03:37:58 DEBUG "status": "success", 03:37:58 DEBUG "data": { 03:37:58 DEBUG "resultType": "vector", 03:37:58 DEBUG "result": [ 03:37:58 DEBUG { 03:37:58 DEBUG "metric": { 03:37:58 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:37:58 DEBUG }, 03:37:58 DEBUG "value": [ 03:37:58 DEBUG 03:37:58 DEBUG 03:37:58 DEBUG 03:37:58 DEBUG [...] 03:37:58 DEBUG 03:37:58 DEBUG 03:37:58 DEBUG 03:37:58 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:37:58 DEBUG }, 03:37:58 DEBUG "value": [ 03:37:58 DEBUG 1713667078, 03:37:58 DEBUG "117014528" 03:37:58 DEBUG ] 03:37:58 DEBUG } 03:37:58 DEBUG ] 03:37:58 DEBUG } 03:37:58 DEBUG } 03:37:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713667079" 03:37:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667079" 03:37:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667079" 03:37:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713667079" 03:37:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713667079" 03:37:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667079" 03:37:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667079" 03:37:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667079" 03:37:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667079" 03:37:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713667079" 03:37:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667079" 03:37:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667079" 03:37:59 INFO [http_cmd]: http status code OK 03:37:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713667079" 03:37:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713667079" 03:37:59 INFO [http_cmd]: http status code OK 03:37:59 INFO [http_cmd]: http status code OK 03:37:59 INFO [http_cmd]: http status code OK 03:37:59 INFO [http_cmd]: http status code OK 03:37:59 INFO [http_cmd]: http status code OK 03:37:59 DEBUG --- status code --- 03:37:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:37:59 INFO [http_cmd]: http status code OK 03:37:59 DEBUG --- status code --- 03:37:59 DEBUG --- status code --- 03:37:59 DEBUG --- status code --- 03:37:59 INFO [http_cmd]: http status code OK 03:37:59 DEBUG --- status code --- 03:37:59 DEBUG --- status code --- 03:37:59 DEBUG --- http response --- 03:37:59 INFO [http_cmd]: http status code OK 03:37:59 DEBUG --- status code --- 03:37:59 INFO [http_cmd]: http status code OK 03:37:59 INFO [http_cmd]: http status code OK 03:37:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:37:59 INFO [http_cmd]: http status code OK 03:37:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:37:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:37:59 DEBUG --- status code --- 03:37:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:37:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:37:59 INFO [http_cmd]: http status code OK 03:37:59 DEBUG --- status code --- 03:37:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:37:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:37:59 DEBUG --- status code --- 03:37:59 DEBUG --- status code --- 03:37:59 DEBUG --- http response --- 03:37:59 DEBUG --- status code --- 03:37:59 DEBUG --- http response --- 03:37:59 DEBUG --- http response --- 03:37:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:37:59 DEBUG --- http response --- 03:37:59 DEBUG --- http response --- 03:37:59 DEBUG --- status code --- 03:37:59 INFO [http_cmd]: http status code OK 03:37:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:37:59 DEBUG ----- output ----- 03:37:59 DEBUG --- http response --- 03:37:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:37:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:37:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:37:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:37:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:37:59 DEBUG --- http response --- 03:37:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:37:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:37:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:37:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:37:59 DEBUG --- status code --- 03:37:59 DEBUG --- http response --- 03:37:59 DEBUG { 03:37:59 DEBUG --- http response --- 03:37:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:37:59 DEBUG --- http response --- 03:37:59 DEBUG ----- output ----- 03:37:59 DEBUG --- http response --- 03:37:59 DEBUG ----- output ----- 03:37:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:37:59 DEBUG ----- output ----- 03:37:59 DEBUG ----- output ----- 03:37:59 DEBUG --- http response --- 03:37:59 DEBUG ----- output ----- 03:37:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:37:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 03:37:59 DEBUG "status": "success", 03:37:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:37:59 DEBUG ----- output ----- 03:37:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:37:59 DEBUG { 03:37:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 03:37:59 DEBUG { 03:37:59 DEBUG ----- output ----- 03:37:59 DEBUG { 03:37:59 DEBUG { 03:37:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:37:59 DEBUG { 03:37:59 DEBUG --- http response --- 03:37:59 DEBUG ----- output ----- 03:37:59 DEBUG "data": { 03:37:59 DEBUG ----- output ----- 03:37:59 DEBUG { 03:37:59 DEBUG ----- output ----- 03:37:59 DEBUG "status": "success", 03:37:59 DEBUG ----- output ----- 03:37:59 DEBUG "status": "success", 03:37:59 DEBUG { 03:37:59 DEBUG "status": "success", 03:37:59 DEBUG "status": "success", 03:37:59 DEBUG ----- output ----- 03:37:59 DEBUG "status": "success", 03:37:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:37:59 DEBUG { 03:37:59 DEBUG "resultType": "vector", 03:37:59 DEBUG { 03:37:59 DEBUG "status": "success", 03:37:59 DEBUG { 03:37:59 DEBUG "data": { 03:37:59 DEBUG { 03:37:59 DEBUG "data": { 03:37:59 DEBUG "status": "success", 03:37:59 DEBUG "data": { 03:37:59 DEBUG "data": { 03:37:59 DEBUG { 03:37:59 DEBUG "data": { 03:37:59 DEBUG ----- output ----- 03:37:59 DEBUG "status": "success", 03:37:59 DEBUG "result": [ 03:37:59 DEBUG "status": "success", 03:37:59 DEBUG "data": { 03:37:59 DEBUG "status": "success", 03:37:59 DEBUG "resultType": "vector", 03:37:59 DEBUG "result": [ 03:37:59 DEBUG "status": "success", 03:37:59 DEBUG "data": { 03:37:59 DEBUG "resultType": "vector", 03:37:59 DEBUG "resultType": "vector", 03:37:59 DEBUG "status": "success", 03:37:59 DEBUG "resultType": "vector", 03:37:59 DEBUG { 03:37:59 DEBUG "data": { 03:37:59 DEBUG { 03:37:59 DEBUG "data": { 03:37:59 DEBUG "resultType": "vector", 03:37:59 DEBUG "data": { 03:37:59 DEBUG { 03:37:59 DEBUG "data": { 03:37:59 DEBUG "resultType": "vector", 03:37:59 DEBUG "resultType": "vector", 03:37:59 DEBUG "result": [ 03:37:59 DEBUG "result": [ 03:37:59 DEBUG "data": { 03:37:59 DEBUG "result": [ 03:37:59 DEBUG "status": "success", 03:37:59 DEBUG "resultType": "vector", 03:37:59 DEBUG "metric": { 03:37:59 DEBUG "resultType": "vector", 03:37:59 DEBUG "result": [ 03:37:59 DEBUG "resultType": "vector", 03:37:59 DEBUG "metric": { 03:37:59 DEBUG "resultType": "vector", 03:37:59 DEBUG "result": [ 03:37:59 DEBUG "result": [ 03:37:59 DEBUG { 03:37:59 DEBUG { 03:37:59 DEBUG "resultType": "vector", 03:37:59 DEBUG { 03:37:59 DEBUG "data": { 03:37:59 DEBUG "result": [ 03:37:59 DEBUG "instance": "10.142.0.107:9100" 03:37:59 DEBUG "result": [ 03:37:59 DEBUG { 03:37:59 DEBUG "result": [ 03:37:59 DEBUG "pod": "am-767f6b67b5-9zw22" 03:37:59 DEBUG "result": [ 03:37:59 DEBUG { 03:37:59 DEBUG { 03:37:59 DEBUG "metric": { 03:37:59 DEBUG "metric": { 03:37:59 DEBUG "result": [ 03:37:59 DEBUG "metric": { 03:37:59 DEBUG "resultType": "vector", 03:37:59 DEBUG { 03:37:59 DEBUG }, 03:37:59 DEBUG { 03:37:59 DEBUG "metric": { 03:37:59 DEBUG { 03:37:59 DEBUG }, 03:37:59 DEBUG { 03:37:59 DEBUG "metric": { 03:37:59 DEBUG "metric": { 03:37:59 DEBUG "instance": "10.142.0.107:9100" 03:37:59 DEBUG "instance": "10.142.0.107:9100" 03:37:59 DEBUG { 03:37:59 DEBUG "pod": "am-767f6b67b5-9zw22" 03:37:59 DEBUG "result": [ 03:37:59 DEBUG { 03:37:59 DEBUG "value": [ 03:37:59 DEBUG "metric": { 03:37:59 DEBUG "pod": "ds-idrepo-0" 03:37:59 DEBUG "metric": { 03:37:59 DEBUG "value": [ 03:37:59 DEBUG "metric": { 03:37:59 DEBUG "pod": "ds-cts-0" 03:37:59 DEBUG "pod": "ds-idrepo-0" 03:37:59 DEBUG }, 03:37:59 DEBUG }, 03:37:59 DEBUG "metric": { 03:37:59 DEBUG }, 03:37:59 DEBUG "metric": { 03:37:59 DEBUG "metric": { 03:37:59 DEBUG 03:37:59 DEBUG "pod": "ds-cts-0" 03:37:59 DEBUG }, 03:37:59 DEBUG "pod": "ds-idrepo-0" 03:37:59 DEBUG 03:37:59 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:37:59 DEBUG }, 03:37:59 DEBUG }, 03:37:59 DEBUG "value": [ 03:37:59 DEBUG 03:37:59 DEBUG "pod": "am-767f6b67b5-9zw22" 03:37:59 DEBUG "value": [ 03:37:59 DEBUG "instance": "10.142.0.107:9100" 03:37:59 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:37:59 DEBUG 03:37:59 DEBUG }, 03:37:59 DEBUG "value": [ 03:37:59 DEBUG }, 03:37:59 DEBUG 03:37:59 DEBUG }, 03:37:59 DEBUG "value": [ 03:37:59 DEBUG "value": [ 03:37:59 DEBUG 03:37:59 DEBUG "value": [ 03:37:59 DEBUG }, 03:37:59 DEBUG 03:37:59 DEBUG }, 03:37:59 DEBUG }, 03:37:59 DEBUG 03:37:59 DEBUG "value": [ 03:37:59 DEBUG 03:37:59 DEBUG "value": [ 03:37:59 DEBUG 03:37:59 DEBUG "value": [ 03:37:59 DEBUG 03:37:59 DEBUG 03:37:59 DEBUG 03:37:59 DEBUG 03:37:59 DEBUG "value": [ 03:37:59 DEBUG 03:37:59 DEBUG "value": [ 03:37:59 DEBUG "value": [ 03:37:59 DEBUG [...] 03:37:59 DEBUG 03:37:59 DEBUG 03:37:59 DEBUG 03:37:59 DEBUG [...] 03:37:59 DEBUG 03:37:59 DEBUG 03:37:59 DEBUG 03:37:59 DEBUG [...] 03:37:59 DEBUG 03:37:59 DEBUG 03:37:59 DEBUG 03:37:59 DEBUG 03:37:59 DEBUG 03:37:59 DEBUG 03:37:59 DEBUG 03:37:59 DEBUG 03:37:59 DEBUG 03:37:59 DEBUG 03:37:59 DEBUG 03:37:59 DEBUG 03:37:59 DEBUG 03:37:59 DEBUG 03:37:59 DEBUG 03:37:59 DEBUG 03:37:59 DEBUG [...] 03:37:59 DEBUG 03:37:59 DEBUG 03:37:59 DEBUG 03:37:59 DEBUG 03:37:59 DEBUG [...] 03:37:59 DEBUG 03:37:59 DEBUG 03:37:59 DEBUG 03:37:59 DEBUG [...] 03:37:59 DEBUG [...] 03:37:59 DEBUG 03:37:59 DEBUG [...] 03:37:59 DEBUG 03:37:59 DEBUG 03:37:59 DEBUG 03:37:59 DEBUG 03:37:59 DEBUG 03:37:59 DEBUG [...] 03:37:59 DEBUG 03:37:59 DEBUG 03:37:59 DEBUG 03:37:59 DEBUG [...] 03:37:59 DEBUG 03:37:59 DEBUG 03:37:59 DEBUG 03:37:59 DEBUG 03:37:59 DEBUG [...] 03:37:59 DEBUG 03:37:59 DEBUG [...] 03:37:59 DEBUG [...] 03:37:59 DEBUG "instance": "10.142.0.43:9100" 03:37:59 DEBUG 03:37:59 DEBUG [...] 03:37:59 DEBUG 03:37:59 DEBUG "pod": "am-767f6b67b5-vbltz" 03:37:59 DEBUG 03:37:59 DEBUG 03:37:59 DEBUG 03:37:59 DEBUG "instance": "10.142.0.43:9100" 03:37:59 DEBUG 03:37:59 DEBUG 03:37:59 DEBUG 03:37:59 DEBUG 03:37:59 DEBUG 03:37:59 DEBUG }, 03:37:59 DEBUG 03:37:59 DEBUG 03:37:59 DEBUG "pod": "ds-idrepo-2" 03:37:59 DEBUG }, 03:37:59 DEBUG 03:37:59 DEBUG 03:37:59 DEBUG 03:37:59 DEBUG }, 03:37:59 DEBUG 03:37:59 DEBUG 03:37:59 DEBUG "pod": "am-767f6b67b5-vbltz" 03:37:59 DEBUG 03:37:59 DEBUG 03:37:59 DEBUG "value": [ 03:37:59 DEBUG 03:37:59 DEBUG 03:37:59 DEBUG }, 03:37:59 DEBUG "value": [ 03:37:59 DEBUG 03:37:59 DEBUG "pod": "ds-cts-2" 03:37:59 DEBUG "pod": "ds-cts-2" 03:37:59 DEBUG "value": [ 03:37:59 DEBUG "instance": "10.142.0.43:9100" 03:37:59 DEBUG 03:37:59 DEBUG }, 03:37:59 DEBUG 03:37:59 DEBUG 03:37:59 DEBUG 1713667079, 03:37:59 DEBUG "pod": "ds-cts-2" 03:37:59 DEBUG 03:37:59 DEBUG "value": [ 03:37:59 DEBUG 1713667079, 03:37:59 DEBUG "pod": "idm-b85f9d978-txpwv" 03:37:59 DEBUG }, 03:37:59 DEBUG }, 03:37:59 DEBUG 1713667079, 03:37:59 DEBUG "4484041632768" 03:37:59 DEBUG "pod": "am-767f6b67b5-vbltz" 03:37:59 DEBUG "value": [ 03:37:59 DEBUG "instance": "10.142.0.43:9100" 03:37:59 DEBUG "pod": "idm-b85f9d978-txpwv" 03:37:59 DEBUG "15594939.566666666" 03:37:59 DEBUG }, 03:37:59 DEBUG "pod": "ds-idrepo-2" 03:37:59 DEBUG 1713667079, 03:37:59 DEBUG "3194.0666666666666" 03:37:59 DEBUG }, 03:37:59 DEBUG "value": [ 03:37:59 DEBUG "value": [ 03:37:59 DEBUG ] 03:37:59 DEBUG }, 03:37:59 DEBUG }, 03:37:59 DEBUG 1713667079, 03:37:59 DEBUG }, 03:37:59 DEBUG }, 03:37:59 DEBUG ] 03:37:59 DEBUG "value": [ 03:37:59 DEBUG }, 03:37:59 DEBUG "0.06666666666666667" 03:37:59 DEBUG ] 03:37:59 DEBUG "value": [ 03:37:59 DEBUG 1713667079, 03:37:59 DEBUG 1713667079, 03:37:59 DEBUG } 03:37:59 DEBUG "value": [ 03:37:59 DEBUG "value": [ 03:37:59 DEBUG 1713667079, 03:37:59 DEBUG "value": [ 03:37:59 DEBUG "value": [ 03:37:59 DEBUG } 03:37:59 DEBUG 1713667079, 03:37:59 DEBUG "value": [ 03:37:59 DEBUG ] 03:37:59 DEBUG } 03:37:59 DEBUG 1713667079, 03:37:59 DEBUG "0" 03:37:59 DEBUG "0" 03:37:59 DEBUG ] 03:37:59 DEBUG 1713667079, 03:37:59 DEBUG "193.301" 03:37:59 DEBUG ] 03:37:59 DEBUG 1713667079, 03:37:59 DEBUG "0.05666666666665871" 03:37:59 DEBUG ] 03:37:59 DEBUG "30.8" 03:37:59 DEBUG 1713667079, 03:37:59 DEBUG } 03:37:59 DEBUG ] 03:37:59 DEBUG "0" 03:37:59 DEBUG ] 03:37:59 DEBUG ] 03:37:59 DEBUG } 03:37:59 DEBUG "15929727.5" 03:37:59 DEBUG } 03:37:59 DEBUG "0.03333333333333333" 03:37:59 DEBUG ] 03:37:59 DEBUG 1713667079, 03:37:59 DEBUG } 03:37:59 DEBUG ] 03:37:59 DEBUG "0.06666666666666667" 03:37:59 DEBUG ] 03:37:59 DEBUG } 03:37:59 DEBUG ] 03:37:59 DEBUG } 03:37:59 DEBUG } 03:37:59 DEBUG ] 03:37:59 DEBUG } 03:37:59 DEBUG ] 03:37:59 DEBUG ] 03:37:59 DEBUG ] 03:37:59 DEBUG "0" 03:37:59 DEBUG } 03:37:59 DEBUG } 03:37:59 DEBUG ] 03:37:59 DEBUG } 03:37:59 DEBUG } 03:37:59 DEBUG } 03:37:59 DEBUG ] 03:37:59 DEBUG } 03:37:59 DEBUG } 03:37:59 DEBUG } 03:37:59 DEBUG } 03:37:59 DEBUG } 03:37:59 DEBUG } 03:37:59 DEBUG ] 03:37:59 DEBUG ] 03:37:59 DEBUG } 03:37:59 DEBUG } 03:37:59 DEBUG ] 03:37:59 DEBUG } 03:37:59 DEBUG } 03:37:59 DEBUG ] 03:37:59 DEBUG ] 03:37:59 DEBUG } 03:37:59 DEBUG } 03:37:59 DEBUG } 03:37:59 DEBUG ] 03:37:59 DEBUG } 03:37:59 DEBUG } 03:37:59 DEBUG } 03:37:59 DEBUG } 03:37:59 DEBUG ] 03:37:59 DEBUG ] 03:37:59 DEBUG } 03:37:59 DEBUG } 03:37:59 DEBUG } 03:37:59 DEBUG } 03:37:59 DEBUG } 03:37:59 DEBUG } 03:37:59 DEBUG } 03:37:59 DEBUG } 03:37:59 DEBUG } 03:38:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667134" 03:38:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713667134" 03:38:54 INFO [http_cmd]: http status code OK 03:38:54 DEBUG --- status code --- 03:38:54 ERROR [http_cmd]: ERROR 03:38:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:38:54 ERROR --- status code --- 03:38:54 DEBUG --- http response --- 03:38:54 ERROR http status code is 400 (expected [200, 401, 403, 404]) 03:38:54 ERROR --- http response --- 03:38:54 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 03:38:54 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 03:38:54 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 03:38:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667136" 03:38:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713667136" 03:38:56 INFO [http_cmd]: http status code OK 03:38:56 INFO [http_cmd]: http status code OK 03:38:56 DEBUG --- status code --- 03:38:56 DEBUG --- status code --- 03:38:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:38:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:38:56 DEBUG --- http response --- 03:38:56 DEBUG --- http response --- 03:38:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:38:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:38:56 DEBUG ----- output ----- 03:38:56 DEBUG ----- output ----- 03:38:56 DEBUG { 03:38:56 DEBUG { 03:38:56 DEBUG "status": "success", 03:38:56 DEBUG "status": "success", 03:38:56 DEBUG "data": { 03:38:56 DEBUG "data": { 03:38:56 DEBUG "resultType": "vector", 03:38:56 DEBUG "resultType": "vector", 03:38:56 DEBUG "result": [ 03:38:56 DEBUG "result": [ 03:38:56 DEBUG { 03:38:56 DEBUG { 03:38:56 DEBUG "metric": { 03:38:56 DEBUG "metric": { 03:38:56 DEBUG "pod": "ds-cts-0" 03:38:56 DEBUG "instance": "10.142.0.107:9100" 03:38:56 DEBUG }, 03:38:56 DEBUG }, 03:38:56 DEBUG "value": [ 03:38:56 DEBUG "value": [ 03:38:56 DEBUG 03:38:56 DEBUG 03:38:56 DEBUG 03:38:56 DEBUG 03:38:56 DEBUG 03:38:56 DEBUG 03:38:56 DEBUG [...] 03:38:56 DEBUG [...] 03:38:56 DEBUG 03:38:56 DEBUG 03:38:56 DEBUG 03:38:56 DEBUG 03:38:56 DEBUG 03:38:56 DEBUG 03:38:56 DEBUG "pod": "ds-cts-2" 03:38:56 DEBUG "instance": "10.142.0.43:9100" 03:38:56 DEBUG }, 03:38:56 DEBUG }, 03:38:56 DEBUG "value": [ 03:38:56 DEBUG "value": [ 03:38:56 DEBUG 1713667136, 03:38:56 DEBUG 1713667136, 03:38:56 DEBUG "26.521966666666664" 03:38:56 DEBUG "0.4369333333333657" 03:38:56 DEBUG ] 03:38:56 DEBUG ] 03:38:56 DEBUG } 03:38:56 DEBUG } 03:38:56 DEBUG ] 03:38:56 DEBUG ] 03:38:56 DEBUG } 03:38:56 DEBUG } 03:38:56 DEBUG } 03:38:56 DEBUG } 03:38:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667136" 03:38:56 INFO [http_cmd]: http status code OK 03:38:56 DEBUG --- status code --- 03:38:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:38:56 DEBUG --- http response --- 03:38:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:38:56 DEBUG ----- output ----- 03:38:56 DEBUG { 03:38:56 DEBUG "status": "success", 03:38:56 DEBUG "data": { 03:38:56 DEBUG "resultType": "vector", 03:38:56 DEBUG "result": [ 03:38:56 DEBUG { 03:38:56 DEBUG "metric": { 03:38:56 DEBUG "pod": "am-767f6b67b5-9zw22" 03:38:56 DEBUG }, 03:38:56 DEBUG "value": [ 03:38:56 DEBUG 03:38:56 DEBUG 03:38:56 DEBUG 03:38:56 DEBUG [...] 03:38:56 DEBUG 03:38:56 DEBUG 03:38:56 DEBUG 03:38:56 DEBUG "pod": "am-767f6b67b5-vbltz" 03:38:56 DEBUG }, 03:38:56 DEBUG "value": [ 03:38:56 DEBUG 1713667136, 03:38:56 DEBUG "0" 03:38:56 DEBUG ] 03:38:56 DEBUG } 03:38:56 DEBUG ] 03:38:56 DEBUG } 03:38:56 DEBUG } 03:38:57 INFO 03:38:57 INFO [loop_until]: kubectl --namespace=xlou top pods 03:38:57 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:38:57 INFO [loop_until]: OK (rc = 0) 03:38:57 DEBUG --- stdout --- 03:38:57 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 2466m 8729Mi am-767f6b67b5-qcqz2 2505m 9069Mi am-767f6b67b5-vbltz 2778m 8784Mi ds-cts-0 18168m 12497Mi ds-cts-1 17600m 12466Mi ds-cts-2 18593m 12549Mi ds-idrepo-0 1081m 4063Mi ds-idrepo-1 703m 3966Mi ds-idrepo-2 746m 3937Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1180Mi idm-b85f9d978-wnhxb 3m 1239Mi lodemon-54c87f8cbb-jzrjs 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-68c5d5b884-p97w5 1016m 1035Mi 03:38:57 DEBUG --- stderr --- 03:38:57 DEBUG 03:38:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667138" 03:38:58 INFO [http_cmd]: http status code OK 03:38:58 DEBUG --- status code --- 03:38:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:38:58 DEBUG --- http response --- 03:38:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:38:58 DEBUG ----- output ----- 03:38:58 DEBUG { 03:38:58 DEBUG "status": "success", 03:38:58 DEBUG "data": { 03:38:58 DEBUG "resultType": "vector", 03:38:58 DEBUG "result": [ 03:38:58 DEBUG { 03:38:58 DEBUG "metric": { 03:38:58 DEBUG "pod": "am-767f6b67b5-9zw22" 03:38:58 DEBUG }, 03:38:58 DEBUG "value": [ 03:38:58 DEBUG 03:38:58 DEBUG 03:38:58 DEBUG 03:38:58 DEBUG [...] 03:38:58 DEBUG 03:38:58 DEBUG 03:38:58 DEBUG 03:38:58 DEBUG "pod": "am-767f6b67b5-vbltz" 03:38:58 DEBUG }, 03:38:58 DEBUG "value": [ 03:38:58 DEBUG 1713667138, 03:38:58 DEBUG "0" 03:38:58 DEBUG ] 03:38:58 DEBUG } 03:38:58 DEBUG ] 03:38:58 DEBUG } 03:38:58 DEBUG } 03:38:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667138" 03:38:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667138" 03:38:58 INFO [http_cmd]: http status code OK 03:38:58 DEBUG --- status code --- 03:38:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:38:58 DEBUG --- http response --- 03:38:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 03:38:58 DEBUG ----- output ----- 03:38:58 DEBUG { 03:38:58 DEBUG "status": "success", 03:38:58 DEBUG "data": { 03:38:58 DEBUG "resultType": "vector", 03:38:58 DEBUG "result": [ 03:38:58 DEBUG { 03:38:58 DEBUG "metric": { 03:38:58 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:38:58 DEBUG }, 03:38:58 DEBUG "value": [ 03:38:58 DEBUG 03:38:58 DEBUG 03:38:58 DEBUG 03:38:58 DEBUG [...] 03:38:58 DEBUG 03:38:58 DEBUG 03:38:58 DEBUG 03:38:58 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 03:38:58 DEBUG }, 03:38:58 DEBUG "value": [ 03:38:58 DEBUG 1713667138, 03:38:58 DEBUG "0" 03:38:58 DEBUG ] 03:38:58 DEBUG } 03:38:58 DEBUG ] 03:38:58 DEBUG } 03:38:58 DEBUG } 03:38:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667138" 03:38:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667138" 03:38:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713667138" 03:38:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667138" 03:38:58 INFO [http_cmd]: http status code OK 03:38:58 DEBUG --- status code --- 03:38:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:38:58 DEBUG --- http response --- 03:38:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:38:58 DEBUG ----- output ----- 03:38:58 DEBUG { 03:38:58 DEBUG "status": "success", 03:38:58 DEBUG "data": { 03:38:58 DEBUG "resultType": "vector", 03:38:58 DEBUG "result": [ 03:38:58 DEBUG { 03:38:58 DEBUG "metric": { 03:38:58 DEBUG "pod": "am-767f6b67b5-9zw22" 03:38:58 DEBUG }, 03:38:58 DEBUG "value": [ 03:38:58 DEBUG 03:38:58 DEBUG 03:38:58 DEBUG 03:38:58 DEBUG [...] 03:38:58 DEBUG 03:38:58 DEBUG 03:38:58 DEBUG 03:38:58 DEBUG "pod": "am-767f6b67b5-vbltz" 03:38:58 DEBUG }, 03:38:58 DEBUG "value": [ 03:38:58 DEBUG 1713667138, 03:38:58 DEBUG "104686" 03:38:58 DEBUG ] 03:38:58 DEBUG } 03:38:58 DEBUG ] 03:38:58 DEBUG } 03:38:58 DEBUG } 03:38:58 INFO [http_cmd]: http status code OK 03:38:58 DEBUG --- status code --- 03:38:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:38:58 DEBUG --- http response --- 03:38:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:38:58 DEBUG ----- output ----- 03:38:58 DEBUG { 03:38:58 DEBUG "status": "success", 03:38:58 DEBUG "data": { 03:38:58 DEBUG "resultType": "vector", 03:38:58 DEBUG "result": [ 03:38:58 DEBUG { 03:38:58 DEBUG "metric": { 03:38:58 DEBUG "pod": "ds-idrepo-0" 03:38:58 DEBUG }, 03:38:58 DEBUG "value": [ 03:38:58 DEBUG 03:38:58 DEBUG 03:38:58 DEBUG 03:38:58 DEBUG [...] 03:38:58 DEBUG 03:38:58 DEBUG 03:38:58 DEBUG 03:38:58 DEBUG "pod": "ds-idrepo-2" 03:38:58 DEBUG }, 03:38:58 DEBUG "value": [ 03:38:58 DEBUG 1713667138, 03:38:58 DEBUG "0.06666666666666667" 03:38:58 DEBUG ] 03:38:58 DEBUG } 03:38:58 DEBUG ] 03:38:58 DEBUG } 03:38:58 DEBUG } 03:38:58 INFO [http_cmd]: http status code OK 03:38:58 INFO [http_cmd]: http status code OK 03:38:58 DEBUG --- status code --- 03:38:58 DEBUG --- status code --- 03:38:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:38:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:38:58 DEBUG --- http response --- 03:38:58 DEBUG --- http response --- 03:38:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:38:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 03:38:58 DEBUG ----- output ----- 03:38:58 DEBUG ----- output ----- 03:38:58 DEBUG { 03:38:58 DEBUG { 03:38:58 DEBUG "status": "success", 03:38:58 DEBUG "status": "success", 03:38:58 DEBUG "data": { 03:38:58 DEBUG "data": { 03:38:58 DEBUG "resultType": "vector", 03:38:58 DEBUG "resultType": "vector", 03:38:58 DEBUG "result": [ 03:38:58 DEBUG "result": [ 03:38:58 DEBUG { 03:38:58 DEBUG { 03:38:58 DEBUG "metric": { 03:38:58 DEBUG "metric": { 03:38:58 DEBUG "pod": "am-767f6b67b5-9zw22" 03:38:58 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:38:58 DEBUG }, 03:38:58 DEBUG "value": [ 03:38:58 DEBUG 03:38:58 DEBUG 03:38:58 DEBUG 03:38:58 DEBUG [...] 03:38:58 DEBUG 03:38:58 DEBUG 03:38:58 DEBUG 03:38:58 DEBUG "pod": "login-ui-5fdf49fc66-dlhjd" 03:38:58 DEBUG }, 03:38:58 DEBUG }, 03:38:58 DEBUG "value": [ 03:38:58 DEBUG "value": [ 03:38:58 DEBUG 03:38:58 DEBUG 1713667138, 03:38:58 DEBUG 03:38:58 DEBUG "0.00002105754173192983" 03:38:58 DEBUG 03:38:58 DEBUG ] 03:38:58 DEBUG [...] 03:38:58 DEBUG } 03:38:58 DEBUG 03:38:58 INFO [http_cmd]: http status code OK 03:38:58 DEBUG ] 03:38:58 DEBUG 03:38:58 DEBUG --- status code --- 03:38:58 DEBUG } 03:38:58 DEBUG 03:38:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:38:58 DEBUG } 03:38:58 DEBUG "pod": "am-767f6b67b5-vbltz" 03:38:58 DEBUG --- http response --- 03:38:58 DEBUG }, 03:38:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:38:58 DEBUG "value": [ 03:38:58 DEBUG ----- output ----- 03:38:58 DEBUG 1713667138, 03:38:58 DEBUG { 03:38:58 DEBUG "0" 03:38:58 DEBUG "status": "success", 03:38:58 DEBUG ] 03:38:58 DEBUG "data": { 03:38:58 DEBUG } 03:38:58 DEBUG "resultType": "vector", 03:38:58 DEBUG ] 03:38:58 DEBUG "result": [ 03:38:58 DEBUG } 03:38:58 DEBUG { 03:38:58 DEBUG } 03:38:58 DEBUG "metric": { 03:38:58 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:38:58 DEBUG }, 03:38:58 DEBUG "value": [ 03:38:58 DEBUG 03:38:58 DEBUG 03:38:58 DEBUG 03:38:58 DEBUG [...] 03:38:58 DEBUG 03:38:58 DEBUG 03:38:58 DEBUG 03:38:58 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:38:58 DEBUG }, 03:38:58 DEBUG "value": [ 03:38:58 DEBUG 1713667138, 03:38:58 DEBUG "1.088840383217362" 03:38:58 DEBUG ] 03:38:58 DEBUG } 03:38:58 DEBUG ] 03:38:58 DEBUG } 03:38:58 DEBUG } 03:38:58 INFO 03:38:58 INFO [loop_until]: kubectl --namespace=xlou top node 03:38:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:38:58 INFO [loop_until]: OK (rc = 0) 03:38:58 DEBUG --- stdout --- 03:38:58 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 66m 0% 6277Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 985m 3% 7772Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 2564m 8% 11578Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 83m 0% 3351Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 18399m 61% 15814Mi 14% gke-xlou-cdm-default-pool-1b0769c8-wpwl 154m 0% 5235Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 2645m 8% 12027Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 75m 0% 3729Mi 3% gke-xlou-cdm-ds-0f601751-hmm7 2985m 9% 11074Mi 9% gke-xlou-cdm-ds-0f601751-j488 775m 2% 7370Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 17782m 59% 17592Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 895m 2% 8009Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 19012m 63% 15252Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 1079m 6% 11394Mi 19% 03:38:58 DEBUG --- stderr --- 03:38:58 DEBUG 03:38:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713667138" 03:38:58 INFO [http_cmd]: http status code OK 03:38:58 DEBUG --- status code --- 03:38:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:38:58 DEBUG --- http response --- 03:38:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:38:58 DEBUG ----- output ----- 03:38:58 DEBUG { 03:38:58 DEBUG "status": "success", 03:38:58 DEBUG "data": { 03:38:58 DEBUG "resultType": "vector", 03:38:58 DEBUG "result": [ 03:38:58 DEBUG { 03:38:58 DEBUG "metric": { 03:38:58 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:38:58 DEBUG }, 03:38:58 DEBUG "value": [ 03:38:58 DEBUG 03:38:58 DEBUG 03:38:58 DEBUG 03:38:58 DEBUG [...] 03:38:58 DEBUG 03:38:58 DEBUG 03:38:58 DEBUG 03:38:58 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:38:58 DEBUG }, 03:38:58 DEBUG "value": [ 03:38:58 DEBUG 1713667138, 03:38:58 DEBUG "829513728" 03:38:58 DEBUG ] 03:38:58 DEBUG } 03:38:58 DEBUG ] 03:38:58 DEBUG } 03:38:58 DEBUG } 03:38:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713667139" 03:38:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667139" 03:38:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713667139" 03:38:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713667139" 03:38:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713667139" 03:38:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667139" 03:38:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667139" 03:38:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667139" 03:38:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667139" 03:38:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667139" 03:38:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713667139" 03:38:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667139" 03:38:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667139" 03:38:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713667139" 03:38:59 INFO [http_cmd]: http status code OK 03:38:59 INFO [http_cmd]: http status code OK 03:38:59 INFO [http_cmd]: http status code OK 03:38:59 INFO [http_cmd]: http status code OK 03:38:59 INFO [http_cmd]: http status code OK 03:38:59 INFO [http_cmd]: http status code OK 03:38:59 INFO [http_cmd]: http status code OK 03:38:59 DEBUG --- status code --- 03:38:59 INFO [http_cmd]: http status code OK 03:38:59 INFO [http_cmd]: http status code OK 03:38:59 DEBUG --- status code --- 03:38:59 DEBUG --- status code --- 03:38:59 DEBUG --- status code --- 03:38:59 DEBUG --- status code --- 03:38:59 DEBUG --- status code --- 03:38:59 INFO [http_cmd]: http status code OK 03:38:59 INFO [http_cmd]: http status code OK 03:38:59 INFO [http_cmd]: http status code OK 03:38:59 DEBUG --- status code --- 03:38:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:38:59 DEBUG --- status code --- 03:38:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:38:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:38:59 DEBUG --- http response --- 03:38:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:38:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:38:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:38:59 DEBUG --- status code --- 03:38:59 DEBUG --- status code --- 03:38:59 DEBUG --- status code --- 03:38:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:38:59 INFO [http_cmd]: http status code OK 03:38:59 DEBUG --- http response --- 03:38:59 DEBUG --- status code --- 03:38:59 DEBUG --- http response --- 03:38:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:38:59 DEBUG --- http response --- 03:38:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:38:59 DEBUG --- http response --- 03:38:59 DEBUG --- http response --- 03:38:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:38:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:38:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:38:59 DEBUG --- http response --- 03:38:59 DEBUG --- status code --- 03:38:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:38:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:38:59 DEBUG --- http response --- 03:38:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:38:59 INFO [http_cmd]: http status code OK 03:38:59 DEBUG --- status code --- 03:38:59 DEBUG ----- output ----- 03:38:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:38:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:38:59 DEBUG --- http response --- 03:38:59 DEBUG --- http response --- 03:38:59 DEBUG --- http response --- 03:38:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:38:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:38:59 DEBUG ----- output ----- 03:38:59 DEBUG { 03:38:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:38:59 DEBUG ----- output ----- 03:38:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:38:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:38:59 DEBUG { 03:38:59 DEBUG ----- output ----- 03:38:59 DEBUG ----- output ----- 03:38:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:38:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:38:59 DEBUG --- http response --- 03:38:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:38:59 DEBUG ----- output ----- 03:38:59 DEBUG --- http response --- 03:38:59 DEBUG "status": "success", 03:38:59 DEBUG ----- output ----- 03:38:59 DEBUG { 03:38:59 DEBUG ----- output ----- 03:38:59 DEBUG --- http response --- 03:38:59 DEBUG "status": "success", 03:38:59 DEBUG { 03:38:59 DEBUG { 03:38:59 DEBUG ----- output ----- 03:38:59 DEBUG ----- output ----- 03:38:59 DEBUG ----- output ----- 03:38:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:38:59 DEBUG { 03:38:59 DEBUG "data": { 03:38:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:38:59 DEBUG { 03:38:59 DEBUG "status": "success", 03:38:59 DEBUG { 03:38:59 DEBUG "data": { 03:38:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:38:59 DEBUG "status": "success", 03:38:59 DEBUG "status": "success", 03:38:59 DEBUG { 03:38:59 DEBUG { 03:38:59 DEBUG { 03:38:59 DEBUG ----- output ----- 03:38:59 DEBUG "status": "success", 03:38:59 DEBUG "resultType": "vector", 03:38:59 DEBUG ----- output ----- 03:38:59 DEBUG "status": "success", 03:38:59 DEBUG "data": { 03:38:59 DEBUG "status": "success", 03:38:59 DEBUG "resultType": "vector", 03:38:59 DEBUG ----- output ----- 03:38:59 DEBUG "data": { 03:38:59 DEBUG "data": { 03:38:59 DEBUG "status": "success", 03:38:59 DEBUG "status": "success", 03:38:59 DEBUG "status": "success", 03:38:59 DEBUG { 03:38:59 DEBUG "data": { 03:38:59 DEBUG "result": [ 03:38:59 DEBUG { 03:38:59 DEBUG "data": { 03:38:59 DEBUG "resultType": "vector", 03:38:59 DEBUG "data": { 03:38:59 DEBUG "result": [ 03:38:59 DEBUG { 03:38:59 DEBUG "resultType": "vector", 03:38:59 DEBUG "resultType": "vector", 03:38:59 DEBUG "data": { 03:38:59 DEBUG "data": { 03:38:59 DEBUG "data": { 03:38:59 DEBUG "status": "success", 03:38:59 DEBUG "resultType": "vector", 03:38:59 DEBUG { 03:38:59 DEBUG "status": "success", 03:38:59 DEBUG "resultType": "vector", 03:38:59 DEBUG "result": [ 03:38:59 DEBUG "resultType": "vector", 03:38:59 DEBUG { 03:38:59 DEBUG "status": "success", 03:38:59 DEBUG "result": [ 03:38:59 DEBUG "result": [ 03:38:59 DEBUG "resultType": "vector", 03:38:59 DEBUG "resultType": "vector", 03:38:59 DEBUG "resultType": "vector", 03:38:59 DEBUG "data": { 03:38:59 DEBUG "result": [ 03:38:59 DEBUG "metric": { 03:38:59 DEBUG "data": { 03:38:59 DEBUG "result": [ 03:38:59 DEBUG { 03:38:59 DEBUG "result": [ 03:38:59 DEBUG "metric": { 03:38:59 DEBUG "data": { 03:38:59 DEBUG { 03:38:59 DEBUG { 03:38:59 DEBUG "result": [ 03:38:59 DEBUG "result": [ 03:38:59 DEBUG "result": [ 03:38:59 DEBUG "resultType": "vector", 03:38:59 DEBUG { 03:38:59 DEBUG "instance": "10.142.0.107:9100" 03:38:59 DEBUG "resultType": "vector", 03:38:59 DEBUG { 03:38:59 DEBUG "metric": { 03:38:59 DEBUG { 03:38:59 DEBUG "pod": "ds-idrepo-0" 03:38:59 DEBUG "resultType": "vector", 03:38:59 DEBUG "metric": { 03:38:59 DEBUG "metric": { 03:38:59 DEBUG { 03:38:59 DEBUG { 03:38:59 DEBUG { 03:38:59 DEBUG "result": [ 03:38:59 DEBUG "metric": { 03:38:59 DEBUG }, 03:38:59 DEBUG "result": [ 03:38:59 DEBUG "metric": { 03:38:59 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:38:59 DEBUG "metric": { 03:38:59 DEBUG }, 03:38:59 DEBUG "result": [ 03:38:59 DEBUG "instance": "10.142.0.107:9100" 03:38:59 DEBUG "pod": "ds-cts-0" 03:38:59 DEBUG "metric": { 03:38:59 DEBUG "metric": { 03:38:59 DEBUG "metric": { 03:38:59 DEBUG { 03:38:59 DEBUG "pod": "am-767f6b67b5-9zw22" 03:38:59 DEBUG "value": [ 03:38:59 DEBUG { 03:38:59 DEBUG "pod": "ds-idrepo-0" 03:38:59 DEBUG }, 03:38:59 DEBUG "pod": "am-767f6b67b5-9zw22" 03:38:59 DEBUG "value": [ 03:38:59 DEBUG { 03:38:59 DEBUG }, 03:38:59 DEBUG }, 03:38:59 DEBUG "pod": "ds-idrepo-0" 03:38:59 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:38:59 DEBUG "pod": "ds-cts-0" 03:38:59 DEBUG "metric": { 03:38:59 DEBUG }, 03:38:59 DEBUG "value": [ 03:38:59 DEBUG "metric": { 03:38:59 DEBUG }, 03:38:59 DEBUG "value": [ 03:38:59 DEBUG }, 03:38:59 DEBUG 03:38:59 DEBUG "metric": { 03:38:59 DEBUG "value": [ 03:38:59 DEBUG "value": [ 03:38:59 DEBUG }, 03:38:59 DEBUG }, 03:38:59 DEBUG }, 03:38:59 DEBUG "instance": "10.142.0.107:9100" 03:38:59 DEBUG 03:38:59 DEBUG 03:38:59 DEBUG "pod": "am-767f6b67b5-9zw22" 03:38:59 DEBUG "value": [ 03:38:59 DEBUG 03:38:59 DEBUG "value": [ 03:38:59 DEBUG 03:38:59 DEBUG "instance": "10.142.0.107:9100" 03:38:59 DEBUG 03:38:59 DEBUG 03:38:59 DEBUG "value": [ 03:38:59 DEBUG "value": [ 03:38:59 DEBUG "value": [ 03:38:59 DEBUG }, 03:38:59 DEBUG 03:38:59 DEBUG 03:38:59 DEBUG }, 03:38:59 DEBUG 03:38:59 DEBUG 03:38:59 DEBUG 03:38:59 DEBUG 03:38:59 DEBUG }, 03:38:59 DEBUG 03:38:59 DEBUG 03:38:59 DEBUG 03:38:59 DEBUG 03:38:59 DEBUG 03:38:59 DEBUG "value": [ 03:38:59 DEBUG 03:38:59 DEBUG 03:38:59 DEBUG "value": [ 03:38:59 DEBUG 03:38:59 DEBUG 03:38:59 DEBUG 03:38:59 DEBUG [...] 03:38:59 DEBUG "value": [ 03:38:59 DEBUG 03:38:59 DEBUG 03:38:59 DEBUG 03:38:59 DEBUG 03:38:59 DEBUG 03:38:59 DEBUG 03:38:59 DEBUG 03:38:59 DEBUG 03:38:59 DEBUG 03:38:59 DEBUG 03:38:59 DEBUG [...] 03:38:59 DEBUG 03:38:59 DEBUG 03:38:59 DEBUG 03:38:59 DEBUG 03:38:59 DEBUG [...] 03:38:59 DEBUG 03:38:59 DEBUG 03:38:59 DEBUG 03:38:59 DEBUG [...] 03:38:59 DEBUG [...] 03:38:59 DEBUG [...] 03:38:59 DEBUG 03:38:59 DEBUG [...] 03:38:59 DEBUG 03:38:59 DEBUG [...] 03:38:59 DEBUG 03:38:59 DEBUG 03:38:59 DEBUG [...] 03:38:59 DEBUG 03:38:59 DEBUG [...] 03:38:59 DEBUG [...] 03:38:59 DEBUG [...] 03:38:59 DEBUG 03:38:59 DEBUG 03:38:59 DEBUG 03:38:59 DEBUG 03:38:59 DEBUG 03:38:59 DEBUG 03:38:59 DEBUG 03:38:59 DEBUG 03:38:59 DEBUG [...] 03:38:59 DEBUG 03:38:59 DEBUG 03:38:59 DEBUG 03:38:59 DEBUG 03:38:59 DEBUG 03:38:59 DEBUG 03:38:59 DEBUG 03:38:59 DEBUG 03:38:59 DEBUG [...] 03:38:59 DEBUG 03:38:59 DEBUG 03:38:59 DEBUG 03:38:59 DEBUG "pod": "ds-cts-2" 03:38:59 DEBUG 03:38:59 DEBUG 03:38:59 DEBUG 03:38:59 DEBUG 03:38:59 DEBUG 03:38:59 DEBUG 03:38:59 DEBUG 03:38:59 DEBUG 03:38:59 DEBUG 03:38:59 DEBUG 03:38:59 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:38:59 DEBUG 03:38:59 DEBUG 03:38:59 DEBUG }, 03:38:59 DEBUG 03:38:59 DEBUG 03:38:59 DEBUG "pod": "ds-cts-2" 03:38:59 DEBUG "pod": "ds-idrepo-2" 03:38:59 DEBUG 03:38:59 DEBUG 03:38:59 DEBUG "instance": "10.142.0.43:9100" 03:38:59 DEBUG "instance": "10.142.0.43:9100" 03:38:59 DEBUG "pod": "am-767f6b67b5-vbltz" 03:38:59 DEBUG 03:38:59 DEBUG }, 03:38:59 DEBUG "pod": "ds-idrepo-2" 03:38:59 DEBUG "pod": "am-767f6b67b5-vbltz" 03:38:59 DEBUG "value": [ 03:38:59 DEBUG 03:38:59 DEBUG "instance": "10.142.0.43:9100" 03:38:59 DEBUG }, 03:38:59 DEBUG }, 03:38:59 DEBUG 03:38:59 DEBUG "pod": "ds-cts-2" 03:38:59 DEBUG }, 03:38:59 DEBUG }, 03:38:59 DEBUG }, 03:38:59 DEBUG 03:38:59 DEBUG "value": [ 03:38:59 DEBUG }, 03:38:59 DEBUG }, 03:38:59 DEBUG 1713667139, 03:38:59 DEBUG "instance": "10.142.0.43:9100" 03:38:59 DEBUG }, 03:38:59 DEBUG "value": [ 03:38:59 DEBUG "value": [ 03:38:59 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:38:59 DEBUG }, 03:38:59 DEBUG "value": [ 03:38:59 DEBUG "value": [ 03:38:59 DEBUG "value": [ 03:38:59 DEBUG "pod": "am-767f6b67b5-vbltz" 03:38:59 DEBUG 1713667139, 03:38:59 DEBUG "value": [ 03:38:59 DEBUG "value": [ 03:38:59 DEBUG "0" 03:38:59 DEBUG }, 03:38:59 DEBUG "value": [ 03:38:59 DEBUG 1713667139, 03:38:59 DEBUG 1713667139, 03:38:59 DEBUG }, 03:38:59 DEBUG "value": [ 03:38:59 DEBUG 1713667139, 03:38:59 DEBUG 1713667139, 03:38:59 DEBUG 1713667139, 03:38:59 DEBUG }, 03:38:59 DEBUG "8769596.219480177" 03:38:59 DEBUG 1713667139, 03:38:59 DEBUG 1713667139, 03:38:59 DEBUG ] 03:38:59 DEBUG "value": [ 03:38:59 DEBUG 1713667139, 03:38:59 DEBUG "0" 03:38:59 DEBUG "0.06666666666666667" 03:38:59 DEBUG "value": [ 03:38:59 DEBUG 1713667139, 03:38:59 DEBUG "16547767.833333332" 03:38:59 DEBUG "0.04766666666666122" 03:38:59 DEBUG "2592.5666666666666" 03:38:59 DEBUG "value": [ 03:38:59 DEBUG ] 03:38:59 DEBUG } 03:38:59 DEBUG "195.3665" 03:38:59 DEBUG } 03:38:59 DEBUG 1713667139, 03:38:59 DEBUG "4487003046912" 03:38:59 DEBUG ] 03:38:59 DEBUG ] 03:38:59 DEBUG 1713667139, 03:38:59 DEBUG "28.733333333333334" 03:38:59 DEBUG ] 03:38:59 DEBUG ] 03:38:59 DEBUG ] 03:38:59 DEBUG 1713667139, 03:38:59 DEBUG ] 03:38:59 DEBUG "0.06666666666666667" 03:38:59 DEBUG ] 03:38:59 DEBUG ] 03:38:59 DEBUG "16554846.233333334" 03:38:59 DEBUG ] 03:38:59 DEBUG } 03:38:59 DEBUG } 03:38:59 DEBUG "0" 03:38:59 DEBUG ] 03:38:59 DEBUG } 03:38:59 DEBUG } 03:38:59 DEBUG } 03:38:59 DEBUG "0" 03:38:59 DEBUG } 03:38:59 DEBUG ] 03:38:59 DEBUG } 03:38:59 DEBUG } 03:38:59 DEBUG ] 03:38:59 DEBUG } 03:38:59 DEBUG ] 03:38:59 DEBUG ] 03:38:59 DEBUG ] 03:38:59 DEBUG } 03:38:59 DEBUG ] 03:38:59 DEBUG ] 03:38:59 DEBUG ] 03:38:59 DEBUG ] 03:38:59 DEBUG } 03:38:59 DEBUG } 03:38:59 DEBUG ] 03:38:59 DEBUG } 03:38:59 DEBUG } 03:38:59 DEBUG ] 03:38:59 DEBUG } 03:38:59 DEBUG } 03:38:59 DEBUG } 03:38:59 DEBUG ] 03:38:59 DEBUG } 03:38:59 DEBUG } 03:38:59 DEBUG } 03:38:59 DEBUG } 03:38:59 DEBUG ] 03:38:59 DEBUG } 03:38:59 DEBUG ] 03:38:59 DEBUG } 03:38:59 DEBUG } 03:38:59 DEBUG } 03:38:59 DEBUG ] 03:38:59 DEBUG } 03:38:59 DEBUG } 03:38:59 DEBUG } 03:38:59 DEBUG } 03:38:59 DEBUG ] 03:38:59 DEBUG } 03:38:59 DEBUG } 03:38:59 DEBUG } 03:38:59 DEBUG } 03:38:59 DEBUG } 03:38:59 DEBUG } 03:38:59 DEBUG } 03:38:59 DEBUG } 03:38:59 DEBUG } 03:38:59 DEBUG } 03:38:59 DEBUG } 03:39:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713667194" 03:39:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667194" 03:39:54 ERROR [http_cmd]: ERROR 03:39:54 INFO [http_cmd]: http status code OK 03:39:54 ERROR --- status code --- 03:39:54 DEBUG --- status code --- 03:39:54 ERROR http status code is 400 (expected [200, 401, 403, 404]) 03:39:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:39:54 ERROR --- http response --- 03:39:54 DEBUG --- http response --- 03:39:54 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 03:39:54 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 03:39:54 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 03:39:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667196" 03:39:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713667196" 03:39:56 INFO [http_cmd]: http status code OK 03:39:56 INFO [http_cmd]: http status code OK 03:39:56 DEBUG --- status code --- 03:39:56 DEBUG --- status code --- 03:39:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:39:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:39:56 DEBUG --- http response --- 03:39:56 DEBUG --- http response --- 03:39:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:39:56 DEBUG ----- output ----- 03:39:56 DEBUG { 03:39:56 DEBUG "status": "success", 03:39:56 DEBUG "data": { 03:39:56 DEBUG "resultType": "vector", 03:39:56 DEBUG "result": [ 03:39:56 DEBUG { 03:39:56 DEBUG "metric": { 03:39:56 DEBUG "pod": "ds-cts-0" 03:39:56 DEBUG }, 03:39:56 DEBUG "value": [ 03:39:56 DEBUG 03:39:56 DEBUG 03:39:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:39:56 DEBUG 03:39:56 DEBUG ----- output ----- 03:39:56 DEBUG [...] 03:39:56 DEBUG { 03:39:56 DEBUG 03:39:56 DEBUG "status": "success", 03:39:56 DEBUG 03:39:56 DEBUG "data": { 03:39:56 DEBUG 03:39:56 DEBUG "resultType": "vector", 03:39:56 DEBUG "pod": "ds-cts-2" 03:39:56 DEBUG "result": [ 03:39:56 DEBUG }, 03:39:56 DEBUG { 03:39:56 DEBUG "value": [ 03:39:56 DEBUG "metric": { 03:39:56 DEBUG 1713667196, 03:39:56 DEBUG "instance": "10.142.0.107:9100" 03:39:56 DEBUG "23.84153333333333" 03:39:56 DEBUG }, 03:39:56 DEBUG ] 03:39:56 DEBUG "value": [ 03:39:56 DEBUG } 03:39:56 DEBUG 03:39:56 DEBUG ] 03:39:56 DEBUG 03:39:56 DEBUG } 03:39:56 DEBUG 03:39:56 DEBUG } 03:39:56 DEBUG [...] 03:39:56 DEBUG 03:39:56 DEBUG 03:39:56 DEBUG 03:39:56 DEBUG "instance": "10.142.0.43:9100" 03:39:56 DEBUG }, 03:39:56 DEBUG "value": [ 03:39:56 DEBUG 1713667196, 03:39:56 DEBUG "0.04080000000006407" 03:39:56 DEBUG ] 03:39:56 DEBUG } 03:39:56 DEBUG ] 03:39:56 DEBUG } 03:39:56 DEBUG } 03:39:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667196" 03:39:56 INFO [http_cmd]: http status code OK 03:39:56 DEBUG --- status code --- 03:39:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:39:56 DEBUG --- http response --- 03:39:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:39:56 DEBUG ----- output ----- 03:39:56 DEBUG { 03:39:56 DEBUG "status": "success", 03:39:56 DEBUG "data": { 03:39:56 DEBUG "resultType": "vector", 03:39:56 DEBUG "result": [ 03:39:56 DEBUG { 03:39:56 DEBUG "metric": { 03:39:56 DEBUG "pod": "am-767f6b67b5-9zw22" 03:39:56 DEBUG }, 03:39:56 DEBUG "value": [ 03:39:56 DEBUG 03:39:56 DEBUG 03:39:56 DEBUG 03:39:56 DEBUG [...] 03:39:56 DEBUG 03:39:56 DEBUG 03:39:56 DEBUG 03:39:56 DEBUG "pod": "am-767f6b67b5-vbltz" 03:39:56 DEBUG }, 03:39:56 DEBUG "value": [ 03:39:56 DEBUG 1713667196, 03:39:56 DEBUG "0" 03:39:56 DEBUG ] 03:39:56 DEBUG } 03:39:56 DEBUG ] 03:39:56 DEBUG } 03:39:56 DEBUG } 03:39:57 INFO 03:39:57 INFO [loop_until]: kubectl --namespace=xlou top pods 03:39:57 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:39:57 INFO [loop_until]: OK (rc = 0) 03:39:57 DEBUG --- stdout --- 03:39:57 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 1264m 8727Mi am-767f6b67b5-qcqz2 814m 9065Mi am-767f6b67b5-vbltz 795m 8780Mi ds-cts-0 17440m 12508Mi ds-cts-1 2916m 12454Mi ds-cts-2 18316m 12541Mi ds-idrepo-0 148m 3935Mi ds-idrepo-1 130m 3966Mi ds-idrepo-2 20m 3937Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1180Mi idm-b85f9d978-wnhxb 3m 1239Mi lodemon-54c87f8cbb-jzrjs 2m 63Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-68c5d5b884-p97w5 994m 2305Mi 03:39:57 DEBUG --- stderr --- 03:39:57 DEBUG 03:39:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667198" 03:39:58 INFO [http_cmd]: http status code OK 03:39:58 DEBUG --- status code --- 03:39:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:39:58 DEBUG --- http response --- 03:39:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:39:58 DEBUG ----- output ----- 03:39:58 DEBUG { 03:39:58 DEBUG "status": "success", 03:39:58 DEBUG "data": { 03:39:58 DEBUG "resultType": "vector", 03:39:58 DEBUG "result": [ 03:39:58 DEBUG { 03:39:58 DEBUG "metric": { 03:39:58 DEBUG "pod": "am-767f6b67b5-9zw22" 03:39:58 DEBUG }, 03:39:58 DEBUG "value": [ 03:39:58 DEBUG 03:39:58 DEBUG 03:39:58 DEBUG 03:39:58 DEBUG [...] 03:39:58 DEBUG 03:39:58 DEBUG 03:39:58 DEBUG 03:39:58 DEBUG "pod": "am-767f6b67b5-vbltz" 03:39:58 DEBUG }, 03:39:58 DEBUG "value": [ 03:39:58 DEBUG 1713667198, 03:39:58 DEBUG "0" 03:39:58 DEBUG ] 03:39:58 DEBUG } 03:39:58 DEBUG ] 03:39:58 DEBUG } 03:39:58 DEBUG } 03:39:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667198" 03:39:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713667198" 03:39:58 INFO [http_cmd]: http status code OK 03:39:58 DEBUG --- status code --- 03:39:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:39:58 DEBUG --- http response --- 03:39:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 03:39:58 DEBUG ----- output ----- 03:39:58 DEBUG { 03:39:58 DEBUG "status": "success", 03:39:58 DEBUG "data": { 03:39:58 DEBUG "resultType": "vector", 03:39:58 DEBUG "result": [ 03:39:58 DEBUG { 03:39:58 DEBUG "metric": { 03:39:58 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:39:58 DEBUG }, 03:39:58 DEBUG "value": [ 03:39:58 DEBUG 03:39:58 DEBUG 03:39:58 DEBUG 03:39:58 DEBUG [...] 03:39:58 DEBUG 03:39:58 DEBUG 03:39:58 DEBUG 03:39:58 DEBUG "pod": "ds-cts-0" 03:39:58 DEBUG }, 03:39:58 DEBUG "value": [ 03:39:58 DEBUG 1713667198, 03:39:58 DEBUG "0" 03:39:58 DEBUG ] 03:39:58 DEBUG } 03:39:58 DEBUG ] 03:39:58 DEBUG } 03:39:58 DEBUG } 03:39:58 INFO [http_cmd]: http status code OK 03:39:58 DEBUG --- status code --- 03:39:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:39:58 DEBUG --- http response --- 03:39:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:39:58 DEBUG ----- output ----- 03:39:58 DEBUG { 03:39:58 DEBUG "status": "success", 03:39:58 DEBUG "data": { 03:39:58 DEBUG "resultType": "vector", 03:39:58 DEBUG "result": [ 03:39:58 DEBUG { 03:39:58 DEBUG "metric": { 03:39:58 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:39:58 DEBUG }, 03:39:58 DEBUG "value": [ 03:39:58 DEBUG 03:39:58 DEBUG 03:39:58 DEBUG 03:39:58 DEBUG [...] 03:39:58 DEBUG 03:39:58 DEBUG 03:39:58 DEBUG 03:39:58 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:39:58 DEBUG }, 03:39:58 DEBUG "value": [ 03:39:58 DEBUG 1713667198, 03:39:58 DEBUG "1.034089740393236" 03:39:58 DEBUG ] 03:39:58 DEBUG } 03:39:58 DEBUG ] 03:39:58 DEBUG } 03:39:58 DEBUG } 03:39:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667198" 03:39:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667198" 03:39:58 INFO [http_cmd]: http status code OK 03:39:58 INFO [http_cmd]: http status code OK 03:39:58 DEBUG --- status code --- 03:39:58 DEBUG --- status code --- 03:39:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:39:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:39:58 DEBUG --- http response --- 03:39:58 DEBUG --- http response --- 03:39:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:39:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:39:58 DEBUG ----- output ----- 03:39:58 DEBUG ----- output ----- 03:39:58 DEBUG { 03:39:58 DEBUG { 03:39:58 DEBUG "status": "success", 03:39:58 DEBUG "status": "success", 03:39:58 DEBUG "data": { 03:39:58 DEBUG "data": { 03:39:58 DEBUG "resultType": "vector", 03:39:58 DEBUG "resultType": "vector", 03:39:58 DEBUG "result": [ 03:39:58 DEBUG "result": [ 03:39:58 DEBUG { 03:39:58 DEBUG { 03:39:58 DEBUG "metric": { 03:39:58 DEBUG "metric": { 03:39:58 DEBUG "pod": "ds-idrepo-0" 03:39:58 DEBUG "pod": "am-767f6b67b5-9zw22" 03:39:58 DEBUG }, 03:39:58 DEBUG }, 03:39:58 DEBUG "value": [ 03:39:58 DEBUG "value": [ 03:39:58 DEBUG 03:39:58 DEBUG 03:39:58 DEBUG 03:39:58 DEBUG 03:39:58 DEBUG 03:39:58 DEBUG 03:39:58 DEBUG [...] 03:39:58 DEBUG [...] 03:39:58 DEBUG 03:39:58 DEBUG 03:39:58 DEBUG 03:39:58 DEBUG 03:39:58 DEBUG 03:39:58 DEBUG 03:39:58 DEBUG "pod": "ds-idrepo-2" 03:39:58 DEBUG "pod": "am-767f6b67b5-vbltz" 03:39:58 DEBUG }, 03:39:58 DEBUG }, 03:39:58 DEBUG "value": [ 03:39:58 DEBUG "value": [ 03:39:58 DEBUG 1713667198, 03:39:58 DEBUG 1713667198, 03:39:58 DEBUG "0" 03:39:58 DEBUG "5960" 03:39:58 DEBUG ] 03:39:58 DEBUG ] 03:39:58 DEBUG } 03:39:58 DEBUG } 03:39:58 DEBUG ] 03:39:58 DEBUG ] 03:39:58 DEBUG } 03:39:58 DEBUG } 03:39:58 DEBUG } 03:39:58 DEBUG } 03:39:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667198" 03:39:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667198" 03:39:58 INFO [http_cmd]: http status code OK 03:39:58 DEBUG --- status code --- 03:39:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:39:58 DEBUG --- http response --- 03:39:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:39:58 DEBUG ----- output ----- 03:39:58 DEBUG { 03:39:58 DEBUG "status": "success", 03:39:58 DEBUG "data": { 03:39:58 DEBUG "resultType": "vector", 03:39:58 DEBUG "result": [ 03:39:58 DEBUG { 03:39:58 DEBUG "metric": { 03:39:58 DEBUG "pod": "am-767f6b67b5-9zw22" 03:39:58 DEBUG }, 03:39:58 DEBUG "value": [ 03:39:58 DEBUG 03:39:58 DEBUG 03:39:58 DEBUG 03:39:58 DEBUG [...] 03:39:58 DEBUG 03:39:58 DEBUG 03:39:58 DEBUG 03:39:58 DEBUG "pod": "am-767f6b67b5-vbltz" 03:39:58 DEBUG }, 03:39:58 DEBUG "value": [ 03:39:58 DEBUG 1713667198, 03:39:58 DEBUG "0" 03:39:58 DEBUG ] 03:39:58 DEBUG } 03:39:58 DEBUG ] 03:39:58 DEBUG } 03:39:58 DEBUG } 03:39:58 INFO [http_cmd]: http status code OK 03:39:58 DEBUG --- status code --- 03:39:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:39:58 DEBUG --- http response --- 03:39:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 03:39:58 DEBUG ----- output ----- 03:39:58 DEBUG { 03:39:58 DEBUG "status": "success", 03:39:58 DEBUG "data": { 03:39:58 DEBUG "resultType": "vector", 03:39:58 DEBUG "result": [ 03:39:58 DEBUG { 03:39:58 DEBUG "metric": { 03:39:58 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:39:58 DEBUG }, 03:39:58 DEBUG "value": [ 03:39:58 DEBUG 03:39:58 DEBUG 03:39:58 DEBUG 03:39:58 DEBUG [...] 03:39:58 DEBUG 03:39:58 DEBUG 03:39:58 DEBUG 03:39:58 DEBUG "pod": "ds-cts-0" 03:39:58 DEBUG }, 03:39:58 DEBUG "value": [ 03:39:58 DEBUG 1713667198, 03:39:58 DEBUG "0" 03:39:58 DEBUG ] 03:39:58 DEBUG } 03:39:58 DEBUG ] 03:39:58 DEBUG } 03:39:58 DEBUG } 03:39:58 INFO 03:39:58 INFO [loop_until]: kubectl --namespace=xlou top node 03:39:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:39:58 INFO [loop_until]: OK (rc = 0) 03:39:58 DEBUG --- stdout --- 03:39:58 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 67m 0% 6283Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 86m 0% 7779Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 979m 3% 11534Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 81m 0% 3357Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 17554m 58% 15798Mi 14% gke-xlou-cdm-default-pool-1b0769c8-wpwl 150m 0% 5244Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 1097m 3% 12008Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 74m 0% 3660Mi 3% gke-xlou-cdm-ds-0f601751-hmm7 792m 2% 11071Mi 9% gke-xlou-cdm-ds-0f601751-j488 65m 0% 7366Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 788m 2% 17600Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 128m 0% 8030Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 18690m 62% 15219Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 1081m 6% 12549Mi 21% 03:39:58 DEBUG --- stderr --- 03:39:58 DEBUG 03:39:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713667198" 03:39:58 INFO [http_cmd]: http status code OK 03:39:58 DEBUG --- status code --- 03:39:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:39:58 DEBUG --- http response --- 03:39:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:39:58 DEBUG ----- output ----- 03:39:58 DEBUG { 03:39:58 DEBUG "status": "success", 03:39:58 DEBUG "data": { 03:39:58 DEBUG "resultType": "vector", 03:39:58 DEBUG "result": [ 03:39:58 DEBUG { 03:39:58 DEBUG "metric": { 03:39:58 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:39:58 DEBUG }, 03:39:58 DEBUG "value": [ 03:39:58 DEBUG 03:39:58 DEBUG 03:39:58 DEBUG 03:39:58 DEBUG [...] 03:39:58 DEBUG 03:39:58 DEBUG 03:39:58 DEBUG 03:39:58 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:39:58 DEBUG }, 03:39:58 DEBUG "value": [ 03:39:58 DEBUG 1713667198, 03:39:58 DEBUG "1802522624" 03:39:58 DEBUG ] 03:39:58 DEBUG } 03:39:58 DEBUG ] 03:39:58 DEBUG } 03:39:58 DEBUG } 03:39:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667199" 03:39:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667199" 03:39:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713667199" 03:39:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667199" 03:39:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713667199" 03:39:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667199" 03:39:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713667199" 03:39:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667199" 03:39:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713667199" 03:39:59 INFO [http_cmd]: http status code OK 03:39:59 INFO [http_cmd]: http status code OK 03:39:59 INFO [http_cmd]: http status code OK 03:39:59 DEBUG --- status code --- 03:39:59 DEBUG --- status code --- 03:39:59 INFO [http_cmd]: http status code OK 03:39:59 DEBUG --- status code --- 03:39:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:39:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:39:59 DEBUG --- status code --- 03:39:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:39:59 DEBUG --- http response --- 03:39:59 DEBUG --- http response --- 03:39:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:39:59 DEBUG --- http response --- 03:39:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:39:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:39:59 DEBUG --- http response --- 03:39:59 DEBUG ----- output ----- 03:39:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:39:59 DEBUG ----- output ----- 03:39:59 DEBUG { 03:39:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:39:59 DEBUG ----- output ----- 03:39:59 DEBUG { 03:39:59 DEBUG "status": "success", 03:39:59 DEBUG ----- output ----- 03:39:59 DEBUG { 03:39:59 DEBUG "status": "success", 03:39:59 DEBUG "data": { 03:39:59 DEBUG { 03:39:59 DEBUG "status": "success", 03:39:59 DEBUG "data": { 03:39:59 DEBUG "resultType": "vector", 03:39:59 DEBUG "status": "success", 03:39:59 DEBUG "data": { 03:39:59 DEBUG "resultType": "vector", 03:39:59 DEBUG "result": [ 03:39:59 DEBUG "data": { 03:39:59 DEBUG "resultType": "vector", 03:39:59 DEBUG "result": [ 03:39:59 DEBUG { 03:39:59 DEBUG "resultType": "vector", 03:39:59 DEBUG "result": [ 03:39:59 DEBUG { 03:39:59 DEBUG "metric": { 03:39:59 DEBUG "result": [ 03:39:59 DEBUG { 03:39:59 DEBUG "metric": { 03:39:59 DEBUG "pod": "ds-idrepo-0" 03:39:59 DEBUG { 03:39:59 DEBUG "metric": { 03:39:59 DEBUG "pod": "ds-idrepo-0" 03:39:59 DEBUG }, 03:39:59 DEBUG "metric": { 03:39:59 DEBUG "instance": "10.142.0.107:9100" 03:39:59 DEBUG }, 03:39:59 DEBUG "value": [ 03:39:59 DEBUG "pod": "ds-idrepo-0" 03:39:59 DEBUG }, 03:39:59 DEBUG "value": [ 03:39:59 DEBUG 03:39:59 DEBUG }, 03:39:59 DEBUG "value": [ 03:39:59 DEBUG 03:39:59 DEBUG 03:39:59 DEBUG "value": [ 03:39:59 DEBUG 03:39:59 DEBUG 03:39:59 DEBUG 03:39:59 DEBUG 03:39:59 DEBUG 03:39:59 DEBUG 03:39:59 DEBUG [...] 03:39:59 DEBUG 03:39:59 DEBUG 03:39:59 DEBUG [...] 03:39:59 DEBUG 03:39:59 DEBUG 03:39:59 DEBUG [...] 03:39:59 DEBUG 03:39:59 DEBUG 03:39:59 DEBUG [...] 03:39:59 DEBUG 03:39:59 DEBUG 03:39:59 DEBUG 03:39:59 DEBUG 03:39:59 DEBUG 03:39:59 DEBUG 03:39:59 DEBUG "pod": "ds-idrepo-2" 03:39:59 DEBUG 03:39:59 DEBUG 03:39:59 DEBUG "pod": "ds-idrepo-2" 03:39:59 DEBUG }, 03:39:59 DEBUG "value": [ 03:39:59 DEBUG "instance": "10.142.0.43:9100" 03:39:59 DEBUG }, 03:39:59 DEBUG 03:39:59 DEBUG 1713667199, 03:39:59 DEBUG }, 03:39:59 DEBUG "value": [ 03:39:59 DEBUG "pod": "idm-b85f9d978-txpwv" 03:39:59 DEBUG "0" 03:39:59 DEBUG "value": [ 03:39:59 DEBUG 1713667199, 03:39:59 DEBUG }, 03:39:59 DEBUG ] 03:39:59 DEBUG 1713667199, 03:39:59 DEBUG "0" 03:39:59 DEBUG "value": [ 03:39:59 DEBUG } 03:39:59 DEBUG "9647441.566666666" 03:39:59 DEBUG ] 03:39:59 DEBUG 1713667199, 03:39:59 DEBUG ] 03:39:59 DEBUG ] 03:39:59 DEBUG } 03:39:59 DEBUG "412.5843602054863" 03:39:59 DEBUG } 03:39:59 DEBUG } 03:39:59 DEBUG ] 03:39:59 DEBUG ] 03:39:59 DEBUG } 03:39:59 DEBUG ] 03:39:59 DEBUG } 03:39:59 DEBUG } 03:39:59 DEBUG } 03:39:59 DEBUG } 03:39:59 DEBUG ] 03:39:59 DEBUG } 03:39:59 DEBUG } 03:39:59 DEBUG } 03:39:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713667199" 03:39:59 INFO [http_cmd]: http status code OK 03:39:59 INFO [http_cmd]: http status code OK 03:39:59 INFO [http_cmd]: http status code OK 03:39:59 INFO [http_cmd]: http status code OK 03:39:59 DEBUG --- status code --- 03:39:59 INFO [http_cmd]: http status code OK 03:39:59 DEBUG --- status code --- 03:39:59 DEBUG --- status code --- 03:39:59 DEBUG --- status code --- 03:39:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:39:59 DEBUG --- status code --- 03:39:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:39:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:39:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:39:59 DEBUG --- http response --- 03:39:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:39:59 DEBUG --- http response --- 03:39:59 DEBUG --- http response --- 03:39:59 DEBUG --- http response --- 03:39:59 DEBUG --- http response --- 03:39:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:39:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:39:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:39:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:39:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:39:59 DEBUG ----- output ----- 03:39:59 DEBUG ----- output ----- 03:39:59 DEBUG ----- output ----- 03:39:59 DEBUG ----- output ----- 03:39:59 DEBUG ----- output ----- 03:39:59 DEBUG { 03:39:59 DEBUG { 03:39:59 DEBUG { 03:39:59 DEBUG { 03:39:59 DEBUG { 03:39:59 DEBUG "status": "success", 03:39:59 DEBUG "status": "success", 03:39:59 DEBUG "status": "success", 03:39:59 DEBUG "status": "success", 03:39:59 DEBUG "status": "success", 03:39:59 DEBUG "data": { 03:39:59 DEBUG "data": { 03:39:59 DEBUG "data": { 03:39:59 DEBUG "data": { 03:39:59 DEBUG "data": { 03:39:59 DEBUG "resultType": "vector", 03:39:59 DEBUG "resultType": "vector", 03:39:59 DEBUG "resultType": "vector", 03:39:59 DEBUG "resultType": "vector", 03:39:59 DEBUG "resultType": "vector", 03:39:59 DEBUG "result": [ 03:39:59 DEBUG "result": [ 03:39:59 DEBUG "result": [ 03:39:59 DEBUG "result": [ 03:39:59 DEBUG "result": [ 03:39:59 DEBUG { 03:39:59 DEBUG { 03:39:59 DEBUG { 03:39:59 DEBUG { 03:39:59 DEBUG { 03:39:59 DEBUG "metric": { 03:39:59 DEBUG "metric": { 03:39:59 DEBUG "metric": { 03:39:59 DEBUG "metric": { 03:39:59 DEBUG "metric": { 03:39:59 DEBUG "instance": "10.142.0.107:9100" 03:39:59 DEBUG "instance": "10.142.0.107:9100" 03:39:59 DEBUG "pod": "am-767f6b67b5-9zw22" 03:39:59 DEBUG "instance": "10.142.0.107:9100" 03:39:59 DEBUG "pod": "ds-cts-0" 03:39:59 DEBUG }, 03:39:59 DEBUG }, 03:39:59 DEBUG }, 03:39:59 DEBUG }, 03:39:59 DEBUG }, 03:39:59 DEBUG "value": [ 03:39:59 DEBUG "value": [ 03:39:59 DEBUG "value": [ 03:39:59 DEBUG "value": [ 03:39:59 DEBUG "value": [ 03:39:59 DEBUG 03:39:59 DEBUG 03:39:59 DEBUG 03:39:59 DEBUG 03:39:59 DEBUG 03:39:59 DEBUG 03:39:59 DEBUG 03:39:59 DEBUG 03:39:59 DEBUG 03:39:59 DEBUG 03:39:59 DEBUG 03:39:59 DEBUG 03:39:59 DEBUG 03:39:59 DEBUG 03:39:59 DEBUG 03:39:59 DEBUG [...] 03:39:59 DEBUG [...] 03:39:59 DEBUG [...] 03:39:59 DEBUG [...] 03:39:59 DEBUG [...] 03:39:59 DEBUG 03:39:59 DEBUG 03:39:59 DEBUG 03:39:59 DEBUG 03:39:59 DEBUG 03:39:59 DEBUG 03:39:59 DEBUG 03:39:59 DEBUG 03:39:59 DEBUG 03:39:59 DEBUG 03:39:59 DEBUG 03:39:59 DEBUG 03:39:59 DEBUG 03:39:59 DEBUG 03:39:59 DEBUG 03:39:59 DEBUG "instance": "10.142.0.43:9100" 03:39:59 DEBUG "instance": "10.142.0.43:9100" 03:39:59 DEBUG "pod": "am-767f6b67b5-vbltz" 03:39:59 DEBUG "instance": "10.142.0.43:9100" 03:39:59 DEBUG "pod": "ds-cts-2" 03:39:59 DEBUG }, 03:39:59 DEBUG }, 03:39:59 DEBUG }, 03:39:59 DEBUG }, 03:39:59 DEBUG }, 03:39:59 INFO [http_cmd]: http status code OK 03:39:59 DEBUG "value": [ 03:39:59 DEBUG "value": [ 03:39:59 DEBUG "value": [ 03:39:59 DEBUG "value": [ 03:39:59 DEBUG "value": [ 03:39:59 DEBUG --- status code --- 03:39:59 DEBUG 1713667199, 03:39:59 DEBUG 1713667199, 03:39:59 DEBUG 1713667199, 03:39:59 DEBUG 1713667199, 03:39:59 DEBUG 1713667199, 03:39:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:39:59 DEBUG "9640064.966666667" 03:39:59 DEBUG "0.016000000000008185" 03:39:59 DEBUG "0" 03:39:59 DEBUG "4488656768000" 03:39:59 DEBUG "0" 03:39:59 DEBUG --- http response --- 03:39:59 DEBUG ] 03:39:59 DEBUG ] 03:39:59 DEBUG ] 03:39:59 DEBUG ] 03:39:59 DEBUG ] 03:39:59 DEBUG } 03:39:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:39:59 DEBUG } 03:39:59 DEBUG } 03:39:59 DEBUG } 03:39:59 DEBUG } 03:39:59 DEBUG ] 03:39:59 DEBUG ----- output ----- 03:39:59 DEBUG ] 03:39:59 DEBUG ] 03:39:59 DEBUG ] 03:39:59 DEBUG ] 03:39:59 DEBUG } 03:39:59 DEBUG { 03:39:59 DEBUG } 03:39:59 DEBUG } 03:39:59 DEBUG } 03:39:59 DEBUG } 03:39:59 DEBUG } 03:39:59 DEBUG "status": "success", 03:39:59 DEBUG } 03:39:59 DEBUG } 03:39:59 DEBUG } 03:39:59 DEBUG } 03:39:59 DEBUG "data": { 03:39:59 DEBUG "resultType": "vector", 03:39:59 DEBUG "result": [ 03:39:59 DEBUG { 03:39:59 DEBUG "metric": { 03:39:59 DEBUG "pod": "ds-idrepo-0" 03:39:59 DEBUG }, 03:39:59 DEBUG "value": [ 03:39:59 DEBUG 03:39:59 DEBUG 03:39:59 DEBUG 03:39:59 DEBUG [...] 03:39:59 DEBUG 03:39:59 DEBUG 03:39:59 DEBUG 03:39:59 DEBUG "pod": "ds-cts-2" 03:39:59 DEBUG }, 03:39:59 DEBUG "value": [ 03:39:59 DEBUG 1713667199, 03:39:59 DEBUG "0" 03:39:59 DEBUG ] 03:39:59 DEBUG } 03:39:59 DEBUG ] 03:39:59 DEBUG } 03:39:59 DEBUG } 03:39:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667199" 03:39:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667199" 03:39:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667199" 03:39:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713667199" 03:39:59 INFO [http_cmd]: http status code OK 03:39:59 INFO [http_cmd]: http status code OK 03:39:59 DEBUG --- status code --- 03:39:59 DEBUG --- status code --- 03:39:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:39:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:39:59 DEBUG --- http response --- 03:39:59 DEBUG --- http response --- 03:39:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:39:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:39:59 DEBUG ----- output ----- 03:39:59 DEBUG ----- output ----- 03:39:59 DEBUG { 03:39:59 DEBUG { 03:39:59 DEBUG "status": "success", 03:39:59 DEBUG "status": "success", 03:39:59 DEBUG "data": { 03:39:59 DEBUG "data": { 03:39:59 DEBUG "resultType": "vector", 03:39:59 DEBUG "resultType": "vector", 03:39:59 DEBUG "result": [ 03:39:59 DEBUG "result": [ 03:39:59 DEBUG { 03:39:59 DEBUG { 03:39:59 DEBUG "metric": { 03:39:59 DEBUG "metric": { 03:39:59 DEBUG "pod": "ds-idrepo-0" 03:39:59 DEBUG "pod": "ds-cts-0" 03:39:59 DEBUG }, 03:39:59 DEBUG }, 03:39:59 DEBUG "value": [ 03:39:59 DEBUG "value": [ 03:39:59 DEBUG 03:39:59 DEBUG 03:39:59 DEBUG 03:39:59 DEBUG 03:39:59 DEBUG 03:39:59 DEBUG 03:39:59 DEBUG [...] 03:39:59 DEBUG [...] 03:39:59 DEBUG 03:39:59 DEBUG 03:39:59 DEBUG 03:39:59 DEBUG 03:39:59 DEBUG 03:39:59 DEBUG 03:39:59 DEBUG "pod": "idm-b85f9d978-txpwv" 03:39:59 DEBUG "pod": "ds-cts-2" 03:39:59 DEBUG }, 03:39:59 DEBUG }, 03:39:59 INFO [http_cmd]: http status code OK 03:39:59 INFO [http_cmd]: http status code OK 03:39:59 DEBUG "value": [ 03:39:59 DEBUG "value": [ 03:39:59 DEBUG --- status code --- 03:39:59 DEBUG --- status code --- 03:39:59 DEBUG 1713667199, 03:39:59 DEBUG 1713667199, 03:39:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:39:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:39:59 DEBUG "0" 03:39:59 DEBUG "29.066666666666666" 03:39:59 DEBUG --- http response --- 03:39:59 DEBUG --- http response --- 03:39:59 DEBUG ] 03:39:59 DEBUG ] 03:39:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:39:59 DEBUG } 03:39:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:39:59 DEBUG } 03:39:59 DEBUG ----- output ----- 03:39:59 DEBUG ] 03:39:59 DEBUG ----- output ----- 03:39:59 DEBUG ] 03:39:59 DEBUG { 03:39:59 DEBUG } 03:39:59 DEBUG { 03:39:59 DEBUG } 03:39:59 DEBUG "status": "success", 03:39:59 DEBUG } 03:39:59 DEBUG "status": "success", 03:39:59 DEBUG } 03:39:59 DEBUG "data": { 03:39:59 DEBUG "data": { 03:39:59 DEBUG "resultType": "vector", 03:39:59 DEBUG "resultType": "vector", 03:39:59 DEBUG "result": [ 03:39:59 DEBUG "result": [ 03:39:59 DEBUG { 03:39:59 DEBUG { 03:39:59 DEBUG "metric": { 03:39:59 DEBUG "metric": { 03:39:59 DEBUG "pod": "am-767f6b67b5-9zw22" 03:39:59 DEBUG "pod": "am-767f6b67b5-9zw22" 03:39:59 DEBUG }, 03:39:59 DEBUG }, 03:39:59 DEBUG "value": [ 03:39:59 DEBUG "value": [ 03:39:59 DEBUG 03:39:59 DEBUG 03:39:59 DEBUG 03:39:59 DEBUG 03:39:59 DEBUG 03:39:59 DEBUG 03:39:59 DEBUG [...] 03:39:59 DEBUG [...] 03:39:59 DEBUG 03:39:59 DEBUG 03:39:59 DEBUG 03:39:59 DEBUG 03:39:59 DEBUG 03:39:59 DEBUG 03:39:59 DEBUG "pod": "am-767f6b67b5-vbltz" 03:39:59 DEBUG "pod": "am-767f6b67b5-vbltz" 03:39:59 DEBUG }, 03:39:59 DEBUG }, 03:39:59 DEBUG "value": [ 03:39:59 DEBUG "value": [ 03:39:59 DEBUG 1713667199, 03:39:59 DEBUG 1713667199, 03:39:59 DEBUG "195.942" 03:39:59 DEBUG "396" 03:39:59 DEBUG ] 03:39:59 DEBUG ] 03:39:59 DEBUG } 03:39:59 DEBUG } 03:39:59 DEBUG ] 03:39:59 DEBUG ] 03:39:59 DEBUG } 03:39:59 DEBUG } 03:39:59 DEBUG } 03:39:59 DEBUG } 03:40:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713667254" 03:40:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667254" 03:40:55 INFO [http_cmd]: http status code OK 03:40:55 ERROR [http_cmd]: ERROR 03:40:55 DEBUG --- status code --- 03:40:55 ERROR --- status code --- 03:40:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:40:55 ERROR http status code is 400 (expected [200, 401, 403, 404]) 03:40:55 DEBUG --- http response --- 03:40:55 ERROR --- http response --- 03:40:55 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 03:40:55 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 03:40:55 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 03:40:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713667256" 03:40:56 INFO [http_cmd]: http status code OK 03:40:56 DEBUG --- status code --- 03:40:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:40:56 DEBUG --- http response --- 03:40:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:40:56 DEBUG ----- output ----- 03:40:56 DEBUG { 03:40:56 DEBUG "status": "success", 03:40:56 DEBUG "data": { 03:40:56 DEBUG "resultType": "vector", 03:40:56 DEBUG "result": [ 03:40:56 DEBUG { 03:40:56 DEBUG "metric": { 03:40:56 DEBUG "instance": "10.142.0.107:9100" 03:40:56 DEBUG }, 03:40:56 DEBUG "value": [ 03:40:56 DEBUG 03:40:56 DEBUG 03:40:56 DEBUG 03:40:56 DEBUG [...] 03:40:56 DEBUG 03:40:56 DEBUG 03:40:56 DEBUG 03:40:56 DEBUG "instance": "10.142.0.43:9100" 03:40:56 DEBUG }, 03:40:56 DEBUG "value": [ 03:40:56 DEBUG 1713667256, 03:40:56 DEBUG "0.0010000000000142488" 03:40:56 DEBUG ] 03:40:56 DEBUG } 03:40:56 DEBUG ] 03:40:56 DEBUG } 03:40:56 DEBUG } 03:40:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667256" 03:40:56 INFO [http_cmd]: http status code OK 03:40:56 DEBUG --- status code --- 03:40:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:40:56 DEBUG --- http response --- 03:40:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:40:56 DEBUG ----- output ----- 03:40:56 DEBUG { 03:40:56 DEBUG "status": "success", 03:40:56 DEBUG "data": { 03:40:56 DEBUG "resultType": "vector", 03:40:56 DEBUG "result": [ 03:40:56 DEBUG { 03:40:56 DEBUG "metric": { 03:40:56 DEBUG "pod": "ds-cts-0" 03:40:56 DEBUG }, 03:40:56 DEBUG "value": [ 03:40:56 DEBUG 03:40:56 DEBUG 03:40:56 DEBUG 03:40:56 DEBUG [...] 03:40:56 DEBUG 03:40:56 DEBUG 03:40:56 DEBUG 03:40:56 DEBUG "pod": "ds-cts-2" 03:40:56 DEBUG }, 03:40:56 DEBUG "value": [ 03:40:56 DEBUG 1713667256, 03:40:56 DEBUG "17.1049" 03:40:56 DEBUG ] 03:40:56 DEBUG } 03:40:56 DEBUG ] 03:40:56 DEBUG } 03:40:56 DEBUG } 03:40:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667256" 03:40:56 INFO [http_cmd]: http status code OK 03:40:56 DEBUG --- status code --- 03:40:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:40:56 DEBUG --- http response --- 03:40:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:40:56 DEBUG ----- output ----- 03:40:56 DEBUG { 03:40:56 DEBUG "status": "success", 03:40:56 DEBUG "data": { 03:40:56 DEBUG "resultType": "vector", 03:40:56 DEBUG "result": [ 03:40:56 DEBUG { 03:40:56 DEBUG "metric": { 03:40:56 DEBUG "pod": "am-767f6b67b5-9zw22" 03:40:56 DEBUG }, 03:40:56 DEBUG "value": [ 03:40:56 DEBUG 03:40:56 DEBUG 03:40:56 DEBUG 03:40:56 DEBUG [...] 03:40:56 DEBUG 03:40:56 DEBUG 03:40:56 DEBUG 03:40:56 DEBUG "pod": "am-767f6b67b5-vbltz" 03:40:56 DEBUG }, 03:40:56 DEBUG "value": [ 03:40:56 DEBUG 1713667256, 03:40:56 DEBUG "0" 03:40:56 DEBUG ] 03:40:56 DEBUG } 03:40:56 DEBUG ] 03:40:56 DEBUG } 03:40:56 DEBUG } 03:40:57 INFO 03:40:57 INFO [loop_until]: kubectl --namespace=xlou top pods 03:40:57 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:40:57 INFO [loop_until]: OK (rc = 0) 03:40:57 DEBUG --- stdout --- 03:40:57 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-tlgkc 1m 9Mi am-767f6b67b5-9zw22 451m 8712Mi am-767f6b67b5-qcqz2 517m 9050Mi am-767f6b67b5-vbltz 411m 8775Mi ds-cts-0 9569m 11712Mi ds-cts-1 16m 12455Mi ds-cts-2 18596m 12538Mi ds-idrepo-0 14m 3936Mi ds-idrepo-1 13m 3966Mi ds-idrepo-2 11m 3937Mi end-user-ui-58878d7668-z8s6j 1m 5Mi idm-b85f9d978-txpwv 4m 1179Mi idm-b85f9d978-wnhxb 3m 1239Mi lodemon-54c87f8cbb-jzrjs 2m 64Mi login-ui-5fdf49fc66-dlhjd 1m 4Mi overseer-0-68c5d5b884-p97w5 755m 2422Mi 03:40:57 DEBUG --- stderr --- 03:40:57 DEBUG 03:40:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667258" 03:40:58 INFO [http_cmd]: http status code OK 03:40:58 DEBUG --- status code --- 03:40:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:40:58 DEBUG --- http response --- 03:40:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:40:58 DEBUG ----- output ----- 03:40:58 DEBUG { 03:40:58 DEBUG "status": "success", 03:40:58 DEBUG "data": { 03:40:58 DEBUG "resultType": "vector", 03:40:58 DEBUG "result": [ 03:40:58 DEBUG { 03:40:58 DEBUG "metric": { 03:40:58 DEBUG "pod": "am-767f6b67b5-9zw22" 03:40:58 DEBUG }, 03:40:58 DEBUG "value": [ 03:40:58 DEBUG 03:40:58 DEBUG 03:40:58 DEBUG 03:40:58 DEBUG [...] 03:40:58 DEBUG 03:40:58 DEBUG 03:40:58 DEBUG 03:40:58 DEBUG "pod": "am-767f6b67b5-vbltz" 03:40:58 DEBUG }, 03:40:58 DEBUG "value": [ 03:40:58 DEBUG 1713667258, 03:40:58 DEBUG "0" 03:40:58 DEBUG ] 03:40:58 DEBUG } 03:40:58 DEBUG ] 03:40:58 DEBUG } 03:40:58 DEBUG } 03:40:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667258" 03:40:58 INFO [http_cmd]: http status code OK 03:40:58 DEBUG --- status code --- 03:40:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:40:58 DEBUG --- http response --- 03:40:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 03:40:58 DEBUG ----- output ----- 03:40:58 DEBUG { 03:40:58 DEBUG "status": "success", 03:40:58 DEBUG "data": { 03:40:58 DEBUG "resultType": "vector", 03:40:58 DEBUG "result": [ 03:40:58 DEBUG { 03:40:58 DEBUG "metric": { 03:40:58 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:40:58 DEBUG }, 03:40:58 DEBUG "value": [ 03:40:58 DEBUG 03:40:58 DEBUG 03:40:58 DEBUG 03:40:58 DEBUG [...] 03:40:58 DEBUG 03:40:58 DEBUG 03:40:58 DEBUG 03:40:58 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:40:58 DEBUG }, 03:40:58 DEBUG "value": [ 03:40:58 DEBUG 1713667258, 03:40:58 DEBUG "0" 03:40:58 DEBUG ] 03:40:58 DEBUG } 03:40:58 DEBUG ] 03:40:58 DEBUG } 03:40:58 DEBUG } 03:40:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667258" 03:40:58 INFO [http_cmd]: http status code OK 03:40:58 DEBUG --- status code --- 03:40:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:40:58 DEBUG --- http response --- 03:40:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:40:58 DEBUG ----- output ----- 03:40:58 DEBUG { 03:40:58 DEBUG "status": "success", 03:40:58 DEBUG "data": { 03:40:58 DEBUG "resultType": "vector", 03:40:58 DEBUG "result": [ 03:40:58 DEBUG { 03:40:58 DEBUG "metric": { 03:40:58 DEBUG "pod": "am-767f6b67b5-9zw22" 03:40:58 DEBUG }, 03:40:58 DEBUG "value": [ 03:40:58 DEBUG 03:40:58 DEBUG 03:40:58 DEBUG 03:40:58 DEBUG [...] 03:40:58 DEBUG 03:40:58 DEBUG 03:40:58 DEBUG 03:40:58 DEBUG "pod": "am-767f6b67b5-vbltz" 03:40:58 DEBUG }, 03:40:58 DEBUG "value": [ 03:40:58 DEBUG 1713667258, 03:40:58 DEBUG "0.5" 03:40:58 DEBUG ] 03:40:58 DEBUG } 03:40:58 DEBUG ] 03:40:58 DEBUG } 03:40:58 DEBUG } 03:40:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713667258" 03:40:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667258" 03:40:58 INFO [http_cmd]: http status code OK 03:40:58 DEBUG --- status code --- 03:40:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:40:58 DEBUG --- http response --- 03:40:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:40:58 DEBUG ----- output ----- 03:40:58 DEBUG { 03:40:58 DEBUG "status": "success", 03:40:58 DEBUG "data": { 03:40:58 DEBUG "resultType": "vector", 03:40:58 DEBUG "result": [ 03:40:58 DEBUG { 03:40:58 DEBUG "metric": { 03:40:58 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:40:58 DEBUG }, 03:40:58 DEBUG "value": [ 03:40:58 DEBUG 03:40:58 DEBUG 03:40:58 DEBUG 03:40:58 DEBUG [...] 03:40:58 DEBUG 03:40:58 DEBUG 03:40:58 DEBUG 03:40:58 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:40:58 DEBUG }, 03:40:58 DEBUG "value": [ 03:40:58 DEBUG 1713667258, 03:40:58 DEBUG "0.7741861237141076" 03:40:58 DEBUG ] 03:40:58 DEBUG } 03:40:58 DEBUG ] 03:40:58 DEBUG } 03:40:58 DEBUG } 03:40:58 INFO [http_cmd]: http status code OK 03:40:58 DEBUG --- status code --- 03:40:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:40:58 DEBUG --- http response --- 03:40:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:40:58 DEBUG ----- output ----- 03:40:58 DEBUG { 03:40:58 DEBUG "status": "success", 03:40:58 DEBUG "data": { 03:40:58 DEBUG "resultType": "vector", 03:40:58 DEBUG "result": [ 03:40:58 DEBUG { 03:40:58 DEBUG "metric": { 03:40:58 DEBUG "pod": "ds-idrepo-0" 03:40:58 DEBUG }, 03:40:58 DEBUG "value": [ 03:40:58 DEBUG 03:40:58 DEBUG 03:40:58 DEBUG 03:40:58 DEBUG [...] 03:40:58 DEBUG 03:40:58 DEBUG 03:40:58 DEBUG 03:40:58 DEBUG "pod": "ds-idrepo-2" 03:40:58 DEBUG }, 03:40:58 DEBUG "value": [ 03:40:58 DEBUG 1713667258, 03:40:58 DEBUG "0" 03:40:58 DEBUG ] 03:40:58 DEBUG } 03:40:58 DEBUG ] 03:40:58 DEBUG } 03:40:58 DEBUG } 03:40:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667258" 03:40:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667258" 03:40:58 INFO [http_cmd]: http status code OK 03:40:58 DEBUG --- status code --- 03:40:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:40:58 DEBUG --- http response --- 03:40:58 INFO [http_cmd]: http status code OK 03:40:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:40:58 DEBUG --- status code --- 03:40:58 DEBUG ----- output ----- 03:40:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:40:58 DEBUG { 03:40:58 DEBUG --- http response --- 03:40:58 DEBUG "status": "success", 03:40:58 DEBUG "data": { 03:40:58 DEBUG "resultType": "vector", 03:40:58 DEBUG "result": [ 03:40:58 DEBUG { 03:40:58 DEBUG "metric": { 03:40:58 DEBUG "pod": "am-767f6b67b5-9zw22" 03:40:58 DEBUG }, 03:40:58 DEBUG "value": [ 03:40:58 DEBUG 03:40:58 DEBUG 03:40:58 DEBUG 03:40:58 DEBUG [...] 03:40:58 DEBUG 03:40:58 DEBUG 03:40:58 DEBUG 03:40:58 DEBUG "pod": "am-767f6b67b5-vbltz" 03:40:58 DEBUG }, 03:40:58 DEBUG "value": [ 03:40:58 DEBUG 1713667258, 03:40:58 DEBUG "0" 03:40:58 DEBUG ] 03:40:58 DEBUG } 03:40:58 DEBUG ] 03:40:58 DEBUG } 03:40:58 DEBUG } 03:40:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 03:40:58 DEBUG ----- output ----- 03:40:58 DEBUG { 03:40:58 DEBUG "status": "success", 03:40:58 DEBUG "data": { 03:40:58 DEBUG "resultType": "vector", 03:40:58 DEBUG "result": [ 03:40:58 DEBUG { 03:40:58 DEBUG "metric": { 03:40:58 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:40:58 DEBUG }, 03:40:58 DEBUG "value": [ 03:40:58 DEBUG 03:40:58 DEBUG 03:40:58 DEBUG 03:40:58 DEBUG [...] 03:40:58 DEBUG 03:40:58 DEBUG 03:40:58 DEBUG 03:40:58 DEBUG "pod": "overseer-0-68c5d5b884-p97w5" 03:40:58 DEBUG }, 03:40:58 DEBUG "value": [ 03:40:58 DEBUG 1713667258, 03:40:58 DEBUG "0" 03:40:58 DEBUG ] 03:40:58 DEBUG } 03:40:58 DEBUG ] 03:40:58 DEBUG } 03:40:58 DEBUG } 03:40:58 INFO 03:40:58 INFO [loop_until]: kubectl --namespace=xlou top node 03:40:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 03:40:58 INFO [loop_until]: OK (rc = 0) 03:40:58 DEBUG --- stdout --- 03:40:58 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 67m 0% 6285Mi 5% gke-xlou-cdm-default-pool-1b0769c8-mpfw 70m 0% 7761Mi 6% gke-xlou-cdm-default-pool-1b0769c8-nhlf 541m 1% 11515Mi 10% gke-xlou-cdm-default-pool-1b0769c8-qkns 79m 0% 3355Mi 3% gke-xlou-cdm-default-pool-1b0769c8-w15q 11066m 37% 15005Mi 13% gke-xlou-cdm-default-pool-1b0769c8-wpwl 151m 0% 5257Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 591m 1% 11984Mi 10% gke-xlou-cdm-ds-0f601751-cf7b 77m 0% 3587Mi 3% gke-xlou-cdm-ds-0f601751-hmm7 570m 1% 11055Mi 9% gke-xlou-cdm-ds-0f601751-j488 65m 0% 7360Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 78m 0% 17589Mi 15% gke-xlou-cdm-ds-0f601751-nj6q 90m 0% 8004Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 18114m 60% 15251Mi 13% gke-xlou-cdm-frontend-a8771548-i82n 877m 5% 12714Mi 21% 03:40:58 DEBUG --- stderr --- 03:40:58 DEBUG 03:40:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713667258" 03:40:58 INFO [http_cmd]: http status code OK 03:40:58 DEBUG --- status code --- 03:40:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:40:58 DEBUG --- http response --- 03:40:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:40:58 DEBUG ----- output ----- 03:40:58 DEBUG { 03:40:58 DEBUG "status": "success", 03:40:58 DEBUG "data": { 03:40:58 DEBUG "resultType": "vector", 03:40:58 DEBUG "result": [ 03:40:58 DEBUG { 03:40:58 DEBUG "metric": { 03:40:58 DEBUG "node": "gke-xlou-cdm-ds-0f601751-cf7b" 03:40:58 DEBUG }, 03:40:58 DEBUG "value": [ 03:40:58 DEBUG 03:40:58 DEBUG 03:40:58 DEBUG 03:40:58 DEBUG [...] 03:40:58 DEBUG 03:40:58 DEBUG 03:40:58 DEBUG 03:40:58 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 03:40:58 DEBUG }, 03:40:58 DEBUG "value": [ 03:40:58 DEBUG 1713667258, 03:40:58 DEBUG "2413744128" 03:40:58 DEBUG ] 03:40:58 DEBUG } 03:40:58 DEBUG ] 03:40:58 DEBUG } 03:40:58 DEBUG } 03:40:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667259" 03:40:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667259" 03:40:59 INFO [http_cmd]: http status code OK 03:40:59 DEBUG --- status code --- 03:40:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:40:59 DEBUG --- http response --- 03:40:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:40:59 DEBUG ----- output ----- 03:40:59 DEBUG { 03:40:59 DEBUG "status": "success", 03:40:59 DEBUG "data": { 03:40:59 DEBUG "resultType": "vector", 03:40:59 DEBUG "result": [ 03:40:59 DEBUG { 03:40:59 DEBUG "metric": { 03:40:59 DEBUG "pod": "ds-idrepo-0" 03:40:59 DEBUG }, 03:40:59 DEBUG "value": [ 03:40:59 DEBUG 03:40:59 DEBUG 03:40:59 DEBUG 03:40:59 DEBUG [...] 03:40:59 DEBUG 03:40:59 DEBUG 03:40:59 DEBUG 03:40:59 DEBUG "pod": "ds-idrepo-2" 03:40:59 DEBUG }, 03:40:59 DEBUG "value": [ 03:40:59 DEBUG 1713667259, 03:40:59 DEBUG "0" 03:40:59 DEBUG ] 03:40:59 DEBUG } 03:40:59 DEBUG ] 03:40:59 DEBUG } 03:40:59 DEBUG } 03:40:59 INFO [http_cmd]: http status code OK 03:40:59 DEBUG --- status code --- 03:40:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:40:59 DEBUG --- http response --- 03:40:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:40:59 DEBUG ----- output ----- 03:40:59 DEBUG { 03:40:59 DEBUG "status": "success", 03:40:59 DEBUG "data": { 03:40:59 DEBUG "resultType": "vector", 03:40:59 DEBUG "result": [ 03:40:59 DEBUG { 03:40:59 DEBUG "metric": { 03:40:59 DEBUG "pod": "ds-idrepo-0" 03:40:59 DEBUG }, 03:40:59 DEBUG "value": [ 03:40:59 DEBUG 03:40:59 DEBUG 03:40:59 DEBUG 03:40:59 DEBUG [...] 03:40:59 DEBUG 03:40:59 DEBUG 03:40:59 DEBUG 03:40:59 DEBUG "pod": "ds-idrepo-2" 03:40:59 DEBUG }, 03:40:59 DEBUG "value": [ 03:40:59 DEBUG 1713667259, 03:40:59 DEBUG "0" 03:40:59 DEBUG ] 03:40:59 DEBUG } 03:40:59 DEBUG ] 03:40:59 DEBUG } 03:40:59 DEBUG } 03:40:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713667259" 03:40:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667259" 03:40:59 INFO [http_cmd]: http status code OK 03:40:59 DEBUG --- status code --- 03:40:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:40:59 DEBUG --- http response --- 03:40:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:40:59 DEBUG ----- output ----- 03:40:59 DEBUG { 03:40:59 DEBUG "status": "success", 03:40:59 DEBUG "data": { 03:40:59 DEBUG "resultType": "vector", 03:40:59 DEBUG "result": [ 03:40:59 DEBUG { 03:40:59 DEBUG "metric": { 03:40:59 DEBUG "instance": "10.142.0.107:9100" 03:40:59 DEBUG }, 03:40:59 DEBUG "value": [ 03:40:59 DEBUG 03:40:59 DEBUG 03:40:59 DEBUG 03:40:59 DEBUG [...] 03:40:59 DEBUG 03:40:59 DEBUG 03:40:59 DEBUG 03:40:59 DEBUG "instance": "10.142.0.43:9100" 03:40:59 DEBUG }, 03:40:59 DEBUG "value": [ 03:40:59 DEBUG 1713667259, 03:40:59 DEBUG "136409.26666666666" 03:40:59 DEBUG ] 03:40:59 DEBUG } 03:40:59 DEBUG ] 03:40:59 DEBUG } 03:40:59 DEBUG } 03:40:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667259" 03:40:59 INFO [http_cmd]: http status code OK 03:40:59 DEBUG --- status code --- 03:40:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:40:59 DEBUG --- http response --- 03:40:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:40:59 DEBUG ----- output ----- 03:40:59 DEBUG { 03:40:59 DEBUG "status": "success", 03:40:59 DEBUG "data": { 03:40:59 DEBUG "resultType": "vector", 03:40:59 DEBUG "result": [ 03:40:59 DEBUG { 03:40:59 DEBUG "metric": { 03:40:59 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:40:59 DEBUG }, 03:40:59 DEBUG "value": [ 03:40:59 DEBUG 03:40:59 DEBUG 03:40:59 DEBUG 03:40:59 DEBUG [...] 03:40:59 DEBUG 03:40:59 DEBUG 03:40:59 DEBUG 03:40:59 DEBUG "pod": "idm-b85f9d978-wnhxb" 03:40:59 DEBUG }, 03:40:59 DEBUG "value": [ 03:40:59 DEBUG 1713667259, 03:40:59 DEBUG "1229.2479255537744" 03:40:59 DEBUG ] 03:40:59 DEBUG } 03:40:59 DEBUG ] 03:40:59 DEBUG } 03:40:59 DEBUG } 03:40:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713667259" 03:40:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713667259" 03:40:59 INFO [http_cmd]: http status code OK 03:40:59 DEBUG --- status code --- 03:40:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:40:59 DEBUG --- http response --- 03:40:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:40:59 DEBUG ----- output ----- 03:40:59 DEBUG { 03:40:59 DEBUG "status": "success", 03:40:59 DEBUG "data": { 03:40:59 DEBUG "resultType": "vector", 03:40:59 DEBUG "result": [ 03:40:59 DEBUG { 03:40:59 DEBUG "metric": { 03:40:59 DEBUG "pod": "admin-ui-c4c7bf584-tlgkc" 03:40:59 DEBUG }, 03:40:59 DEBUG "value": [ 03:40:59 DEBUG 03:40:59 DEBUG 03:40:59 DEBUG 03:40:59 DEBUG [...] 03:40:59 DEBUG 03:40:59 DEBUG 03:40:59 DEBUG 03:40:59 DEBUG "pod": "idm-b85f9d978-wnhxb" 03:40:59 DEBUG }, 03:40:59 DEBUG "value": [ 03:40:59 DEBUG 1713667259, 03:40:59 DEBUG "0" 03:40:59 DEBUG ] 03:40:59 DEBUG } 03:40:59 DEBUG ] 03:40:59 DEBUG } 03:40:59 DEBUG } 03:40:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667259" 03:40:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713667259" 03:40:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667259" 03:40:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713667259" 03:40:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713667259" 03:40:59 INFO [http_cmd]: http status code OK 03:40:59 INFO [http_cmd]: http status code OK 03:40:59 DEBUG --- status code --- 03:40:59 DEBUG --- status code --- 03:40:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:40:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:40:59 DEBUG --- http response --- 03:40:59 DEBUG --- http response --- 03:40:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:40:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:40:59 DEBUG ----- output ----- 03:40:59 INFO [http_cmd]: http status code OK 03:40:59 DEBUG ----- output ----- 03:40:59 DEBUG { 03:40:59 DEBUG --- status code --- 03:40:59 DEBUG { 03:40:59 DEBUG "status": "success", 03:40:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:40:59 DEBUG "status": "success", 03:40:59 DEBUG "data": { 03:40:59 DEBUG --- http response --- 03:40:59 DEBUG "data": { 03:40:59 DEBUG "resultType": "vector", 03:40:59 DEBUG "resultType": "vector", 03:40:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:40:59 DEBUG "result": [ 03:40:59 DEBUG "result": [ 03:40:59 DEBUG ----- output ----- 03:40:59 DEBUG { 03:40:59 DEBUG { 03:40:59 DEBUG { 03:40:59 DEBUG "metric": { 03:40:59 DEBUG "metric": { 03:40:59 DEBUG "status": "success", 03:40:59 DEBUG "pod": "am-767f6b67b5-9zw22" 03:40:59 DEBUG "pod": "ds-idrepo-0" 03:40:59 DEBUG "data": { 03:40:59 DEBUG }, 03:40:59 DEBUG }, 03:40:59 DEBUG "resultType": "vector", 03:40:59 DEBUG "value": [ 03:40:59 DEBUG "value": [ 03:40:59 INFO [http_cmd]: http status code OK 03:40:59 DEBUG "result": [ 03:40:59 DEBUG 03:40:59 DEBUG 03:40:59 DEBUG --- status code --- 03:40:59 DEBUG { 03:40:59 DEBUG 03:40:59 DEBUG 03:40:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:40:59 DEBUG "metric": { 03:40:59 DEBUG 03:40:59 DEBUG 03:40:59 DEBUG --- http response --- 03:40:59 DEBUG "pod": "am-767f6b67b5-9zw22" 03:40:59 DEBUG [...] 03:40:59 INFO [http_cmd]: http status code OK 03:40:59 DEBUG [...] 03:40:59 DEBUG }, 03:40:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:40:59 DEBUG 03:40:59 DEBUG --- status code --- 03:40:59 DEBUG 03:40:59 DEBUG "value": [ 03:40:59 DEBUG ----- output ----- 03:40:59 DEBUG 03:40:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:40:59 DEBUG 03:40:59 DEBUG 03:40:59 DEBUG { 03:40:59 DEBUG 03:40:59 DEBUG --- http response --- 03:40:59 DEBUG 03:40:59 DEBUG 03:40:59 DEBUG "status": "success", 03:40:59 DEBUG "pod": "am-767f6b67b5-vbltz" 03:40:59 INFO [http_cmd]: http status code OK 03:40:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:40:59 DEBUG "pod": "ds-cts-2" 03:40:59 DEBUG 03:40:59 DEBUG "data": { 03:40:59 DEBUG }, 03:40:59 DEBUG --- status code --- 03:40:59 DEBUG ----- output ----- 03:40:59 DEBUG }, 03:40:59 DEBUG [...] 03:40:59 DEBUG "resultType": "vector", 03:40:59 INFO [http_cmd]: http status code OK 03:40:59 DEBUG "value": [ 03:40:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:40:59 DEBUG { 03:40:59 DEBUG "value": [ 03:40:59 DEBUG 03:40:59 DEBUG "result": [ 03:40:59 DEBUG --- status code --- 03:40:59 DEBUG 1713667259, 03:40:59 DEBUG --- http response --- 03:40:59 DEBUG "status": "success", 03:40:59 DEBUG 1713667259, 03:40:59 DEBUG 03:40:59 DEBUG { 03:40:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:40:59 DEBUG "196.295" 03:40:59 DEBUG "data": { 03:40:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:40:59 DEBUG "0" 03:40:59 DEBUG 03:40:59 DEBUG "metric": { 03:40:59 DEBUG --- http response --- 03:40:59 DEBUG ] 03:40:59 DEBUG "resultType": "vector", 03:40:59 DEBUG ----- output ----- 03:40:59 DEBUG ] 03:40:59 DEBUG "pod": "am-767f6b67b5-vbltz" 03:40:59 DEBUG "pod": "ds-cts-0" 03:40:59 DEBUG } 03:40:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 03:40:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667259" 03:40:59 DEBUG "result": [ 03:40:59 DEBUG { 03:40:59 DEBUG } 03:40:59 DEBUG }, 03:40:59 DEBUG }, 03:40:59 DEBUG ] 03:40:59 DEBUG ----- output ----- 03:40:59 DEBUG { 03:40:59 DEBUG "status": "success", 03:40:59 DEBUG ] 03:40:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713667259" 03:40:59 DEBUG "value": [ 03:40:59 DEBUG "value": [ 03:40:59 DEBUG } 03:40:59 DEBUG { 03:40:59 DEBUG "metric": { 03:40:59 DEBUG "data": { 03:40:59 DEBUG } 03:40:59 DEBUG 1713667259, 03:40:59 DEBUG 03:40:59 DEBUG } 03:40:59 DEBUG "status": "success", 03:40:59 DEBUG "instance": "10.142.0.107:9100" 03:40:59 DEBUG "resultType": "vector", 03:40:59 DEBUG } 03:40:59 DEBUG "0" 03:40:59 DEBUG 03:40:59 DEBUG "data": { 03:40:59 DEBUG }, 03:40:59 DEBUG "result": [ 03:40:59 DEBUG ] 03:40:59 DEBUG 03:40:59 DEBUG "resultType": "vector", 03:40:59 DEBUG "value": [ 03:40:59 DEBUG { 03:40:59 DEBUG } 03:40:59 DEBUG [...] 03:40:59 DEBUG "result": [ 03:40:59 DEBUG 03:40:59 DEBUG "metric": { 03:40:59 DEBUG ] 03:40:59 DEBUG 03:40:59 DEBUG { 03:40:59 DEBUG 03:40:59 DEBUG "instance": "10.142.0.107:9100" 03:40:59 DEBUG } 03:40:59 DEBUG 03:40:59 DEBUG "metric": { 03:40:59 DEBUG 03:40:59 DEBUG }, 03:40:59 DEBUG } 03:40:59 DEBUG 03:40:59 DEBUG "instance": "10.142.0.107:9100" 03:40:59 DEBUG [...] 03:40:59 DEBUG "value": [ 03:40:59 DEBUG "pod": "ds-cts-2" 03:40:59 DEBUG }, 03:40:59 DEBUG 03:40:59 DEBUG 03:40:59 DEBUG }, 03:40:59 DEBUG "value": [ 03:40:59 DEBUG 03:40:59 DEBUG 03:40:59 DEBUG "value": [ 03:40:59 DEBUG 03:40:59 DEBUG 03:40:59 DEBUG 03:40:59 DEBUG 1713667259, 03:40:59 DEBUG 03:40:59 DEBUG "instance": "10.142.0.43:9100" 03:40:59 DEBUG [...] 03:40:59 DEBUG "0" 03:40:59 DEBUG 03:40:59 DEBUG }, 03:40:59 DEBUG 03:40:59 DEBUG ] 03:40:59 DEBUG [...] 03:40:59 DEBUG "value": [ 03:40:59 DEBUG 03:40:59 DEBUG } 03:40:59 DEBUG 03:40:59 DEBUG 1713667259, 03:40:59 DEBUG 03:40:59 DEBUG ] 03:40:59 DEBUG 03:40:59 DEBUG "0.0003333333333330302" 03:40:59 DEBUG "instance": "10.142.0.43:9100" 03:40:59 DEBUG } 03:40:59 DEBUG 03:40:59 DEBUG ] 03:40:59 DEBUG }, 03:40:59 DEBUG } 03:40:59 DEBUG "instance": "10.142.0.43:9100" 03:40:59 DEBUG } 03:40:59 DEBUG "value": [ 03:40:59 DEBUG }, 03:40:59 DEBUG ] 03:40:59 DEBUG 1713667259, 03:40:59 DEBUG "value": [ 03:40:59 DEBUG } 03:40:59 DEBUG "4488800381952" 03:40:59 INFO [http_cmd]: http status code OK 03:40:59 INFO [http_cmd]: http status code OK 03:40:59 DEBUG 1713667259, 03:40:59 DEBUG } 03:40:59 DEBUG ] 03:40:59 DEBUG --- status code --- 03:40:59 DEBUG --- status code --- 03:40:59 DEBUG "131004.4" 03:40:59 DEBUG } 03:40:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:40:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 03:40:59 DEBUG ] 03:40:59 DEBUG ] 03:40:59 DEBUG --- http response --- 03:40:59 DEBUG --- http response --- 03:40:59 DEBUG } 03:40:59 DEBUG } 03:40:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 03:40:59 DEBUG ] 03:40:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 03:40:59 DEBUG } 03:40:59 DEBUG ----- output ----- 03:40:59 DEBUG } 03:40:59 DEBUG ----- output ----- 03:40:59 DEBUG { 03:40:59 DEBUG } 03:40:59 DEBUG { 03:40:59 DEBUG "status": "success", 03:40:59 DEBUG "status": "success", 03:40:59 DEBUG "data": { 03:40:59 DEBUG "data": { 03:40:59 DEBUG "resultType": "vector", 03:40:59 DEBUG "resultType": "vector", 03:40:59 DEBUG "result": [ 03:40:59 DEBUG "result": [ 03:40:59 DEBUG { 03:40:59 DEBUG { 03:40:59 DEBUG "metric": { 03:40:59 DEBUG "metric": { 03:40:59 DEBUG "pod": "am-767f6b67b5-9zw22" 03:40:59 DEBUG "pod": "ds-cts-0" 03:40:59 DEBUG }, 03:40:59 DEBUG }, 03:40:59 DEBUG "value": [ 03:40:59 DEBUG "value": [ 03:40:59 DEBUG 03:40:59 DEBUG 03:40:59 DEBUG 03:40:59 DEBUG 03:40:59 DEBUG 03:40:59 DEBUG 03:40:59 DEBUG [...] 03:40:59 DEBUG [...] 03:40:59 DEBUG 03:40:59 DEBUG 03:40:59 DEBUG 03:40:59 DEBUG 03:40:59 DEBUG 03:40:59 DEBUG 03:40:59 DEBUG "pod": "am-767f6b67b5-vbltz" 03:40:59 DEBUG "pod": "ds-cts-2" 03:40:59 DEBUG }, 03:40:59 DEBUG }, 03:40:59 DEBUG "value": [ 03:40:59 DEBUG "value": [ 03:40:59 DEBUG 1713667259, 03:40:59 DEBUG 1713667259, 03:40:59 DEBUG "0.03333333333333333" 03:40:59 DEBUG "29.833333333333332" 03:40:59 DEBUG ] 03:40:59 DEBUG ] 03:40:59 DEBUG } 03:40:59 DEBUG } 03:40:59 DEBUG ] 03:40:59 DEBUG ] 03:40:59 DEBUG } 03:40:59 DEBUG } 03:40:59 DEBUG } 03:40:59 DEBUG } 03:41:10 INFO Finished: True 03:41:10 INFO Waiting for threads to register finish flag 03:41:59 INFO Done. Have a nice day! :) 127.0.0.1 - - [21/Apr/2024 03:41:59] "GET /monitoring/stop HTTP/1.1" 200 - 03:42:02 WARNING File /tmp/lodemon_data-20240421_025941/node_disk_read_bytes_total.json does not exist. Skipping graph generation... WARNING:bokeh.core.validation.check:W-1000 (MISSING_RENDERERS): Plot has no renderers: figure(id='p6481', ...) 127.0.0.1 - - [21/Apr/2024 03:42:06] "GET /monitoring/process HTTP/1.1" 200 -