==================================================================================================== ========================================= Pod describe ========================================= ==================================================================================================== Name: lodemon-bc756c647-cr7x6 Namespace: xlou Priority: 0 Service Account: k8s-svc-acct-lodemon Node: gke-xlou-cdm-ds-0f601751-mc5d/10.142.0.43 Start Time: Tue, 16 Apr 2024 21:09:43 +0000 Labels: app=lodemon app.kubernetes.io/name=lodemon pod-template-hash=bc756c647 skaffold.dev/run-id=2d42b4a1-cfdb-4fba-a228-98f02b0b4ec6 Annotations: Status: Running IP: 10.106.46.155 IPs: IP: 10.106.46.155 Controlled By: ReplicaSet/lodemon-bc756c647 Containers: lodemon: Container ID: containerd://c4e0ae439e3b622debefcb6503e5e30aa62f7924ba168ccc1db7b609c0ad634c Image: gcr.io/engineeringpit/lodestar-images/lodestarbox:master-stable Image ID: gcr.io/engineeringpit/lodestar-images/lodestarbox@sha256:8ebc4b3f3093af7a1156c744d7231c3add77b4e994da0c73152ecc0b152e1dd6 Port: 8080/TCP Host Port: 0/TCP Command: python3 Args: /lodestar/scripts/lodemon_run.py -W default State: Running Started: Tue, 16 Apr 2024 21:09:44 +0000 Ready: True Restart Count: 0 Limits: cpu: 2 memory: 2Gi Requests: cpu: 1 memory: 1Gi Liveness: exec [cat /tmp/lodemon_alive] delay=20s timeout=1s period=10s #success=1 #failure=3 Readiness: exec [cat /tmp/lodemon_alive] delay=20s timeout=1s period=10s #success=1 #failure=3 Environment: SKAFFOLD_PROFILE: medium Mounts: /lodestar/config/config.yaml from config (rw,path="config.yaml") /lodestar/config/deployments_to_monitor.yaml from deployments-config (rw,path="deployments_to_monitor.yaml") /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-6h97s (ro) Conditions: Type Status Initialized True Ready True ContainersReady True PodScheduled True Volumes: config: Type: ConfigMap (a volume populated by a ConfigMap) Name: lodemon-config Optional: false deployments-config: Type: ConfigMap (a volume populated by a ConfigMap) Name: lodemon-deployments Optional: false kube-api-access-6h97s: Type: Projected (a volume that contains injected data from multiple sources) TokenExpirationSeconds: 3607 ConfigMapName: kube-root-ca.crt ConfigMapOptional: DownwardAPI: true QoS Class: Burstable Node-Selectors: Tolerations: node.kubernetes.io/not-ready:NoExecute op=Exists for 300s node.kubernetes.io/unreachable:NoExecute op=Exists for 300s Events: ==================================================================================================== =========================================== Pod logs =========================================== ==================================================================================================== * Serving Flask app 'lodemon_run' * Debug mode: off WARNING: This is a development server. Do not use it in a production deployment. Use a production WSGI server instead. * Running on all addresses (0.0.0.0) * Running on http://127.0.0.1:8080 * Running on http://10.106.46.155:8080 Press CTRL+C to quit 22:10:50 INFO 22:10:50 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 22:10:50 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:10:50 INFO [loop_until]: OK (rc = 0) 22:10:50 DEBUG --- stdout --- 22:10:50 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 22:10:50 DEBUG --- stderr --- 22:10:50 DEBUG 22:10:50 INFO 22:10:50 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 22:10:50 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:10:50 INFO [loop_until]: OK (rc = 0) 22:10:50 DEBUG --- stdout --- 22:10:50 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 22:10:50 DEBUG --- stderr --- 22:10:50 DEBUG 22:10:50 INFO 22:10:50 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 22:10:50 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:10:50 INFO [loop_until]: OK (rc = 0) 22:10:50 DEBUG --- stdout --- 22:10:50 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 22:10:50 DEBUG --- stderr --- 22:10:50 DEBUG 22:10:50 INFO 22:10:50 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 22:10:50 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:10:51 INFO [loop_until]: OK (rc = 0) 22:10:51 DEBUG --- stdout --- 22:10:51 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 22:10:51 DEBUG --- stderr --- 22:10:51 DEBUG 22:10:51 INFO 22:10:51 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 22:10:51 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:10:51 INFO [loop_until]: OK (rc = 0) 22:10:51 DEBUG --- stdout --- 22:10:51 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 22:10:51 DEBUG --- stderr --- 22:10:51 DEBUG 22:10:51 INFO 22:10:51 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 22:10:51 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:10:51 INFO [loop_until]: OK (rc = 0) 22:10:51 DEBUG --- stdout --- 22:10:51 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 22:10:51 DEBUG --- stderr --- 22:10:51 DEBUG 22:10:51 INFO 22:10:51 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 22:10:51 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:10:51 INFO [loop_until]: OK (rc = 0) 22:10:51 DEBUG --- stdout --- 22:10:51 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 22:10:51 DEBUG --- stderr --- 22:10:51 DEBUG 22:10:51 INFO 22:10:51 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 22:10:51 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:10:51 INFO [loop_until]: OK (rc = 0) 22:10:51 DEBUG --- stdout --- 22:10:51 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 22:10:51 DEBUG --- stderr --- 22:10:51 DEBUG 22:10:51 INFO 22:10:51 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 22:10:51 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:10:51 INFO [loop_until]: OK (rc = 0) 22:10:51 DEBUG --- stdout --- 22:10:51 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 22:10:51 DEBUG --- stderr --- 22:10:51 DEBUG 22:10:51 INFO 22:10:51 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 22:10:51 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:10:51 INFO [loop_until]: OK (rc = 0) 22:10:51 DEBUG --- stdout --- 22:10:51 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 22:10:51 DEBUG --- stderr --- 22:10:51 DEBUG 22:10:51 INFO 22:10:51 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 22:10:51 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:10:51 INFO [loop_until]: OK (rc = 0) 22:10:51 DEBUG --- stdout --- 22:10:51 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 22:10:51 DEBUG --- stderr --- 22:10:51 DEBUG 22:10:52 INFO 22:10:52 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 22:10:52 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:10:52 INFO [loop_until]: OK (rc = 0) 22:10:52 DEBUG --- stdout --- 22:10:52 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 22:10:52 DEBUG --- stderr --- 22:10:52 DEBUG 22:10:52 INFO 22:10:52 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 22:10:52 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:10:52 INFO [loop_until]: OK (rc = 0) 22:10:52 DEBUG --- stdout --- 22:10:52 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 22:10:52 DEBUG --- stderr --- 22:10:52 DEBUG 22:10:52 INFO 22:10:52 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 22:10:52 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:10:52 INFO [loop_until]: OK (rc = 0) 22:10:52 DEBUG --- stdout --- 22:10:52 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 22:10:52 DEBUG --- stderr --- 22:10:52 DEBUG 22:10:52 INFO 22:10:52 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 22:10:52 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:10:52 INFO [loop_until]: OK (rc = 0) 22:10:52 DEBUG --- stdout --- 22:10:52 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 22:10:52 DEBUG --- stderr --- 22:10:52 DEBUG 22:10:52 INFO 22:10:52 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 22:10:52 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:10:52 INFO [loop_until]: OK (rc = 0) 22:10:52 DEBUG --- stdout --- 22:10:52 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 22:10:52 DEBUG --- stderr --- 22:10:52 DEBUG 22:10:52 INFO 22:10:52 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 22:10:52 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:10:52 INFO [loop_until]: OK (rc = 0) 22:10:52 DEBUG --- stdout --- 22:10:52 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 22:10:52 DEBUG --- stderr --- 22:10:52 DEBUG 22:10:52 INFO 22:10:52 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 22:10:52 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:10:52 INFO [loop_until]: OK (rc = 0) 22:10:52 DEBUG --- stdout --- 22:10:52 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 22:10:52 DEBUG --- stderr --- 22:10:52 DEBUG 22:10:52 INFO 22:10:52 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 22:10:52 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:10:53 INFO [loop_until]: OK (rc = 0) 22:10:53 DEBUG --- stdout --- 22:10:53 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 22:10:53 DEBUG --- stderr --- 22:10:53 DEBUG 22:10:53 INFO 22:10:53 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 22:10:53 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:10:53 INFO [loop_until]: OK (rc = 0) 22:10:53 DEBUG --- stdout --- 22:10:53 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 22:10:53 DEBUG --- stderr --- 22:10:53 DEBUG 22:10:53 INFO 22:10:53 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 22:10:53 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:10:53 INFO [loop_until]: OK (rc = 0) 22:10:53 DEBUG --- stdout --- 22:10:53 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 22:10:53 DEBUG --- stderr --- 22:10:53 DEBUG 22:10:53 INFO 22:10:53 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 22:10:53 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:10:53 INFO [loop_until]: OK (rc = 0) 22:10:53 DEBUG --- stdout --- 22:10:53 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 22:10:53 DEBUG --- stderr --- 22:10:53 DEBUG 22:10:53 INFO 22:10:53 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 22:10:53 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:10:53 INFO [loop_until]: OK (rc = 0) 22:10:53 DEBUG --- stdout --- 22:10:53 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 22:10:53 DEBUG --- stderr --- 22:10:53 DEBUG 22:10:53 INFO 22:10:53 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 22:10:53 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:10:53 INFO [loop_until]: OK (rc = 0) 22:10:53 DEBUG --- stdout --- 22:10:53 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 22:10:53 DEBUG --- stderr --- 22:10:53 DEBUG 22:10:53 INFO 22:10:53 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 22:10:53 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:10:53 INFO [loop_until]: OK (rc = 0) 22:10:53 DEBUG --- stdout --- 22:10:53 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 22:10:53 DEBUG --- stderr --- 22:10:53 DEBUG 22:10:53 INFO 22:10:53 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 22:10:53 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:10:53 INFO [loop_until]: OK (rc = 0) 22:10:53 DEBUG --- stdout --- 22:10:53 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 22:10:53 DEBUG --- stderr --- 22:10:53 DEBUG 22:10:54 INFO 22:10:54 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 22:10:54 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:10:54 INFO [loop_until]: OK (rc = 0) 22:10:54 DEBUG --- stdout --- 22:10:54 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 22:10:54 DEBUG --- stderr --- 22:10:54 DEBUG 22:10:54 INFO Initializing monitoring instance threads 22:10:54 DEBUG Monitoring instance thread list: [, , , , , , , , , , , , , , , , , , , , , , , , , , , , ] 22:10:54 INFO Starting instance threads 22:10:54 INFO 22:10:54 INFO Thread started 22:10:54 INFO [loop_until]: kubectl --namespace=xlou top node 22:10:54 INFO 22:10:54 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:10:54 INFO Thread started 22:10:54 INFO [loop_until]: kubectl --namespace=xlou top pods 22:10:54 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:10:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713301854" 22:10:54 INFO Thread started 22:10:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713301854" 22:10:54 INFO Thread started 22:10:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713301854" 22:10:54 INFO Thread started 22:10:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713301854" 22:10:54 INFO Thread started 22:10:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713301854" 22:10:54 INFO Thread started 22:10:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713301854" 22:10:54 INFO Thread started 22:10:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713301854" 22:10:54 INFO Thread started 22:10:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713301854" 22:10:54 INFO Thread started 22:10:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713301854" 22:10:54 INFO Thread started 22:10:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713301854" 22:10:54 INFO Thread started 22:10:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713301854" 22:10:54 INFO Thread started 22:10:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713301854" 22:10:54 INFO [http_cmd]: http status code OK 22:10:54 INFO Thread started 22:10:54 DEBUG --- status code --- 22:10:54 INFO [http_cmd]: http status code OK 22:10:54 DEBUG --- status code --- 22:10:54 INFO [http_cmd]: http status code OK 22:10:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713301854" 22:10:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:10:54 INFO Thread started 22:10:54 INFO [http_cmd]: http status code OK 22:10:54 INFO [http_cmd]: http status code OK 22:10:54 DEBUG --- status code --- 22:10:54 INFO [http_cmd]: http status code OK 22:10:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:10:54 DEBUG --- status code --- 22:10:54 DEBUG --- http response --- 22:10:54 DEBUG --- status code --- 22:10:54 INFO [http_cmd]: http status code OK 22:10:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713301854" 22:10:54 INFO [http_cmd]: http status code OK 22:10:54 DEBUG --- status code --- 22:10:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:10:54 INFO Thread started 22:10:54 DEBUG --- status code --- 22:10:54 DEBUG --- http response --- 22:10:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:10:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:10:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:10:54 DEBUG --- status code --- 22:10:54 INFO [http_cmd]: http status code OK 22:10:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:10:54 INFO [http_cmd]: http status code OK 22:10:54 DEBUG --- http response --- 22:10:54 INFO [http_cmd]: http status code OK 22:10:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713301854" 22:10:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:10:54 INFO Thread started 22:10:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:10:54 DEBUG --- http response --- 22:10:54 INFO [http_cmd]: http status code OK 22:10:54 DEBUG ----- output ----- 22:10:54 DEBUG --- http response --- 22:10:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:10:54 DEBUG --- status code --- 22:10:54 DEBUG --- http response --- 22:10:54 DEBUG --- status code --- 22:10:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:10:54 DEBUG --- status code --- 22:10:54 INFO [http_cmd]: http status code OK 22:10:54 DEBUG --- http response --- 22:10:54 DEBUG ----- output ----- 22:10:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713301854" 22:10:54 INFO Thread started 22:10:54 DEBUG --- status code --- 22:10:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 22:10:54 DEBUG { 22:10:54 DEBUG --- http response --- 22:10:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 22:10:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:10:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:10:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:10:54 DEBUG ----- output ----- 22:10:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:10:54 DEBUG --- status code --- 22:10:54 DEBUG { 22:10:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:10:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713301854" 22:10:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:10:54 INFO Thread started 22:10:54 DEBUG ----- output ----- 22:10:54 DEBUG "status": "success", 22:10:54 DEBUG ----- output ----- 22:10:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:10:54 INFO [http_cmd]: http status code OK 22:10:54 DEBUG --- http response --- 22:10:54 DEBUG ----- output ----- 22:10:54 DEBUG --- http response --- 22:10:54 DEBUG { 22:10:54 DEBUG "status": "success", 22:10:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:10:54 DEBUG "status": "success", 22:10:54 DEBUG ----- output ----- 22:10:54 DEBUG --- http response --- 22:10:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713301854" 22:10:54 DEBUG { 22:10:54 INFO Thread started 22:10:54 DEBUG "data": { 22:10:54 DEBUG { 22:10:54 DEBUG ----- output ----- 22:10:54 DEBUG --- status code --- 22:10:54 DEBUG { 22:10:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:10:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:10:54 DEBUG --- http response --- 22:10:54 DEBUG "data": { 22:10:54 DEBUG --- http response --- 22:10:54 DEBUG "data": { 22:10:54 DEBUG { 22:10:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:10:54 DEBUG "status": "success", 22:10:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713301854" 22:10:54 INFO [http_cmd]: http status code OK 22:10:54 DEBUG --- status code --- 22:10:54 DEBUG "resultType": "vector", 22:10:54 DEBUG "status": "success", 22:10:54 DEBUG { 22:10:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:10:54 DEBUG "status": "success", 22:10:54 DEBUG ----- output ----- 22:10:54 DEBUG ----- output ----- 22:10:54 DEBUG "resultType": "vector", 22:10:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:10:54 DEBUG "resultType": "vector", 22:10:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:10:54 INFO [http_cmd]: http status code OK 22:10:54 DEBUG "status": "success", 22:10:54 DEBUG ----- output ----- 22:10:54 DEBUG "data": { 22:10:54 INFO Thread started 22:10:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:10:54 DEBUG "result": [ 22:10:54 DEBUG "data": { 22:10:54 DEBUG "status": "success", 22:10:54 INFO [loop_until]: OK (rc = 0) 22:10:54 DEBUG --- http response --- 22:10:54 DEBUG "data": { 22:10:54 DEBUG { 22:10:54 DEBUG { 22:10:54 DEBUG "result": [ 22:10:54 DEBUG ----- output ----- 22:10:54 DEBUG "result": [ 22:10:54 DEBUG ----- output ----- 22:10:54 DEBUG --- status code --- 22:10:54 DEBUG "data": { 22:10:54 DEBUG { 22:10:54 DEBUG "resultType": "vector", 22:10:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713301854" 22:10:54 DEBUG --- http response --- 22:10:54 INFO Thread started 22:10:54 DEBUG { 22:10:54 INFO [http_cmd]: http status code OK 22:10:54 DEBUG "resultType": "vector", 22:10:54 DEBUG "data": { 22:10:54 DEBUG --- stdout --- 22:10:54 DEBUG "resultType": "vector", 22:10:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:10:54 DEBUG "status": "success", 22:10:54 DEBUG "status": "success", 22:10:54 DEBUG { 22:10:54 DEBUG { 22:10:54 DEBUG { 22:10:54 DEBUG { 22:10:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:10:54 DEBUG "resultType": "vector", 22:10:54 DEBUG "status": "success", 22:10:54 DEBUG "result": [ 22:10:54 DEBUG { 22:10:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713301854" 22:10:54 DEBUG "metric": { 22:10:54 INFO Thread started 22:10:54 DEBUG --- status code --- 22:10:54 DEBUG "result": [ 22:10:54 DEBUG "resultType": "vector", 22:10:54 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 9Mi am-555f4db595-4lbss 3m 3352Mi am-555f4db595-fz8sn 3m 3290Mi am-555f4db595-kzxsp 3m 1184Mi ds-cts-0 5m 8290Mi ds-cts-1 5m 8306Mi ds-cts-2 4m 8447Mi ds-idrepo-0 7m 5031Mi ds-idrepo-1 8m 2780Mi ds-idrepo-2 8m 2852Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 3m 1208Mi idm-b85f9d978-rl76f 4m 1226Mi lodemon-bc756c647-cr7x6 1m 54Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-7c9b667974-9mr2s 1m 15Mi 22:10:54 DEBUG "result": [ 22:10:54 DEBUG ----- output ----- 22:10:54 DEBUG "data": { 22:10:54 DEBUG "data": { 22:10:54 DEBUG "metric": { 22:10:54 DEBUG "status": "success", 22:10:54 DEBUG "metric": { 22:10:54 DEBUG "status": "success", 22:10:54 INFO [http_cmd]: http status code OK 22:10:54 DEBUG --- http response --- 22:10:54 DEBUG "result": [ 22:10:54 DEBUG "data": { 22:10:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:10:54 DEBUG "metric": { 22:10:54 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:10:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713301854" 22:10:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:10:54 INFO Thread started 22:10:54 DEBUG { 22:10:54 DEBUG "result": [ 22:10:54 DEBUG --- stderr --- 22:10:54 DEBUG { 22:10:54 DEBUG { 22:10:54 DEBUG "resultType": "vector", 22:10:54 DEBUG "resultType": "vector", 22:10:54 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:10:54 DEBUG "data": { 22:10:54 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:10:54 DEBUG "data": { 22:10:54 DEBUG --- status code --- 22:10:54 DEBUG { 22:10:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:10:54 INFO [http_cmd]: http status code OK 22:10:54 DEBUG "resultType": "vector", 22:10:54 DEBUG ----- output ----- 22:10:54 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:10:54 DEBUG }, 22:10:54 DEBUG --- http response --- 22:10:54 INFO [http_cmd]: http status code OK 22:10:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713301854" 22:10:54 DEBUG "metric": { 22:10:54 INFO Thread started 22:10:54 DEBUG { 22:10:54 DEBUG 22:10:54 DEBUG "metric": { 22:10:54 DEBUG "status": "success", 22:10:54 DEBUG "result": [ 22:10:54 DEBUG "result": [ 22:10:54 DEBUG }, 22:10:54 DEBUG "resultType": "vector", 22:10:54 DEBUG }, 22:10:54 DEBUG "resultType": "vector", 22:10:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:10:54 DEBUG "metric": { 22:10:54 DEBUG ----- output ----- 22:10:54 DEBUG --- status code --- 22:10:54 DEBUG "result": [ 22:10:54 DEBUG { 22:10:54 DEBUG }, 22:10:54 DEBUG "value": [ 22:10:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:10:54 DEBUG --- status code --- 22:10:54 INFO [loop_until]: OK (rc = 0) 22:10:54 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:10:54 DEBUG "metric": { 22:10:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713301854" 22:10:54 INFO Thread started 22:10:54 DEBUG "instance": "10.142.0.107:9100" 22:10:54 DEBUG "data": { 22:10:54 DEBUG { 22:10:54 DEBUG { 22:10:54 DEBUG "value": [ 22:10:54 DEBUG "result": [ 22:10:54 DEBUG "value": [ 22:10:54 DEBUG "result": [ 22:10:54 DEBUG --- http response --- 22:10:54 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:10:54 INFO [http_cmd]: http status code OK 22:10:54 DEBUG { 22:10:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:10:54 DEBUG { 22:10:54 DEBUG "status": "success", 22:10:54 DEBUG "value": [ 22:10:54 DEBUG 22:10:54 DEBUG ----- output ----- 22:10:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:10:54 DEBUG --- stdout --- 22:10:54 DEBUG }, 22:10:54 DEBUG "instance": "10.142.0.107:9100" 22:10:54 DEBUG }, 22:10:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713301854" 22:10:54 DEBUG "resultType": "vector", 22:10:54 INFO Thread started 22:10:54 DEBUG "metric": { 22:10:54 DEBUG "metric": { 22:10:54 DEBUG 22:10:54 DEBUG { 22:10:54 DEBUG 22:10:54 DEBUG { 22:10:54 DEBUG }, 22:10:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:10:54 DEBUG --- status code --- 22:10:54 DEBUG "status": "success", 22:10:54 DEBUG --- http response --- 22:10:54 DEBUG "metric": { 22:10:54 DEBUG "data": { 22:10:54 DEBUG 22:10:54 DEBUG 22:10:54 DEBUG { 22:10:54 DEBUG --- http response --- 22:10:54 DEBUG "value": [ 22:10:54 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 63m 0% 5088Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 63m 0% 10816Mi 9% gke-xlou-cdm-default-pool-1b0769c8-nhlf 52m 0% 5498Mi 4% gke-xlou-cdm-default-pool-1b0769c8-qkns 87m 0% 2668Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 60m 0% 6189Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 131m 0% 12979Mi 11% gke-xlou-cdm-default-pool-1b0769c8-xnmp 69m 0% 3153Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 59m 0% 5109Mi 4% gke-xlou-cdm-ds-0f601751-hmm7 64m 0% 5495Mi 4% gke-xlou-cdm-ds-0f601751-j488 54m 0% 10339Mi 9% gke-xlou-cdm-ds-0f601751-mc5d 104m 0% 3383Mi 3% gke-xlou-cdm-ds-0f601751-nj6q 85m 0% 3015Mi 2% gke-xlou-cdm-ds-0f601751-nmxm 63m 0% 7369Mi 6% gke-xlou-cdm-frontend-a8771548-i82n 65m 0% 5433Mi 9% 22:10:54 DEBUG }, 22:10:54 DEBUG "value": [ 22:10:54 INFO [http_cmd]: http status code OK 22:10:54 DEBUG "result": [ 22:10:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713301854" 22:10:54 DEBUG "instance": "10.142.0.107:9100" 22:10:54 INFO Thread started 22:10:54 DEBUG "pod": "am-555f4db595-fz8sn" 22:10:54 DEBUG 22:10:54 DEBUG "metric": { 22:10:54 INFO [http_cmd]: http status code OK 22:10:54 DEBUG 22:10:54 DEBUG "metric": { 22:10:54 DEBUG "value": [ 22:10:54 DEBUG ----- output ----- 22:10:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:10:54 DEBUG "data": { 22:10:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:10:54 DEBUG "pod": "am-555f4db595-fz8sn" 22:10:54 DEBUG "resultType": "vector", 22:10:54 DEBUG 22:10:54 DEBUG 22:10:54 DEBUG "status": "success", 22:10:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:10:54 DEBUG 22:10:54 DEBUG --- stderr --- 22:10:54 DEBUG "value": [ 22:10:54 DEBUG 22:10:54 DEBUG --- status code --- 22:10:54 DEBUG { 22:10:54 ERROR [http_cmd]: ERROR 22:10:54 DEBUG }, 22:10:54 DEBUG }, 22:10:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713301854" 22:10:54 INFO Thread started 22:10:54 DEBUG 22:10:54 DEBUG "pod": "ds-idrepo-2" 22:10:54 DEBUG --- status code --- 22:10:54 DEBUG 22:10:54 DEBUG "pod": "ds-idrepo-2" 22:10:54 DEBUG 22:10:54 DEBUG { 22:10:54 DEBUG --- http response --- 22:10:54 DEBUG "resultType": "vector", 22:10:54 DEBUG ----- output ----- 22:10:54 DEBUG }, 22:10:54 DEBUG "result": [ 22:10:54 DEBUG 22:10:54 DEBUG [...] 22:10:54 DEBUG "data": { 22:10:54 DEBUG ----- output ----- 22:10:54 DEBUG 22:10:54 DEBUG 22:10:54 DEBUG 22:10:54 DEBUG 22:10:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:10:54 DEBUG "metric": { 22:10:54 ERROR --- status code --- 22:10:54 DEBUG "value": [ 22:10:54 DEBUG "value": [ 22:10:54 INFO All threads has been started 127.0.0.1 - - [16/Apr/2024 22:10:54] "GET /monitoring/start HTTP/1.1" 200 - 22:10:54 DEBUG [...] 22:10:54 DEBUG }, 22:10:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:10:54 DEBUG [...] 22:10:54 DEBUG }, 22:10:54 DEBUG "value": [ 22:10:54 DEBUG "status": "success", 22:10:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:10:54 DEBUG "result": [ 22:10:54 DEBUG { 22:10:54 DEBUG "value": [ 22:10:54 DEBUG { 22:10:54 DEBUG [...] 22:10:54 DEBUG 22:10:54 DEBUG "resultType": "vector", 22:10:54 DEBUG { 22:10:54 INFO [http_cmd]: http status code OK 22:10:54 DEBUG 22:10:54 DEBUG 22:10:54 DEBUG 22:10:54 DEBUG --- http response --- 22:10:54 DEBUG "pod": "ds-idrepo-2" 22:10:54 ERROR http status code is 400 (expected [200, 401, 403, 404]) 22:10:54 DEBUG 22:10:54 DEBUG 22:10:54 DEBUG 22:10:54 DEBUG "value": [ 22:10:54 DEBUG 22:10:54 DEBUG 22:10:54 DEBUG 22:10:54 DEBUG 22:10:54 DEBUG "data": { 22:10:54 DEBUG ----- output ----- 22:10:54 DEBUG { 22:10:54 DEBUG "status": "success", 22:10:54 DEBUG 22:10:54 DEBUG "metric": { 22:10:54 DEBUG "pod": "ds-idrepo-2" 22:10:54 DEBUG 22:10:54 DEBUG "result": [ 22:10:54 DEBUG "status": "success", 22:10:54 DEBUG --- status code --- 22:10:54 DEBUG [...] 22:10:54 DEBUG 22:10:54 DEBUG [...] 22:10:54 INFO [http_cmd]: http status code OK 22:10:54 DEBUG }, 22:10:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:10:54 ERROR --- http response --- 22:10:54 DEBUG 22:10:54 DEBUG 22:10:54 DEBUG 22:10:54 DEBUG --- http response --- 22:10:54 DEBUG 22:10:54 DEBUG 22:10:54 DEBUG 22:10:54 DEBUG 22:10:54 DEBUG "resultType": "vector", 22:10:54 DEBUG { 22:10:54 DEBUG "metric": { 22:10:54 DEBUG "data": { 22:10:54 DEBUG 22:10:54 DEBUG 22:10:54 DEBUG }, 22:10:54 DEBUG 22:10:54 DEBUG { 22:10:54 DEBUG "data": { 22:10:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:10:54 DEBUG 22:10:54 DEBUG [...] 22:10:54 DEBUG 22:10:54 DEBUG --- status code --- 22:10:54 DEBUG "value": [ 22:10:54 INFO [http_cmd]: http status code OK 22:10:54 DEBUG ----- output ----- 22:10:54 DEBUG 22:10:54 DEBUG 22:10:54 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 22:10:54 DEBUG 22:10:54 DEBUG 22:10:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:10:54 DEBUG 22:10:54 DEBUG [...] 22:10:54 DEBUG 22:10:54 DEBUG "result": [ 22:10:54 DEBUG "status": "success", 22:10:54 DEBUG "data": { 22:10:54 DEBUG "pod": "am-555f4db595-fz8sn" 22:10:54 DEBUG 22:10:54 DEBUG 22:10:54 DEBUG "value": [ 22:10:54 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:10:54 DEBUG "metric": { 22:10:54 DEBUG "resultType": "vector", 22:10:54 DEBUG --- http response --- 22:10:54 DEBUG 22:10:54 DEBUG 22:10:54 DEBUG 22:10:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:10:54 DEBUG 22:10:54 DEBUG --- status code --- 22:10:54 DEBUG { 22:10:54 DEBUG "status": "success", 22:10:54 DEBUG [...] 22:10:54 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 22:10:54 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:10:54 DEBUG [...] 22:10:54 DEBUG ----- output ----- 22:10:54 DEBUG "pod": "overseer-0-7c9b667974-9mr2s" 22:10:54 DEBUG }, 22:10:54 DEBUG 22:10:54 DEBUG { 22:10:54 DEBUG "resultType": "vector", 22:10:54 DEBUG "resultType": "vector", 22:10:54 DEBUG }, 22:10:54 DEBUG [...] 22:10:54 DEBUG 22:10:54 DEBUG 22:10:54 DEBUG }, 22:10:54 DEBUG "pod": "am-555f4db595-fz8sn" 22:10:54 DEBUG "result": [ 22:10:54 DEBUG 22:10:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:10:54 DEBUG 22:10:54 DEBUG 22:10:54 DEBUG --- http response --- 22:10:54 DEBUG 22:10:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:10:54 DEBUG [...] 22:10:54 DEBUG "data": { 22:10:54 DEBUG 22:10:54 DEBUG }, 22:10:54 DEBUG 22:10:54 DEBUG { 22:10:54 DEBUG [...] 22:10:54 DEBUG "value": [ 22:10:54 DEBUG 22:10:54 DEBUG "metric": { 22:10:54 DEBUG "result": [ 22:10:54 DEBUG "result": [ 22:10:54 DEBUG "value": [ 22:10:54 DEBUG 22:10:54 DEBUG "pod": "ds-cts-1" 22:10:54 DEBUG 22:10:54 DEBUG "value": [ 22:10:54 DEBUG }, 22:10:54 DEBUG { 22:10:54 DEBUG "pod": "ds-cts-1" 22:10:54 DEBUG ----- output ----- 22:10:54 DEBUG 22:10:54 DEBUG "instance": "10.142.0.43:9100" 22:10:54 DEBUG 22:10:54 DEBUG --- http response --- 22:10:54 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 22:10:54 DEBUG 22:10:54 DEBUG 22:10:54 DEBUG 22:10:54 DEBUG 22:10:54 DEBUG 22:10:54 DEBUG "status": "success", 22:10:54 DEBUG 22:10:54 DEBUG 1713301854, 22:10:54 DEBUG 22:10:54 DEBUG "pod": "am-555f4db595-fz8sn" 22:10:54 DEBUG { 22:10:54 DEBUG { 22:10:54 DEBUG 22:10:54 DEBUG 22:10:54 DEBUG }, 22:10:54 DEBUG 22:10:54 DEBUG 1713301854, 22:10:54 DEBUG "value": [ 22:10:54 DEBUG "metric": { 22:10:54 DEBUG }, 22:10:54 DEBUG { 22:10:54 DEBUG "instance": "10.142.0.43:9100" 22:10:54 DEBUG }, 22:10:54 DEBUG [...] 22:10:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:10:54 DEBUG "resultType": "vector", 22:10:54 DEBUG 22:10:54 DEBUG "value": [ 22:10:54 DEBUG "instance": "10.142.0.43:9100" 22:10:54 DEBUG 22:10:54 DEBUG "data": { 22:10:54 DEBUG 22:10:54 DEBUG "0" 22:10:54 DEBUG "pod": "overseer-0-7c9b667974-9mr2s" 22:10:54 DEBUG }, 22:10:54 DEBUG "metric": { 22:10:54 DEBUG "metric": { 22:10:54 DEBUG 22:10:54 DEBUG 22:10:54 DEBUG "value": [ 22:10:54 DEBUG [...] 22:10:54 DEBUG "0.023930129682068056" 22:10:54 DEBUG 22:10:54 DEBUG "pod": "ds-idrepo-2" 22:10:54 DEBUG "value": [ 22:10:54 DEBUG "status": "success", 22:10:54 DEBUG }, 22:10:54 DEBUG "value": [ 22:10:54 DEBUG 22:10:54 DEBUG ----- output ----- 22:10:54 DEBUG "result": [ 22:10:54 DEBUG "pod": "am-555f4db595-kzxsp" 22:10:54 DEBUG 1713301854, 22:10:54 DEBUG }, 22:10:54 DEBUG "pod": "ds-idrepo-0" 22:10:54 DEBUG "resultType": "vector", 22:10:54 DEBUG 22:10:54 DEBUG ] 22:10:54 DEBUG }, 22:10:54 DEBUG "value": [ 22:10:54 DEBUG "pod": "ds-idrepo-2" 22:10:54 DEBUG "pod": "am-555f4db595-fz8sn" 22:10:54 DEBUG 22:10:54 DEBUG "pod": "am-555f4db595-kzxsp" 22:10:54 DEBUG 1713301854, 22:10:54 DEBUG 22:10:54 DEBUG ] 22:10:54 DEBUG 22:10:54 DEBUG }, 22:10:54 DEBUG 1713301854, 22:10:54 DEBUG "data": { 22:10:54 DEBUG "value": [ 22:10:54 DEBUG 1713301854, 22:10:54 DEBUG 22:10:54 DEBUG { 22:10:54 DEBUG { 22:10:54 DEBUG }, 22:10:54 DEBUG "112275456" 22:10:54 DEBUG "value": [ 22:10:54 DEBUG }, 22:10:54 DEBUG "result": [ 22:10:54 DEBUG "pod": "ds-idrepo-0" 22:10:54 DEBUG } 22:10:54 DEBUG ] 22:10:54 DEBUG 22:10:54 DEBUG }, 22:10:54 DEBUG }, 22:10:54 DEBUG [...] 22:10:54 DEBUG }, 22:10:54 DEBUG "0" 22:10:54 DEBUG 22:10:54 DEBUG } 22:10:54 DEBUG 22:10:54 DEBUG "value": [ 22:10:54 DEBUG 22:10:54 DEBUG "resultType": "vector", 22:10:54 DEBUG 1713301854, 22:10:54 DEBUG "15778.499999999996" 22:10:54 DEBUG 22:10:54 DEBUG "status": "success", 22:10:54 DEBUG "metric": { 22:10:54 DEBUG "value": [ 22:10:54 DEBUG ] 22:10:54 DEBUG 1713301854, 22:10:54 DEBUG "value": [ 22:10:54 DEBUG { 22:10:54 DEBUG }, 22:10:54 DEBUG "value": [ 22:10:54 DEBUG } 22:10:54 DEBUG 22:10:54 DEBUG "value": [ 22:10:54 DEBUG "value": [ 22:10:54 DEBUG 22:10:54 DEBUG "value": [ 22:10:54 DEBUG ] 22:10:54 DEBUG 22:10:54 DEBUG ] 22:10:54 DEBUG [...] 22:10:54 DEBUG 22:10:54 DEBUG "688.5885280075314" 22:10:54 DEBUG "result": [ 22:10:54 DEBUG "0.000333333333333341" 22:10:54 DEBUG ] 22:10:54 DEBUG "pod": "ds-idrepo-0" 22:10:54 DEBUG "data": { 22:10:54 DEBUG "pod": "am-555f4db595-fz8sn" 22:10:54 DEBUG 1713301854, 22:10:54 DEBUG } 22:10:54 DEBUG "7388.166666666667" 22:10:54 DEBUG 1713301854, 22:10:54 DEBUG "metric": { 22:10:54 DEBUG "value": [ 22:10:54 DEBUG 1713301854, 22:10:54 DEBUG } 22:10:54 DEBUG 22:10:54 DEBUG 22:10:54 DEBUG 22:10:54 DEBUG 22:10:54 DEBUG 1713301854, 22:10:54 DEBUG } 22:10:54 DEBUG "pod": "ds-idrepo-0" 22:10:54 DEBUG } 22:10:54 DEBUG 22:10:54 DEBUG 22:10:54 DEBUG ] 22:10:54 DEBUG { 22:10:54 DEBUG ] 22:10:54 DEBUG } 22:10:54 DEBUG }, 22:10:54 DEBUG "resultType": "vector", 22:10:54 DEBUG }, 22:10:54 DEBUG "0" 22:10:54 DEBUG ] 22:10:54 DEBUG ] 22:10:54 DEBUG "0" 22:10:54 DEBUG "pod": "ds-idrepo-2" 22:10:54 DEBUG 1713301854, 22:10:54 DEBUG "0" 22:10:54 DEBUG [...] 22:10:54 DEBUG 22:10:54 DEBUG 22:10:54 DEBUG 22:10:54 DEBUG "0" 22:10:54 DEBUG ] 22:10:54 DEBUG }, 22:10:54 DEBUG } 22:10:54 DEBUG 22:10:54 DEBUG 22:10:54 DEBUG } 22:10:54 DEBUG "metric": { 22:10:54 DEBUG } 22:10:54 DEBUG ] 22:10:54 DEBUG "value": [ 22:10:54 DEBUG "result": [ 22:10:54 DEBUG "value": [ 22:10:54 DEBUG ] 22:10:54 DEBUG } 22:10:54 DEBUG } 22:10:54 DEBUG ] 22:10:54 DEBUG }, 22:10:54 DEBUG "0" 22:10:54 DEBUG ] 22:10:54 DEBUG 22:10:54 DEBUG 22:10:54 DEBUG 22:10:54 DEBUG "pod": "am-555f4db595-kzxsp" 22:10:54 DEBUG ] 22:10:54 DEBUG } 22:10:54 DEBUG "value": [ 22:10:54 DEBUG [...] 22:10:54 DEBUG "pod": "am-555f4db595-kzxsp" 22:10:54 DEBUG }, 22:10:54 DEBUG "instance": "10.142.0.107:9100" 22:10:54 DEBUG ] 22:10:54 DEBUG } 22:10:54 DEBUG 1713301854, 22:10:54 DEBUG { 22:10:54 DEBUG 22:10:54 DEBUG } 22:10:54 DEBUG } 22:10:54 DEBUG ] 22:10:54 DEBUG } 22:10:54 DEBUG "value": [ 22:10:54 DEBUG ] 22:10:54 DEBUG } 22:10:54 DEBUG 22:10:54 DEBUG [...] 22:10:54 DEBUG [...] 22:10:54 DEBUG }, 22:10:54 DEBUG } 22:10:54 DEBUG } 22:10:54 DEBUG 1713301854, 22:10:54 DEBUG 22:10:54 DEBUG "value": [ 22:10:54 DEBUG ] 22:10:54 DEBUG }, 22:10:54 DEBUG } 22:10:54 DEBUG } 22:10:54 DEBUG "0" 22:10:54 DEBUG "metric": { 22:10:54 DEBUG 22:10:54 DEBUG ] 22:10:54 DEBUG } 22:10:54 DEBUG ] 22:10:54 DEBUG 22:10:54 DEBUG } 22:10:54 DEBUG ] 22:10:54 DEBUG 22:10:54 DEBUG 22:10:54 DEBUG 22:10:54 DEBUG "value": [ 22:10:54 DEBUG ] 22:10:54 DEBUG "0" 22:10:54 DEBUG 22:10:54 DEBUG 1713301854, 22:10:54 DEBUG } 22:10:54 DEBUG "value": [ 22:10:54 DEBUG } 22:10:54 DEBUG ] 22:10:54 DEBUG "instance": "10.142.0.107:9100" 22:10:54 DEBUG 22:10:54 DEBUG } 22:10:54 DEBUG } 22:10:54 DEBUG } 22:10:54 DEBUG 22:10:54 DEBUG ] 22:10:54 DEBUG } 22:10:54 DEBUG "pod": "am-555f4db595-kzxsp" 22:10:54 DEBUG 22:10:54 DEBUG 22:10:54 DEBUG 1713301854, 22:10:54 DEBUG } 22:10:54 DEBUG ] 22:10:54 DEBUG 22:10:54 DEBUG "0" 22:10:54 DEBUG } 22:10:54 DEBUG 22:10:54 DEBUG } 22:10:54 DEBUG }, 22:10:54 DEBUG [...] 22:10:54 DEBUG } 22:10:54 DEBUG } 22:10:54 DEBUG 22:10:54 DEBUG } 22:10:54 DEBUG } 22:10:54 DEBUG }, 22:10:54 DEBUG 22:10:54 DEBUG 22:10:54 DEBUG "0" 22:10:54 DEBUG } 22:10:54 DEBUG } 22:10:54 DEBUG "pod": "ds-idrepo-0" 22:10:54 DEBUG ] 22:10:54 DEBUG 22:10:54 DEBUG ] 22:10:54 DEBUG "value": [ 22:10:54 DEBUG 22:10:54 DEBUG [...] 22:10:54 DEBUG } 22:10:54 DEBUG "value": [ 22:10:54 DEBUG "pod": "ds-idrepo-0" 22:10:54 DEBUG "pod": "am-555f4db595-kzxsp" 22:10:54 DEBUG ] 22:10:54 DEBUG ] 22:10:54 DEBUG }, 22:10:54 DEBUG } 22:10:54 DEBUG 22:10:54 DEBUG } 22:10:54 DEBUG 22:10:54 DEBUG 22:10:54 DEBUG 22:10:54 DEBUG 1713301854, 22:10:54 DEBUG }, 22:10:54 DEBUG }, 22:10:54 DEBUG } 22:10:54 DEBUG } 22:10:54 DEBUG "value": [ 22:10:54 DEBUG ] 22:10:54 DEBUG [...] 22:10:54 DEBUG } 22:10:54 DEBUG 22:10:54 DEBUG 22:10:54 DEBUG 22:10:54 DEBUG "0" 22:10:54 DEBUG "value": [ 22:10:54 DEBUG "value": [ 22:10:54 DEBUG ] 22:10:54 DEBUG } 22:10:54 DEBUG 1713301854, 22:10:54 DEBUG } 22:10:54 DEBUG 22:10:54 DEBUG 22:10:54 DEBUG "pod": "am-555f4db595-kzxsp" 22:10:54 DEBUG 22:10:54 DEBUG ] 22:10:54 DEBUG 1713301854, 22:10:54 DEBUG 1713301854, 22:10:54 DEBUG } 22:10:54 DEBUG "0" 22:10:54 DEBUG ] 22:10:54 DEBUG 22:10:54 DEBUG [...] 22:10:54 DEBUG }, 22:10:54 DEBUG "pod": "ds-cts-2" 22:10:54 DEBUG } 22:10:54 DEBUG "0" 22:10:54 DEBUG "1" 22:10:54 DEBUG } 22:10:54 DEBUG } 22:10:54 DEBUG } 22:10:54 DEBUG 22:10:54 DEBUG 22:10:54 DEBUG "value": [ 22:10:54 DEBUG }, 22:10:54 DEBUG ] 22:10:54 DEBUG ] 22:10:54 DEBUG ] 22:10:54 DEBUG ] 22:10:54 DEBUG "instance": "10.142.0.43:9100" 22:10:54 DEBUG 22:10:54 DEBUG 1713301854, 22:10:54 DEBUG "value": [ 22:10:54 DEBUG } 22:10:54 DEBUG } 22:10:54 DEBUG } 22:10:54 DEBUG } 22:10:54 DEBUG }, 22:10:54 DEBUG 22:10:54 DEBUG "0.023" 22:10:54 DEBUG 1713301854, 22:10:54 DEBUG } 22:10:54 DEBUG ] 22:10:54 DEBUG ] 22:10:54 DEBUG } 22:10:54 DEBUG "value": [ 22:10:54 DEBUG "instance": "10.142.0.43:9100" 22:10:54 DEBUG ] 22:10:54 DEBUG "0" 22:10:54 DEBUG } 22:10:54 DEBUG } 22:10:54 DEBUG 1713301854, 22:10:54 DEBUG }, 22:10:54 DEBUG } 22:10:54 DEBUG ] 22:10:54 DEBUG } 22:10:54 DEBUG } 22:10:54 DEBUG "19941348352" 22:10:54 DEBUG "value": [ 22:10:54 DEBUG ] 22:10:54 DEBUG } 22:10:54 DEBUG ] 22:10:54 DEBUG 1713301854, 22:10:54 DEBUG } 22:10:54 DEBUG ] 22:10:54 DEBUG } 22:10:54 DEBUG "0.0005000000000004927" 22:10:54 DEBUG } 22:10:54 DEBUG } 22:10:54 DEBUG ] 22:10:54 DEBUG ] 22:10:54 DEBUG } 22:10:54 DEBUG } 22:10:54 DEBUG } 22:10:54 DEBUG } 22:10:54 DEBUG ] 22:10:54 DEBUG } 22:10:54 DEBUG } 22:11:54 INFO 22:11:54 INFO [loop_until]: kubectl --namespace=xlou top pods 22:11:54 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:11:54 INFO 22:11:54 INFO [loop_until]: kubectl --namespace=xlou top node 22:11:54 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:11:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713301914" 22:11:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713301914" 22:11:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713301914" 22:11:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713301914" 22:11:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713301914" 22:11:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713301914" 22:11:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713301914" 22:11:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713301914" 22:11:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713301914" 22:11:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713301914" 22:11:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713301914" 22:11:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713301914" 22:11:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713301914" 22:11:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713301914" 22:11:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713301914" 22:11:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713301914" 22:11:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713301914" 22:11:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713301914" 22:11:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713301914" 22:11:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713301914" 22:11:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713301914" 22:11:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713301914" 22:11:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713301914" 22:11:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713301914" 22:11:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713301914" 22:11:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713301914" 22:11:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713301914" 22:11:54 INFO [http_cmd]: http status code OK 22:11:54 ERROR [http_cmd]: ERROR 22:11:54 INFO [http_cmd]: http status code OK 22:11:54 INFO [http_cmd]: http status code OK 22:11:54 INFO [http_cmd]: http status code OK 22:11:54 INFO [http_cmd]: http status code OK 22:11:54 INFO [http_cmd]: http status code OK 22:11:54 INFO [http_cmd]: http status code OK 22:11:54 INFO [loop_until]: OK (rc = 0) 22:11:54 INFO [http_cmd]: http status code OK 22:11:54 INFO [http_cmd]: http status code OK 22:11:54 INFO [http_cmd]: http status code OK 22:11:54 INFO [http_cmd]: http status code OK 22:11:54 INFO [http_cmd]: http status code OK 22:11:54 INFO [http_cmd]: http status code OK 22:11:54 INFO [http_cmd]: http status code OK 22:11:54 INFO [http_cmd]: http status code OK 22:11:54 INFO [http_cmd]: http status code OK 22:11:54 INFO [http_cmd]: http status code OK 22:11:54 INFO [loop_until]: OK (rc = 0) 22:11:54 INFO [http_cmd]: http status code OK 22:11:54 DEBUG --- status code --- 22:11:54 INFO [http_cmd]: http status code OK 22:11:54 INFO [http_cmd]: http status code OK 22:11:54 ERROR --- status code --- 22:11:54 INFO [http_cmd]: http status code OK 22:11:54 DEBUG --- status code --- 22:11:54 DEBUG --- status code --- 22:11:54 DEBUG --- status code --- 22:11:54 DEBUG --- status code --- 22:11:54 DEBUG --- status code --- 22:11:54 INFO [http_cmd]: http status code OK 22:11:54 DEBUG --- status code --- 22:11:54 INFO [http_cmd]: http status code OK 22:11:54 INFO [http_cmd]: http status code OK 22:11:54 DEBUG --- stdout --- 22:11:54 DEBUG --- status code --- 22:11:54 DEBUG --- status code --- 22:11:54 DEBUG --- status code --- 22:11:54 DEBUG --- status code --- 22:11:54 DEBUG --- status code --- 22:11:54 DEBUG --- status code --- 22:11:54 DEBUG --- status code --- 22:11:54 DEBUG --- status code --- 22:11:54 DEBUG --- status code --- 22:11:54 DEBUG --- status code --- 22:11:54 DEBUG --- stdout --- 22:11:54 DEBUG --- status code --- 22:11:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:11:54 DEBUG --- status code --- 22:11:54 DEBUG --- status code --- 22:11:54 INFO [http_cmd]: http status code OK 22:11:54 INFO [http_cmd]: http status code OK 22:11:54 ERROR http status code is 400 (expected [200, 401, 403, 404]) 22:11:54 DEBUG --- status code --- 22:11:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:11:54 DEBUG --- http response --- 22:11:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:11:54 DEBUG --- http response --- 22:11:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:11:54 DEBUG --- status code --- 22:11:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:11:54 DEBUG --- status code --- 22:11:54 DEBUG --- status code --- 22:11:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:11:54 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 9Mi am-555f4db595-4lbss 20m 3360Mi am-555f4db595-fz8sn 183m 3310Mi am-555f4db595-kzxsp 1119m 1276Mi ds-cts-0 5m 8290Mi ds-cts-1 21m 8308Mi ds-cts-2 245m 8454Mi ds-idrepo-0 68m 5032Mi ds-idrepo-1 37m 2781Mi ds-idrepo-2 61m 2853Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 4m 1208Mi idm-b85f9d978-rl76f 3m 1225Mi lodemon-bc756c647-cr7x6 177m 61Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-7c9b667974-9mr2s 1980m 181Mi 22:11:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:11:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:11:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:11:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:11:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:11:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:11:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:11:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:11:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:11:54 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 62m 0% 5088Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 109m 0% 10814Mi 9% gke-xlou-cdm-default-pool-1b0769c8-nhlf 677m 2% 5522Mi 4% gke-xlou-cdm-default-pool-1b0769c8-qkns 85m 0% 2672Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 253m 0% 6200Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 389m 1% 12944Mi 11% gke-xlou-cdm-default-pool-1b0769c8-xnmp 67m 0% 3142Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 1354m 4% 5184Mi 4% gke-xlou-cdm-ds-0f601751-hmm7 173m 0% 5522Mi 4% gke-xlou-cdm-ds-0f601751-j488 111m 0% 10333Mi 9% gke-xlou-cdm-ds-0f601751-mc5d 738m 2% 3271Mi 2% gke-xlou-cdm-ds-0f601751-nj6q 402m 1% 3061Mi 2% gke-xlou-cdm-ds-0f601751-nmxm 354m 1% 7360Mi 6% gke-xlou-cdm-frontend-a8771548-i82n 2308m 14% 5892Mi 10% 22:11:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:11:54 DEBUG --- http response --- 22:11:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:11:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:11:54 DEBUG --- status code --- 22:11:54 DEBUG --- status code --- 22:11:54 ERROR --- http response --- 22:11:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:11:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:11:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:11:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:11:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 22:11:54 DEBUG --- http response --- 22:11:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:11:54 DEBUG --- http response --- 22:11:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:11:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:11:54 DEBUG --- http response --- 22:11:54 DEBUG --- stderr --- 22:11:54 DEBUG --- http response --- 22:11:54 DEBUG --- http response --- 22:11:54 DEBUG --- http response --- 22:11:54 DEBUG --- http response --- 22:11:54 DEBUG --- http response --- 22:11:54 DEBUG --- http response --- 22:11:54 DEBUG --- http response --- 22:11:54 DEBUG --- http response --- 22:11:54 DEBUG --- http response --- 22:11:54 DEBUG --- stderr --- 22:11:54 DEBUG --- http response --- 22:11:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:11:54 DEBUG --- http response --- 22:11:54 DEBUG --- http response --- 22:11:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:11:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:11:54 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 22:11:54 DEBUG --- http response --- 22:11:54 DEBUG --- http response --- 22:11:54 DEBUG ----- output ----- 22:11:54 DEBUG --- http response --- 22:11:54 DEBUG ----- output ----- 22:11:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:11:54 DEBUG --- http response --- 22:11:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:11:54 DEBUG --- http response --- 22:11:54 DEBUG --- http response --- 22:11:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:11:54 DEBUG 22:11:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:11:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 22:11:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:11:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:11:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:11:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:11:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:11:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:11:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:11:54 DEBUG 22:11:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:11:54 DEBUG ----- output ----- 22:11:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:11:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:11:54 DEBUG --- http response --- 22:11:54 DEBUG --- http response --- 22:11:54 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 22:11:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:11:54 DEBUG { 22:11:54 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 22:11:54 DEBUG { 22:11:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:11:54 DEBUG ----- output ----- 22:11:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:11:54 DEBUG ----- output ----- 22:11:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:11:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:11:54 DEBUG ----- output ----- 22:11:54 DEBUG ----- output ----- 22:11:54 DEBUG ----- output ----- 22:11:54 DEBUG ----- output ----- 22:11:54 DEBUG ----- output ----- 22:11:54 DEBUG ----- output ----- 22:11:54 DEBUG ----- output ----- 22:11:54 DEBUG ----- output ----- 22:11:54 DEBUG ----- output ----- 22:11:54 DEBUG ----- output ----- 22:11:54 DEBUG ----- output ----- 22:11:54 DEBUG { 22:11:54 DEBUG ----- output ----- 22:11:54 DEBUG ----- output ----- 22:11:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:11:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:11:54 DEBUG ----- output ----- 22:11:54 DEBUG "status": "success", 22:11:54 DEBUG "status": "success", 22:11:54 DEBUG ----- output ----- 22:11:54 DEBUG { 22:11:54 DEBUG ----- output ----- 22:11:54 DEBUG { 22:11:54 DEBUG ----- output ----- 22:11:54 DEBUG ----- output ----- 22:11:54 DEBUG { 22:11:54 DEBUG { 22:11:54 DEBUG { 22:11:54 DEBUG { 22:11:54 DEBUG { 22:11:54 DEBUG { 22:11:54 DEBUG { 22:11:54 DEBUG { 22:11:54 DEBUG { 22:11:54 DEBUG { 22:11:54 DEBUG { 22:11:54 DEBUG "status": "success", 22:11:54 DEBUG { 22:11:54 DEBUG { 22:11:54 DEBUG ----- output ----- 22:11:54 DEBUG ----- output ----- 22:11:54 DEBUG { 22:11:54 DEBUG "data": { 22:11:54 DEBUG "data": { 22:11:54 DEBUG { 22:11:54 DEBUG "status": "success", 22:11:54 DEBUG { 22:11:54 DEBUG "status": "success", 22:11:54 DEBUG { 22:11:54 DEBUG { 22:11:54 DEBUG "status": "success", 22:11:54 DEBUG "status": "success", 22:11:54 DEBUG "status": "success", 22:11:54 DEBUG "status": "success", 22:11:54 DEBUG "status": "success", 22:11:54 DEBUG "status": "success", 22:11:54 DEBUG "status": "success", 22:11:54 DEBUG "status": "success", 22:11:54 DEBUG "status": "success", 22:11:54 DEBUG "status": "success", 22:11:54 DEBUG "status": "success", 22:11:54 DEBUG "data": { 22:11:54 DEBUG "status": "success", 22:11:54 DEBUG "status": "success", 22:11:54 DEBUG { 22:11:54 DEBUG { 22:11:54 DEBUG "status": "success", 22:11:54 DEBUG "resultType": "vector", 22:11:54 DEBUG "resultType": "vector", 22:11:54 DEBUG "result": [ 22:11:54 DEBUG "data": { 22:11:54 DEBUG "status": "success", 22:11:54 DEBUG "data": { 22:11:54 DEBUG "status": "success", 22:11:54 DEBUG "status": "success", 22:11:54 DEBUG "data": { 22:11:54 DEBUG "data": { 22:11:54 DEBUG "data": { 22:11:54 DEBUG "data": { 22:11:54 DEBUG "data": { 22:11:54 DEBUG "data": { 22:11:54 DEBUG "data": { 22:11:54 DEBUG "data": { 22:11:54 DEBUG "data": { 22:11:54 DEBUG "data": { 22:11:54 DEBUG "data": { 22:11:54 DEBUG "resultType": "vector", 22:11:54 DEBUG "data": { 22:11:54 DEBUG "data": { 22:11:54 DEBUG "status": "success", 22:11:54 DEBUG "status": "success", 22:11:54 DEBUG "data": { 22:11:54 DEBUG "result": [ 22:11:54 DEBUG { 22:11:54 DEBUG "status": "success", 22:11:54 DEBUG "resultType": "vector", 22:11:54 DEBUG "data": { 22:11:54 DEBUG "resultType": "vector", 22:11:54 DEBUG "data": { 22:11:54 DEBUG "data": { 22:11:54 DEBUG "resultType": "vector", 22:11:54 DEBUG "resultType": "vector", 22:11:54 DEBUG "resultType": "vector", 22:11:54 DEBUG "resultType": "vector", 22:11:54 DEBUG "resultType": "vector", 22:11:54 DEBUG "resultType": "vector", 22:11:54 DEBUG "resultType": "vector", 22:11:54 DEBUG "resultType": "vector", 22:11:54 DEBUG "resultType": "vector", 22:11:54 DEBUG "resultType": "vector", 22:11:54 DEBUG "resultType": "vector", 22:11:54 DEBUG "result": [ 22:11:54 DEBUG "resultType": "vector", 22:11:54 DEBUG "resultType": "vector", 22:11:54 DEBUG "result": [ 22:11:54 DEBUG "data": { 22:11:54 DEBUG "resultType": "vector", 22:11:54 DEBUG { 22:11:54 DEBUG "metric": { 22:11:54 DEBUG "data": { 22:11:54 DEBUG "result": [ 22:11:54 DEBUG "resultType": "vector", 22:11:54 DEBUG "result": [ 22:11:54 DEBUG "resultType": "vector", 22:11:54 DEBUG "resultType": "vector", 22:11:54 DEBUG "result": [ 22:11:54 DEBUG "result": [ 22:11:54 DEBUG "result": [ 22:11:54 DEBUG "result": [ 22:11:54 DEBUG "result": [ 22:11:54 DEBUG "result": [ 22:11:54 DEBUG "result": [ 22:11:54 DEBUG "result": [ 22:11:54 DEBUG "result": [ 22:11:54 DEBUG "result": [ 22:11:54 DEBUG "result": [ 22:11:54 DEBUG { 22:11:54 DEBUG "result": [ 22:11:54 DEBUG { 22:11:54 DEBUG "data": { 22:11:54 DEBUG "resultType": "vector", 22:11:54 DEBUG "result": [ 22:11:54 DEBUG "metric": { 22:11:54 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:11:54 DEBUG "resultType": "vector", 22:11:54 DEBUG { 22:11:54 DEBUG "result": [ 22:11:54 DEBUG { 22:11:54 DEBUG "result": [ 22:11:54 DEBUG "result": [ 22:11:54 DEBUG { 22:11:54 DEBUG { 22:11:54 DEBUG { 22:11:54 DEBUG { 22:11:54 DEBUG { 22:11:54 DEBUG { 22:11:54 DEBUG { 22:11:54 DEBUG { 22:11:54 DEBUG { 22:11:54 DEBUG { 22:11:54 DEBUG { 22:11:54 DEBUG "metric": { 22:11:54 DEBUG { 22:11:54 DEBUG "metric": { 22:11:54 DEBUG "resultType": "vector", 22:11:54 DEBUG "result": [ 22:11:54 DEBUG { 22:11:54 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:11:54 DEBUG }, 22:11:54 DEBUG "result": [ 22:11:54 DEBUG "metric": { 22:11:54 DEBUG { 22:11:54 DEBUG "metric": { 22:11:54 DEBUG { 22:11:54 DEBUG { 22:11:54 DEBUG "metric": { 22:11:54 DEBUG "metric": { 22:11:54 DEBUG "metric": { 22:11:54 DEBUG "metric": { 22:11:54 DEBUG "metric": { 22:11:54 DEBUG "metric": { 22:11:54 DEBUG "metric": { 22:11:54 DEBUG "metric": { 22:11:54 DEBUG "metric": { 22:11:54 DEBUG "metric": { 22:11:54 DEBUG "metric": { 22:11:54 DEBUG "pod": "am-555f4db595-fz8sn" 22:11:54 DEBUG "metric": { 22:11:54 DEBUG "pod": "am-555f4db595-fz8sn" 22:11:54 DEBUG "result": [ 22:11:54 DEBUG { 22:11:54 DEBUG "metric": { 22:11:54 DEBUG }, 22:11:54 DEBUG "value": [ 22:11:54 DEBUG { 22:11:54 DEBUG "instance": "10.142.0.107:9100" 22:11:54 DEBUG "metric": { 22:11:54 DEBUG "instance": "10.142.0.107:9100" 22:11:54 DEBUG }, 22:11:54 DEBUG "metric": { 22:11:54 DEBUG "pod": "am-555f4db595-fz8sn" 22:11:54 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:11:54 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:11:54 DEBUG "instance": "10.142.0.107:9100" 22:11:54 DEBUG "pod": "ds-idrepo-2" 22:11:54 DEBUG "pod": "am-555f4db595-fz8sn" 22:11:54 DEBUG "pod": "ds-idrepo-2" 22:11:54 DEBUG "pod": "am-555f4db595-fz8sn" 22:11:54 DEBUG "pod": "ds-idrepo-2" 22:11:54 DEBUG "pod": "ds-idrepo-2" 22:11:54 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:11:54 DEBUG }, 22:11:54 DEBUG "pod": "am-555f4db595-fz8sn" 22:11:54 DEBUG }, 22:11:54 DEBUG { 22:11:54 DEBUG "metric": { 22:11:54 DEBUG "instance": "10.142.0.107:9100" 22:11:54 DEBUG "value": [ 22:11:54 DEBUG 22:11:54 DEBUG "metric": { 22:11:54 DEBUG }, 22:11:54 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:11:54 DEBUG "metric": { 22:11:54 DEBUG "value": [ 22:11:54 DEBUG "pod": "ds-idrepo-2" 22:11:54 DEBUG }, 22:11:54 DEBUG }, 22:11:54 DEBUG }, 22:11:54 DEBUG }, 22:11:54 DEBUG }, 22:11:54 DEBUG }, 22:11:54 DEBUG }, 22:11:54 DEBUG }, 22:11:54 DEBUG }, 22:11:54 DEBUG }, 22:11:54 DEBUG }, 22:11:54 DEBUG "value": [ 22:11:54 DEBUG }, 22:11:54 DEBUG "value": [ 22:11:54 DEBUG "metric": { 22:11:54 DEBUG "pod": "ds-idrepo-2" 22:11:54 DEBUG }, 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG "pod": "ds-idrepo-2" 22:11:54 DEBUG "value": [ 22:11:54 DEBUG }, 22:11:54 DEBUG "instance": "10.142.0.107:9100" 22:11:54 DEBUG 22:11:54 DEBUG }, 22:11:54 DEBUG "value": [ 22:11:54 DEBUG "value": [ 22:11:54 DEBUG "value": [ 22:11:54 DEBUG "value": [ 22:11:54 DEBUG 22:11:54 DEBUG "value": [ 22:11:54 DEBUG "value": [ 22:11:54 DEBUG "value": [ 22:11:54 DEBUG "value": [ 22:11:54 DEBUG "value": [ 22:11:54 DEBUG "value": [ 22:11:54 DEBUG 22:11:54 DEBUG "value": [ 22:11:54 DEBUG 22:11:54 DEBUG "pod": "am-555f4db595-fz8sn" 22:11:54 DEBUG }, 22:11:54 DEBUG "value": [ 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG }, 22:11:54 DEBUG 22:11:54 DEBUG "value": [ 22:11:54 DEBUG }, 22:11:54 DEBUG 22:11:54 DEBUG "value": [ 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG "value": [ 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG }, 22:11:54 DEBUG "value": [ 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG [...] 22:11:54 DEBUG "value": [ 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG "value": [ 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG "value": [ 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG [...] 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG [...] 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG [...] 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG [...] 22:11:54 DEBUG 22:11:54 DEBUG [...] 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG [...] 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG [...] 22:11:54 DEBUG [...] 22:11:54 DEBUG [...] 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG [...] 22:11:54 DEBUG [...] 22:11:54 DEBUG 22:11:54 DEBUG [...] 22:11:54 DEBUG 22:11:54 DEBUG [...] 22:11:54 DEBUG 22:11:54 DEBUG [...] 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG [...] 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG [...] 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG [...] 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG [...] 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG [...] 22:11:54 DEBUG 22:11:54 DEBUG [...] 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG [...] 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG "pod": "am-555f4db595-kzxsp" 22:11:54 DEBUG [...] 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG [...] 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG [...] 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG "pod": "ds-idrepo-2" 22:11:54 DEBUG }, 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG "instance": "10.142.0.43:9100" 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG "instance": "10.142.0.43:9100" 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG "pod": "ds-idrepo-0" 22:11:54 DEBUG 22:11:54 DEBUG "pod": "ds-idrepo-0" 22:11:54 DEBUG 22:11:54 DEBUG "pod": "am-555f4db595-kzxsp" 22:11:54 DEBUG }, 22:11:54 DEBUG "pod": "am-555f4db595-kzxsp" 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG }, 22:11:54 DEBUG "value": [ 22:11:54 DEBUG 22:11:54 DEBUG "instance": "10.142.0.43:9100" 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG }, 22:11:54 DEBUG 22:11:54 DEBUG "pod": "am-555f4db595-kzxsp" 22:11:54 DEBUG "pod": "ds-idrepo-2" 22:11:54 DEBUG "pod": "am-555f4db595-kzxsp" 22:11:54 DEBUG 22:11:54 DEBUG }, 22:11:54 DEBUG "pod": "am-555f4db595-kzxsp" 22:11:54 DEBUG 22:11:54 DEBUG }, 22:11:54 DEBUG 22:11:54 DEBUG }, 22:11:54 DEBUG "pod": "ds-idrepo-0" 22:11:54 DEBUG "value": [ 22:11:54 DEBUG 22:11:54 DEBUG }, 22:11:54 DEBUG 22:11:54 DEBUG 22:11:54 DEBUG "instance": "10.142.0.43:9100" 22:11:54 DEBUG "value": [ 22:11:54 DEBUG 1713301914, 22:11:54 DEBUG 22:11:54 DEBUG }, 22:11:54 DEBUG "pod": "ds-cts-2" 22:11:54 DEBUG 22:11:54 DEBUG "value": [ 22:11:54 DEBUG "pod": "ds-idrepo-0" 22:11:54 DEBUG }, 22:11:54 DEBUG }, 22:11:54 DEBUG }, 22:11:54 DEBUG "pod": "ds-idrepo-0" 22:11:54 DEBUG "value": [ 22:11:54 DEBUG }, 22:11:54 DEBUG "pod": "am-555f4db595-kzxsp" 22:11:54 DEBUG "value": [ 22:11:54 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:11:54 DEBUG "value": [ 22:11:54 DEBUG }, 22:11:54 DEBUG 1713301914, 22:11:54 DEBUG "pod": "am-555f4db595-kzxsp" 22:11:54 DEBUG "value": [ 22:11:54 DEBUG 22:11:54 DEBUG "pod": "ds-idrepo-0" 22:11:54 DEBUG }, 22:11:54 DEBUG "0" 22:11:54 DEBUG 1713301914, 22:11:54 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:11:54 DEBUG "value": [ 22:11:54 DEBUG }, 22:11:54 DEBUG "instance": "10.142.0.43:9100" 22:11:54 DEBUG 1713301914, 22:11:54 DEBUG }, 22:11:54 DEBUG "value": [ 22:11:54 DEBUG "value": [ 22:11:54 DEBUG "value": [ 22:11:54 DEBUG }, 22:11:54 DEBUG 1713301914, 22:11:54 DEBUG "value": [ 22:11:54 DEBUG }, 22:11:54 DEBUG 1713301914, 22:11:54 DEBUG }, 22:11:54 DEBUG 1713301914, 22:11:54 DEBUG "value": [ 22:11:54 DEBUG "1743.5" 22:11:54 DEBUG }, 22:11:54 DEBUG 1713301914, 22:11:54 DEBUG "pod": "am-555f4db595-kzxsp" 22:11:54 DEBUG }, 22:11:54 DEBUG "value": [ 22:11:54 DEBUG ] 22:11:54 DEBUG "0" 22:11:54 DEBUG }, 22:11:54 DEBUG 1713301914, 22:11:54 DEBUG "value": [ 22:11:54 DEBUG }, 22:11:54 DEBUG "12245394.06609889" 22:11:54 DEBUG "value": [ 22:11:54 DEBUG 1713301914, 22:11:54 DEBUG 1713301914, 22:11:54 DEBUG 1713301914, 22:11:54 DEBUG "value": [ 22:11:54 DEBUG "13613829.466666669" 22:11:54 DEBUG 1713301914, 22:11:54 DEBUG "value": [ 22:11:54 DEBUG "0" 22:11:54 DEBUG "value": [ 22:11:54 DEBUG "0" 22:11:54 DEBUG 1713301914, 22:11:54 DEBUG "0" 22:11:54 DEBUG "value": [ 22:11:54 DEBUG "113.242375" 22:11:54 DEBUG }, 22:11:54 DEBUG "value": [ 22:11:54 DEBUG 1713301914, 22:11:54 DEBUG } 22:11:54 DEBUG ] 22:11:54 DEBUG "value": [ 22:11:54 DEBUG "0.0009999999999999788" 22:11:54 DEBUG 1713301914, 22:11:54 DEBUG "value": [ 22:11:54 DEBUG ] 22:11:54 DEBUG 1713301914, 22:11:54 DEBUG "112.99725" 22:11:54 DEBUG "0" 22:11:54 DEBUG "484.1178382531099" 22:11:54 DEBUG 1713301914, 22:11:54 DEBUG ] 22:11:54 DEBUG "116.56666666666666" 22:11:54 DEBUG 1713301914, 22:11:54 DEBUG ] 22:11:54 DEBUG 1713301914, 22:11:54 DEBUG ] 22:11:54 DEBUG ] 22:11:54 DEBUG ] 22:11:54 DEBUG 1713301914, 22:11:54 DEBUG ] 22:11:54 DEBUG "value": [ 22:11:54 DEBUG 1713301914, 22:11:54 DEBUG "19944430592" 22:11:54 DEBUG ] 22:11:54 DEBUG } 22:11:54 DEBUG 1713301914, 22:11:54 DEBUG ] 22:11:54 DEBUG "0" 22:11:54 DEBUG 1713301914, 22:11:54 DEBUG } 22:11:54 DEBUG "0" 22:11:54 DEBUG ] 22:11:54 DEBUG ] 22:11:54 DEBUG ] 22:11:54 DEBUG "0" 22:11:54 DEBUG } 22:11:54 DEBUG ] 22:11:54 DEBUG "0" 22:11:54 DEBUG } 22:11:54 DEBUG "3.1741708561740425" 22:11:54 DEBUG } 22:11:54 DEBUG } 22:11:54 DEBUG } 22:11:54 DEBUG "0" 22:11:54 DEBUG } 22:11:54 DEBUG 1713301914, 22:11:54 DEBUG "0" 22:11:54 DEBUG ] 22:11:54 DEBUG } 22:11:54 DEBUG ] 22:11:54 DEBUG "585678848" 22:11:54 DEBUG } 22:11:54 DEBUG ] 22:11:54 DEBUG "0.0012000000000000454" 22:11:54 DEBUG ] 22:11:54 DEBUG ] 22:11:54 DEBUG } 22:11:54 DEBUG } 22:11:54 DEBUG } 22:11:54 DEBUG ] 22:11:54 DEBUG ] 22:11:54 DEBUG } 22:11:54 DEBUG ] 22:11:54 DEBUG ] 22:11:54 DEBUG ] 22:11:54 DEBUG ] 22:11:54 DEBUG ] 22:11:54 DEBUG ] 22:11:54 DEBUG ] 22:11:54 DEBUG ] 22:11:54 DEBUG "0.023" 22:11:54 DEBUG ] 22:11:54 DEBUG } 22:11:54 DEBUG ] 22:11:54 DEBUG } 22:11:54 DEBUG ] 22:11:54 DEBUG ] 22:11:54 DEBUG } 22:11:54 DEBUG ] 22:11:54 DEBUG } 22:11:54 DEBUG } 22:11:54 DEBUG ] 22:11:54 DEBUG ] 22:11:54 DEBUG ] 22:11:54 DEBUG } 22:11:54 DEBUG } 22:11:54 DEBUG ] 22:11:54 DEBUG } 22:11:54 DEBUG } 22:11:54 DEBUG } 22:11:54 DEBUG } 22:11:54 DEBUG } 22:11:54 DEBUG } 22:11:54 DEBUG } 22:11:54 DEBUG } 22:11:54 DEBUG ] 22:11:54 DEBUG } 22:11:54 DEBUG } 22:11:54 DEBUG } 22:11:54 DEBUG } 22:11:54 DEBUG } 22:11:54 DEBUG } 22:11:54 DEBUG ] 22:11:54 DEBUG } 22:11:54 DEBUG } 22:11:54 DEBUG ] 22:11:54 DEBUG } 22:11:54 DEBUG } 22:11:54 DEBUG } 22:11:54 DEBUG ] 22:11:54 DEBUG } 22:11:54 DEBUG } 22:11:54 DEBUG ] 22:11:54 DEBUG } 22:11:54 DEBUG ] 22:11:54 DEBUG } 22:11:54 DEBUG } 22:11:54 DEBUG } 22:11:54 DEBUG ] 22:11:54 DEBUG } 22:11:54 DEBUG } 22:11:54 DEBUG ] 22:11:54 DEBUG } 22:11:54 DEBUG ] 22:11:54 DEBUG } 22:11:54 DEBUG } 22:11:54 DEBUG ] 22:11:54 DEBUG } 22:11:54 DEBUG } 22:11:54 DEBUG } 22:11:54 DEBUG } 22:11:54 DEBUG } 22:11:54 DEBUG } 22:11:54 DEBUG } 22:11:54 DEBUG } 22:11:54 DEBUG } 22:11:54 DEBUG ] 22:11:54 DEBUG } 22:11:54 DEBUG } 22:11:54 DEBUG } 22:11:54 DEBUG } 22:11:54 DEBUG } 22:11:54 DEBUG } 22:11:54 DEBUG } 22:11:54 DEBUG } 22:11:54 DEBUG } 22:11:54 DEBUG } 22:11:54 DEBUG } 22:11:54 DEBUG } 22:11:54 DEBUG } 22:11:54 DEBUG } 22:12:54 INFO 22:12:54 INFO [loop_until]: kubectl --namespace=xlou top pods 22:12:54 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:12:54 INFO 22:12:54 INFO [loop_until]: kubectl --namespace=xlou top node 22:12:54 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:12:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713301974" 22:12:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713301974" 22:12:54 INFO [http_cmd]: http status code OK 22:12:54 DEBUG --- status code --- 22:12:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:12:54 DEBUG --- http response --- 22:12:54 ERROR [http_cmd]: ERROR 22:12:54 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 22:12:54 ERROR --- status code --- 22:12:54 ERROR http status code is 400 (expected [200, 401, 403, 404]) 22:12:54 ERROR --- http response --- 22:12:54 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 22:12:54 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 22:12:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713301974" 22:12:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713301974" 22:12:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713301974" 22:12:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713301974" 22:12:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713301974" 22:12:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713301974" 22:12:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713301974" 22:12:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713301974" 22:12:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713301974" 22:12:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713301974" 22:12:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713301974" 22:12:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713301974" 22:12:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713301974" 22:12:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713301974" 22:12:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713301974" 22:12:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713301974" 22:12:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713301974" 22:12:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713301974" 22:12:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713301974" 22:12:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713301974" 22:12:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713301974" 22:12:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713301974" 22:12:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713301974" 22:12:54 INFO [loop_until]: OK (rc = 0) 22:12:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713301974" 22:12:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713301974" 22:12:54 INFO [http_cmd]: http status code OK 22:12:54 INFO [http_cmd]: http status code OK 22:12:54 INFO [loop_until]: OK (rc = 0) 22:12:54 INFO [http_cmd]: http status code OK 22:12:54 INFO [http_cmd]: http status code OK 22:12:54 INFO [http_cmd]: http status code OK 22:12:54 INFO [http_cmd]: http status code OK 22:12:54 INFO [http_cmd]: http status code OK 22:12:54 INFO [http_cmd]: http status code OK 22:12:54 INFO [http_cmd]: http status code OK 22:12:54 INFO [http_cmd]: http status code OK 22:12:54 INFO [http_cmd]: http status code OK 22:12:54 INFO [http_cmd]: http status code OK 22:12:54 INFO [http_cmd]: http status code OK 22:12:54 INFO [http_cmd]: http status code OK 22:12:54 INFO [http_cmd]: http status code OK 22:12:54 DEBUG --- stdout --- 22:12:54 INFO [http_cmd]: http status code OK 22:12:54 INFO [http_cmd]: http status code OK 22:12:54 INFO [http_cmd]: http status code OK 22:12:54 INFO [http_cmd]: http status code OK 22:12:54 DEBUG --- status code --- 22:12:54 INFO [http_cmd]: http status code OK 22:12:54 DEBUG --- status code --- 22:12:54 DEBUG --- stdout --- 22:12:54 DEBUG --- status code --- 22:12:54 DEBUG --- status code --- 22:12:54 DEBUG --- status code --- 22:12:54 DEBUG --- status code --- 22:12:54 DEBUG --- status code --- 22:12:54 DEBUG --- status code --- 22:12:54 DEBUG --- status code --- 22:12:54 DEBUG --- status code --- 22:12:54 DEBUG --- status code --- 22:12:54 DEBUG --- status code --- 22:12:54 DEBUG --- status code --- 22:12:54 INFO [http_cmd]: http status code OK 22:12:54 DEBUG --- status code --- 22:12:54 DEBUG --- status code --- 22:12:54 DEBUG --- status code --- 22:12:54 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 9Mi am-555f4db595-4lbss 9082m 4639Mi am-555f4db595-fz8sn 9252m 4664Mi am-555f4db595-kzxsp 9458m 4708Mi ds-cts-0 5547m 7109Mi ds-cts-1 5053m 6896Mi ds-cts-2 4778m 6850Mi ds-idrepo-0 4773m 5033Mi ds-idrepo-1 4498m 2825Mi ds-idrepo-2 4572m 2870Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 4m 1208Mi idm-b85f9d978-rl76f 3m 1226Mi lodemon-bc756c647-cr7x6 9m 62Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-7c9b667974-9mr2s 6520m 477Mi 22:12:54 DEBUG --- status code --- 22:12:54 DEBUG --- status code --- 22:12:54 DEBUG --- status code --- 22:12:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:12:54 DEBUG --- status code --- 22:12:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:12:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:12:54 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 62m 0% 5086Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5024m 16% 9459Mi 8% gke-xlou-cdm-default-pool-1b0769c8-nhlf 9332m 31% 6847Mi 6% gke-xlou-cdm-default-pool-1b0769c8-qkns 87m 0% 2654Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 4740m 15% 6215Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 6035m 20% 11681Mi 10% gke-xlou-cdm-default-pool-1b0769c8-xnmp 66m 0% 3145Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 9388m 31% 6442Mi 5% gke-xlou-cdm-ds-0f601751-hmm7 4704m 15% 5557Mi 4% gke-xlou-cdm-ds-0f601751-j488 4843m 16% 8779Mi 7% gke-xlou-cdm-ds-0f601751-mc5d 10310m 34% 3331Mi 2% gke-xlou-cdm-ds-0f601751-nj6q 9254m 30% 6649Mi 5% gke-xlou-cdm-ds-0f601751-nmxm 4691m 15% 7356Mi 6% gke-xlou-cdm-frontend-a8771548-i82n 6588m 41% 5922Mi 10% 22:12:54 INFO [http_cmd]: http status code OK 22:12:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:12:54 INFO [http_cmd]: http status code OK 22:12:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:12:54 INFO [http_cmd]: http status code OK 22:12:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:12:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:12:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:12:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:12:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:12:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:12:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:12:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:12:54 DEBUG --- status code --- 22:12:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:12:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:12:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:12:54 DEBUG --- stderr --- 22:12:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:12:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:12:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:12:54 DEBUG --- http response --- 22:12:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:12:54 DEBUG --- http response --- 22:12:54 DEBUG --- http response --- 22:12:54 INFO [http_cmd]: http status code OK 22:12:54 DEBUG --- stderr --- 22:12:54 DEBUG --- status code --- 22:12:54 DEBUG --- http response --- 22:12:54 DEBUG --- status code --- 22:12:54 DEBUG --- http response --- 22:12:54 DEBUG --- status code --- 22:12:54 DEBUG --- http response --- 22:12:54 DEBUG --- http response --- 22:12:54 DEBUG --- http response --- 22:12:54 DEBUG --- http response --- 22:12:54 DEBUG --- http response --- 22:12:54 DEBUG --- http response --- 22:12:54 DEBUG --- http response --- 22:12:54 DEBUG --- http response --- 22:12:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:12:54 DEBUG --- http response --- 22:12:54 DEBUG --- http response --- 22:12:54 DEBUG --- http response --- 22:12:54 DEBUG 22:12:54 DEBUG --- http response --- 22:12:54 DEBUG --- http response --- 22:12:54 DEBUG --- http response --- 22:12:54 DEBUG --- http response --- 22:12:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 22:12:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:12:54 DEBUG --- status code --- 22:12:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 22:12:54 DEBUG 22:12:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:12:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:12:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:12:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:12:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:12:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:12:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:12:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:12:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:12:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:12:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:12:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 22:12:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:12:54 DEBUG --- http response --- 22:12:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:12:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:12:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:12:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:12:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:12:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:12:54 DEBUG ----- output ----- 22:12:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 22:12:54 DEBUG ----- output ----- 22:12:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:12:54 DEBUG ----- output ----- 22:12:54 DEBUG --- http response --- 22:12:54 DEBUG ----- output ----- 22:12:54 DEBUG --- http response --- 22:12:54 DEBUG ----- output ----- 22:12:54 DEBUG --- http response --- 22:12:54 DEBUG ----- output ----- 22:12:54 DEBUG ----- output ----- 22:12:54 DEBUG ----- output ----- 22:12:54 DEBUG ----- output ----- 22:12:54 DEBUG ----- output ----- 22:12:54 DEBUG ----- output ----- 22:12:54 DEBUG ----- output ----- 22:12:54 DEBUG ----- output ----- 22:12:54 DEBUG ----- output ----- 22:12:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:12:54 DEBUG ----- output ----- 22:12:54 DEBUG ----- output ----- 22:12:54 DEBUG ----- output ----- 22:12:54 DEBUG ----- output ----- 22:12:54 DEBUG ----- output ----- 22:12:54 DEBUG { 22:12:54 DEBUG ----- output ----- 22:12:54 DEBUG { 22:12:54 DEBUG --- http response --- 22:12:54 DEBUG { 22:12:54 DEBUG { 22:12:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:12:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:12:54 DEBUG { 22:12:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:12:54 DEBUG { 22:12:54 DEBUG { 22:12:54 DEBUG { 22:12:54 DEBUG { 22:12:54 DEBUG { 22:12:54 DEBUG { 22:12:54 DEBUG { 22:12:54 DEBUG { 22:12:54 DEBUG { 22:12:54 DEBUG ----- output ----- 22:12:54 DEBUG { 22:12:54 DEBUG { 22:12:54 DEBUG { 22:12:54 DEBUG { 22:12:54 DEBUG { 22:12:54 DEBUG "status": "success", 22:12:54 DEBUG { 22:12:54 DEBUG "status": "success", 22:12:54 DEBUG "status": "success", 22:12:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:12:54 DEBUG "status": "success", 22:12:54 DEBUG ----- output ----- 22:12:54 DEBUG ----- output ----- 22:12:54 DEBUG "status": "success", 22:12:54 DEBUG ----- output ----- 22:12:54 DEBUG "status": "success", 22:12:54 DEBUG "status": "success", 22:12:54 DEBUG "status": "success", 22:12:54 DEBUG "status": "success", 22:12:54 DEBUG "status": "success", 22:12:54 DEBUG "status": "success", 22:12:54 DEBUG "status": "success", 22:12:54 DEBUG "status": "success", 22:12:54 DEBUG "status": "success", 22:12:54 DEBUG { 22:12:54 DEBUG "status": "success", 22:12:54 DEBUG "status": "success", 22:12:54 DEBUG "status": "success", 22:12:54 DEBUG "status": "success", 22:12:54 DEBUG "status": "success", 22:12:54 DEBUG "data": { 22:12:54 DEBUG "status": "success", 22:12:54 DEBUG "data": { 22:12:54 DEBUG "data": { 22:12:54 DEBUG ----- output ----- 22:12:54 DEBUG "data": { 22:12:54 DEBUG { 22:12:54 DEBUG { 22:12:54 DEBUG "data": { 22:12:54 DEBUG { 22:12:54 DEBUG "data": { 22:12:54 DEBUG "data": { 22:12:54 DEBUG "data": { 22:12:54 DEBUG "data": { 22:12:54 DEBUG "data": { 22:12:54 DEBUG "data": { 22:12:54 DEBUG "data": { 22:12:54 DEBUG "data": { 22:12:54 DEBUG "data": { 22:12:54 DEBUG "status": "success", 22:12:54 DEBUG "data": { 22:12:54 DEBUG "data": { 22:12:54 DEBUG "data": { 22:12:54 DEBUG "data": { 22:12:54 DEBUG "data": { 22:12:54 DEBUG "resultType": "vector", 22:12:54 DEBUG "data": { 22:12:54 DEBUG "resultType": "vector", 22:12:54 DEBUG "resultType": "vector", 22:12:54 DEBUG { 22:12:54 DEBUG "resultType": "vector", 22:12:54 DEBUG "status": "success", 22:12:54 DEBUG "status": "success", 22:12:54 DEBUG "resultType": "vector", 22:12:54 DEBUG "status": "success", 22:12:54 DEBUG "resultType": "vector", 22:12:54 DEBUG "resultType": "vector", 22:12:54 DEBUG "resultType": "vector", 22:12:54 DEBUG "resultType": "vector", 22:12:54 DEBUG "resultType": "vector", 22:12:54 DEBUG "resultType": "vector", 22:12:54 DEBUG "resultType": "vector", 22:12:54 DEBUG "resultType": "vector", 22:12:54 DEBUG "resultType": "vector", 22:12:54 DEBUG "data": { 22:12:54 DEBUG "resultType": "vector", 22:12:54 DEBUG "resultType": "vector", 22:12:54 DEBUG "resultType": "vector", 22:12:54 DEBUG "resultType": "vector", 22:12:54 DEBUG "resultType": "vector", 22:12:54 DEBUG "result": [ 22:12:54 DEBUG "resultType": "vector", 22:12:54 DEBUG "result": [ 22:12:54 DEBUG "result": [ 22:12:54 DEBUG "status": "success", 22:12:54 DEBUG "result": [ 22:12:54 DEBUG "data": { 22:12:54 DEBUG "data": { 22:12:54 DEBUG "result": [ 22:12:54 DEBUG "data": { 22:12:54 DEBUG "result": [ 22:12:54 DEBUG "result": [ 22:12:54 DEBUG "result": [ 22:12:54 DEBUG "result": [ 22:12:54 DEBUG "result": [ 22:12:54 DEBUG "result": [ 22:12:54 DEBUG "result": [ 22:12:54 DEBUG "result": [ 22:12:54 DEBUG "result": [ 22:12:54 DEBUG "resultType": "vector", 22:12:54 DEBUG "result": [ 22:12:54 DEBUG "result": [ 22:12:54 DEBUG { 22:12:54 DEBUG "result": [ 22:12:54 DEBUG "result": [ 22:12:54 DEBUG { 22:12:54 DEBUG "result": [ 22:12:54 DEBUG { 22:12:54 DEBUG { 22:12:54 DEBUG "data": { 22:12:54 DEBUG { 22:12:54 DEBUG "resultType": "vector", 22:12:54 DEBUG "resultType": "vector", 22:12:54 DEBUG { 22:12:54 DEBUG "resultType": "vector", 22:12:54 DEBUG { 22:12:54 DEBUG { 22:12:54 DEBUG { 22:12:54 DEBUG { 22:12:54 DEBUG { 22:12:54 DEBUG { 22:12:54 DEBUG { 22:12:54 DEBUG { 22:12:54 DEBUG { 22:12:54 DEBUG "result": [ 22:12:54 DEBUG { 22:12:54 DEBUG "result": [ 22:12:54 DEBUG "metric": { 22:12:54 DEBUG { 22:12:54 DEBUG { 22:12:54 DEBUG "metric": { 22:12:54 DEBUG { 22:12:54 DEBUG "metric": { 22:12:54 DEBUG "metric": { 22:12:54 DEBUG "resultType": "vector", 22:12:54 DEBUG "metric": { 22:12:54 DEBUG "result": [ 22:12:54 DEBUG "result": [ 22:12:54 DEBUG "metric": { 22:12:54 DEBUG "result": [ 22:12:54 DEBUG "metric": { 22:12:54 DEBUG "metric": { 22:12:54 DEBUG "metric": { 22:12:54 DEBUG "metric": { 22:12:54 DEBUG "metric": { 22:12:54 DEBUG "metric": { 22:12:54 DEBUG "metric": { 22:12:54 DEBUG "metric": { 22:12:54 DEBUG "metric": { 22:12:54 DEBUG { 22:12:54 DEBUG "metric": { 22:12:54 DEBUG { 22:12:54 DEBUG "pod": "am-555f4db595-fz8sn" 22:12:54 DEBUG "metric": { 22:12:54 DEBUG "metric": { 22:12:54 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:12:54 DEBUG "metric": { 22:12:54 DEBUG "instance": "10.142.0.107:9100" 22:12:54 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:12:54 DEBUG "result": [ 22:12:54 DEBUG "pod": "ds-idrepo-2" 22:12:54 DEBUG { 22:12:54 DEBUG { 22:12:54 DEBUG "pod": "am-555f4db595-fz8sn" 22:12:54 DEBUG { 22:12:54 DEBUG "instance": "10.142.0.107:9100" 22:12:54 DEBUG "pod": "am-555f4db595-fz8sn" 22:12:54 DEBUG "pod": "ds-idrepo-2" 22:12:54 DEBUG "pod": "ds-idrepo-2" 22:12:54 DEBUG "instance": "10.142.0.107:9100" 22:12:54 DEBUG "pod": "am-555f4db595-fz8sn" 22:12:54 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:12:54 DEBUG "pod": "ds-idrepo-2" 22:12:54 DEBUG "pod": "am-555f4db595-fz8sn" 22:12:54 DEBUG "metric": { 22:12:54 DEBUG "pod": "ds-idrepo-2" 22:12:54 DEBUG "metric": { 22:12:54 DEBUG }, 22:12:54 DEBUG "pod": "am-555f4db595-fz8sn" 22:12:54 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:12:54 DEBUG }, 22:12:54 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:12:54 DEBUG }, 22:12:54 DEBUG }, 22:12:54 DEBUG { 22:12:54 DEBUG }, 22:12:54 DEBUG "metric": { 22:12:54 DEBUG "pod": "am-555f4db595-fz8sn" 22:12:54 DEBUG }, 22:12:54 DEBUG "metric": { 22:12:54 DEBUG }, 22:12:54 DEBUG }, 22:12:54 DEBUG }, 22:12:54 DEBUG }, 22:12:54 DEBUG }, 22:12:54 DEBUG }, 22:12:54 DEBUG }, 22:12:54 DEBUG }, 22:12:54 DEBUG }, 22:12:54 DEBUG "instance": "10.142.0.107:9100" 22:12:54 DEBUG }, 22:12:54 DEBUG "pod": "ds-idrepo-2" 22:12:54 DEBUG "value": [ 22:12:54 DEBUG }, 22:12:54 DEBUG }, 22:12:54 DEBUG "value": [ 22:12:54 DEBUG }, 22:12:54 DEBUG "value": [ 22:12:54 DEBUG "value": [ 22:12:54 DEBUG "metric": { 22:12:54 DEBUG "value": [ 22:12:54 DEBUG "metric": { 22:12:54 DEBUG }, 22:12:54 DEBUG "value": [ 22:12:54 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:12:54 DEBUG "value": [ 22:12:54 DEBUG "value": [ 22:12:54 DEBUG "value": [ 22:12:54 DEBUG "value": [ 22:12:54 DEBUG "value": [ 22:12:54 DEBUG "value": [ 22:12:54 DEBUG "value": [ 22:12:54 DEBUG "value": [ 22:12:54 DEBUG "value": [ 22:12:54 DEBUG }, 22:12:54 DEBUG "value": [ 22:12:54 DEBUG }, 22:12:54 DEBUG 22:12:54 DEBUG "value": [ 22:12:54 DEBUG "value": [ 22:12:54 DEBUG 22:12:54 DEBUG "value": [ 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG "instance": "10.142.0.107:9100" 22:12:54 DEBUG 22:12:54 DEBUG "pod": "ds-idrepo-2" 22:12:54 DEBUG "value": [ 22:12:54 DEBUG 22:12:54 DEBUG }, 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG "value": [ 22:12:54 DEBUG 22:12:54 DEBUG "value": [ 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG }, 22:12:54 DEBUG 22:12:54 DEBUG }, 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG "value": [ 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG "value": [ 22:12:54 DEBUG 22:12:54 DEBUG "value": [ 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG [...] 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG [...] 22:12:54 DEBUG 22:12:54 DEBUG [...] 22:12:54 DEBUG [...] 22:12:54 DEBUG 22:12:54 DEBUG [...] 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG [...] 22:12:54 DEBUG 22:12:54 DEBUG [...] 22:12:54 DEBUG [...] 22:12:54 DEBUG [...] 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG [...] 22:12:54 DEBUG [...] 22:12:54 DEBUG [...] 22:12:54 DEBUG [...] 22:12:54 DEBUG 22:12:54 DEBUG [...] 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG [...] 22:12:54 DEBUG [...] 22:12:54 DEBUG 22:12:54 DEBUG [...] 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG [...] 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG [...] 22:12:54 DEBUG [...] 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG [...] 22:12:54 DEBUG 22:12:54 DEBUG [...] 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG [...] 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG "pod": "ds-idrepo-0" 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG [...] 22:12:54 DEBUG 22:12:54 DEBUG [...] 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG "instance": "10.142.0.43:9100" 22:12:54 DEBUG 22:12:54 DEBUG }, 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG "pod": "am-555f4db595-kzxsp" 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:12:54 DEBUG 22:12:54 DEBUG "instance": "10.142.0.43:9100" 22:12:54 DEBUG "pod": "ds-idrepo-2" 22:12:54 DEBUG 22:12:54 DEBUG "pod": "ds-idrepo-0" 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG "pod": "am-555f4db595-kzxsp" 22:12:54 DEBUG 22:12:54 DEBUG }, 22:12:54 DEBUG 22:12:54 DEBUG "value": [ 22:12:54 DEBUG 22:12:54 DEBUG 22:12:54 DEBUG "pod": "am-555f4db595-kzxsp" 22:12:54 DEBUG "pod": "ds-idrepo-2" 22:12:54 DEBUG "pod": "ds-idrepo-0" 22:12:54 DEBUG "pod": "am-555f4db595-kzxsp" 22:12:54 DEBUG 22:12:54 DEBUG "pod": "ds-idrepo-0" 22:12:54 DEBUG 22:12:54 DEBUG }, 22:12:54 DEBUG "pod": "am-555f4db595-kzxsp" 22:12:54 DEBUG "pod": "ds-idrepo-2" 22:12:54 DEBUG }, 22:12:54 DEBUG "pod": "ds-idrepo-2" 22:12:54 DEBUG }, 22:12:54 DEBUG }, 22:12:54 DEBUG 22:12:54 DEBUG }, 22:12:54 DEBUG 22:12:54 DEBUG "pod": "am-555f4db595-kzxsp" 22:12:54 DEBUG }, 22:12:54 DEBUG 22:12:54 DEBUG "value": [ 22:12:54 DEBUG "pod": "am-555f4db595-kzxsp" 22:12:54 DEBUG 1713301974, 22:12:54 DEBUG "instance": "10.142.0.43:9100" 22:12:54 DEBUG "pod": "ds-idrepo-0" 22:12:54 DEBUG }, 22:12:54 DEBUG }, 22:12:54 DEBUG }, 22:12:54 DEBUG }, 22:12:54 DEBUG "value": [ 22:12:54 DEBUG }, 22:12:54 DEBUG "pod": "ds-cts-2" 22:12:54 DEBUG "value": [ 22:12:54 DEBUG }, 22:12:54 DEBUG }, 22:12:54 DEBUG "value": [ 22:12:54 DEBUG }, 22:12:54 DEBUG "value": [ 22:12:54 DEBUG "value": [ 22:12:54 DEBUG 22:12:54 DEBUG "value": [ 22:12:54 DEBUG 22:12:54 DEBUG }, 22:12:54 DEBUG "value": [ 22:12:54 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:12:54 DEBUG 1713301974, 22:12:54 DEBUG }, 22:12:54 DEBUG "0" 22:12:54 DEBUG }, 22:12:54 DEBUG }, 22:12:54 DEBUG "value": [ 22:12:54 DEBUG "value": [ 22:12:54 DEBUG "value": [ 22:12:54 DEBUG "instance": "10.142.0.43:9100" 22:12:54 DEBUG 1713301974, 22:12:54 DEBUG "value": [ 22:12:54 DEBUG }, 22:12:54 DEBUG "value": [ 22:12:54 DEBUG "value": [ 22:12:54 DEBUG "value": [ 22:12:54 DEBUG 1713301974, 22:12:54 DEBUG "value": [ 22:12:54 DEBUG 1713301974, 22:12:54 DEBUG 1713301974, 22:12:54 DEBUG "instance": "10.142.0.43:9100" 22:12:54 DEBUG 1713301974, 22:12:54 DEBUG "pod": "ds-idrepo-0" 22:12:54 DEBUG "value": [ 22:12:54 DEBUG 1713301974, 22:12:54 DEBUG }, 22:12:54 DEBUG "45569371.93333333" 22:12:54 DEBUG "value": [ 22:12:54 DEBUG ] 22:12:54 DEBUG "value": [ 22:12:54 DEBUG "value": [ 22:12:54 DEBUG 1713301974, 22:12:54 DEBUG "2004.9333333333334" 22:12:54 DEBUG 1713301974, 22:12:54 DEBUG }, 22:12:54 DEBUG "2004.6666666666667" 22:12:54 DEBUG 1713301974, 22:12:54 DEBUG 1713301974, 22:12:54 DEBUG 1713301974, 22:12:54 DEBUG 1713301974, 22:12:54 DEBUG 1713301974, 22:12:54 DEBUG "6.447623087463699" 22:12:54 DEBUG 1713301974, 22:12:54 DEBUG "45560085.666666664" 22:12:54 DEBUG "2.945623509013241" 22:12:54 DEBUG }, 22:12:54 DEBUG "0" 22:12:54 DEBUG }, 22:12:54 DEBUG 1713301974, 22:12:54 DEBUG "2005" 22:12:54 DEBUG "value": [ 22:12:54 DEBUG ] 22:12:54 DEBUG 1713301974, 22:12:54 DEBUG } 22:12:54 DEBUG 1713301974, 22:12:54 DEBUG 1713301974, 22:12:54 DEBUG ] 22:12:54 DEBUG 1713301974, 22:12:54 DEBUG "0" 22:12:54 DEBUG "value": [ 22:12:54 DEBUG ] 22:12:54 DEBUG "0" 22:12:54 DEBUG "0" 22:12:54 DEBUG "0" 22:12:54 DEBUG "0" 22:12:54 DEBUG "105.28390417221831" 22:12:54 DEBUG ] 22:12:54 DEBUG "236783.500483319" 22:12:54 DEBUG ] 22:12:54 DEBUG ] 22:12:54 DEBUG "value": [ 22:12:54 DEBUG ] 22:12:54 DEBUG "value": [ 22:12:54 DEBUG "0.023" 22:12:54 DEBUG ] 22:12:54 DEBUG 1713301974, 22:12:54 DEBUG } 22:12:54 DEBUG "80609.5" 22:12:54 DEBUG ] 22:12:54 DEBUG "0.0006666666666666598" 22:12:54 DEBUG "0" 22:12:54 DEBUG } 22:12:54 DEBUG "3684.936646027641" 22:12:54 DEBUG ] 22:12:54 DEBUG 1713301974, 22:12:54 DEBUG } 22:12:54 DEBUG ] 22:12:54 DEBUG ] 22:12:54 DEBUG } 22:12:54 DEBUG ] 22:12:54 DEBUG ] 22:12:54 DEBUG } 22:12:54 DEBUG ] 22:12:54 DEBUG } 22:12:54 DEBUG } 22:12:54 DEBUG 1713301974, 22:12:54 DEBUG } 22:12:54 DEBUG 1713301974, 22:12:54 DEBUG ] 22:12:54 DEBUG } 22:12:54 DEBUG "600666112" 22:12:54 DEBUG ] 22:12:54 DEBUG ] 22:12:54 DEBUG } 22:12:54 DEBUG ] 22:12:54 DEBUG ] 22:12:54 DEBUG ] 22:12:54 DEBUG ] 22:12:54 DEBUG } 22:12:54 DEBUG "19946370048" 22:12:54 DEBUG ] 22:12:54 DEBUG } 22:12:54 DEBUG ] 22:12:54 DEBUG ] 22:12:54 DEBUG } 22:12:54 DEBUG } 22:12:54 DEBUG ] 22:12:54 DEBUG } 22:12:54 DEBUG ] 22:12:54 DEBUG } 22:12:54 DEBUG "0.0006333333333335152" 22:12:54 DEBUG ] 22:12:54 DEBUG "0" 22:12:54 DEBUG } 22:12:54 DEBUG ] 22:12:54 DEBUG ] 22:12:54 DEBUG } 22:12:54 DEBUG } 22:12:54 DEBUG } 22:12:54 DEBUG } 22:12:54 DEBUG } 22:12:54 DEBUG } 22:12:54 DEBUG } 22:12:54 DEBUG ] 22:12:54 DEBUG ] 22:12:54 DEBUG } 22:12:54 DEBUG ] 22:12:54 DEBUG } 22:12:54 DEBUG } 22:12:54 DEBUG ] 22:12:54 DEBUG ] 22:12:54 DEBUG } 22:12:54 DEBUG ] 22:12:54 DEBUG ] 22:12:54 DEBUG } 22:12:54 DEBUG ] 22:12:54 DEBUG } 22:12:54 DEBUG ] 22:12:54 DEBUG ] 22:12:54 DEBUG } 22:12:54 DEBUG } 22:12:54 DEBUG } 22:12:54 DEBUG ] 22:12:54 DEBUG ] 22:12:54 DEBUG ] 22:12:54 DEBUG } 22:12:54 DEBUG ] 22:12:54 DEBUG } 22:12:54 DEBUG } 22:12:54 DEBUG } 22:12:54 DEBUG } 22:12:54 DEBUG ] 22:12:54 DEBUG } 22:12:54 DEBUG } 22:12:54 DEBUG } 22:12:54 DEBUG } 22:12:54 DEBUG } 22:12:54 DEBUG } 22:12:54 DEBUG } 22:12:54 DEBUG } 22:12:54 DEBUG } 22:12:54 DEBUG } 22:12:54 DEBUG } 22:12:54 DEBUG ] 22:12:54 DEBUG } 22:12:54 DEBUG } 22:12:54 DEBUG } 22:12:54 DEBUG } 22:12:54 DEBUG } 22:12:54 DEBUG ] 22:12:54 DEBUG } 22:12:54 DEBUG } 22:12:54 DEBUG } 22:12:54 DEBUG } 22:12:54 DEBUG } 22:12:54 DEBUG } 22:12:54 DEBUG ] 22:12:54 DEBUG ] 22:12:54 DEBUG } 22:12:54 DEBUG } 22:12:54 DEBUG } 22:12:54 DEBUG } 22:12:54 DEBUG } 22:12:54 DEBUG } 22:12:54 DEBUG } 22:12:54 DEBUG } 22:12:54 DEBUG } 22:12:54 DEBUG } 22:12:54 DEBUG } 22:12:54 DEBUG } 22:12:54 DEBUG } 22:12:54 DEBUG } 22:13:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302034" 22:13:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713302034" 22:13:54 ERROR [http_cmd]: ERROR 22:13:54 INFO [http_cmd]: http status code OK 22:13:54 ERROR --- status code --- 22:13:54 DEBUG --- status code --- 22:13:54 ERROR http status code is 400 (expected [200, 401, 403, 404]) 22:13:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:13:54 ERROR --- http response --- 22:13:54 DEBUG --- http response --- 22:13:54 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 22:13:54 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 22:13:54 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 22:13:54 INFO 22:13:54 INFO [loop_until]: kubectl --namespace=xlou top pods 22:13:54 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:13:54 INFO 22:13:54 INFO [loop_until]: kubectl --namespace=xlou top node 22:13:54 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:13:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302034" 22:13:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302034" 22:13:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302034" 22:13:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302034" 22:13:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302034" 22:13:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302034" 22:13:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713302034" 22:13:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302034" 22:13:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302034" 22:13:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302034" 22:13:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302034" 22:13:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302034" 22:13:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302034" 22:13:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302034" 22:13:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302034" 22:13:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302034" 22:13:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302034" 22:13:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302034" 22:13:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713302034" 22:13:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302034" 22:13:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713302034" 22:13:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302034" 22:13:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302034" 22:13:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713302034" 22:13:54 INFO [loop_until]: OK (rc = 0) 22:13:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302034" 22:13:54 INFO [http_cmd]: http status code OK 22:13:54 INFO [http_cmd]: http status code OK 22:13:54 INFO [http_cmd]: http status code OK 22:13:54 INFO [http_cmd]: http status code OK 22:13:54 INFO [http_cmd]: http status code OK 22:13:54 INFO [http_cmd]: http status code OK 22:13:54 INFO [http_cmd]: http status code OK 22:13:54 INFO [http_cmd]: http status code OK 22:13:54 INFO [http_cmd]: http status code OK 22:13:54 INFO [http_cmd]: http status code OK 22:13:54 INFO [http_cmd]: http status code OK 22:13:54 INFO [http_cmd]: http status code OK 22:13:54 INFO [http_cmd]: http status code OK 22:13:54 INFO [http_cmd]: http status code OK 22:13:54 INFO [http_cmd]: http status code OK 22:13:54 INFO [http_cmd]: http status code OK 22:13:54 DEBUG --- stdout --- 22:13:54 INFO [http_cmd]: http status code OK 22:13:54 DEBUG --- status code --- 22:13:54 INFO [http_cmd]: http status code OK 22:13:54 INFO [http_cmd]: http status code OK 22:13:54 DEBUG --- status code --- 22:13:54 INFO [http_cmd]: http status code OK 22:13:54 DEBUG --- status code --- 22:13:54 DEBUG --- status code --- 22:13:54 DEBUG --- status code --- 22:13:54 DEBUG --- status code --- 22:13:54 DEBUG --- status code --- 22:13:54 DEBUG --- status code --- 22:13:54 DEBUG --- status code --- 22:13:54 DEBUG --- status code --- 22:13:54 INFO [http_cmd]: http status code OK 22:13:54 DEBUG --- status code --- 22:13:54 DEBUG --- status code --- 22:13:54 DEBUG --- status code --- 22:13:54 DEBUG --- status code --- 22:13:54 DEBUG --- status code --- 22:13:54 DEBUG --- status code --- 22:13:54 DEBUG --- status code --- 22:13:54 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 9Mi am-555f4db595-4lbss 9520m 4698Mi am-555f4db595-fz8sn 9154m 4684Mi am-555f4db595-kzxsp 9347m 4911Mi ds-cts-0 5625m 7154Mi ds-cts-1 5041m 6959Mi ds-cts-2 4877m 6887Mi ds-idrepo-0 4648m 5032Mi ds-idrepo-1 4394m 2833Mi ds-idrepo-2 4478m 2870Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 3m 1208Mi idm-b85f9d978-rl76f 3m 1225Mi lodemon-bc756c647-cr7x6 2m 62Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-7c9b667974-9mr2s 6382m 504Mi 22:13:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:13:54 INFO [http_cmd]: http status code OK 22:13:54 DEBUG --- status code --- 22:13:54 INFO [http_cmd]: http status code OK 22:13:54 INFO [http_cmd]: http status code OK 22:13:54 DEBUG --- status code --- 22:13:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:13:54 DEBUG --- status code --- 22:13:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:13:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:13:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:13:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:13:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:13:54 DEBUG --- http response --- 22:13:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:13:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:13:54 DEBUG --- status code --- 22:13:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:13:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:13:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:13:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:13:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:13:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:13:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:13:54 DEBUG --- stderr --- 22:13:54 DEBUG --- http response --- 22:13:54 DEBUG --- status code --- 22:13:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:13:54 DEBUG --- status code --- 22:13:54 DEBUG --- status code --- 22:13:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:13:54 DEBUG --- http response --- 22:13:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:13:54 DEBUG --- http response --- 22:13:54 DEBUG --- http response --- 22:13:54 DEBUG --- http response --- 22:13:54 DEBUG --- http response --- 22:13:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:13:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:13:54 DEBUG --- http response --- 22:13:54 DEBUG --- http response --- 22:13:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:13:54 DEBUG --- http response --- 22:13:54 DEBUG --- http response --- 22:13:54 DEBUG --- http response --- 22:13:54 DEBUG --- http response --- 22:13:54 DEBUG --- http response --- 22:13:54 DEBUG --- http response --- 22:13:54 DEBUG --- http response --- 22:13:54 DEBUG 22:13:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:13:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:13:54 DEBUG --- http response --- 22:13:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:13:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:13:54 DEBUG --- http response --- 22:13:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:13:54 DEBUG --- http response --- 22:13:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:13:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:13:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:13:54 INFO [http_cmd]: http status code OK 22:13:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:13:54 DEBUG --- http response --- 22:13:54 DEBUG ----- output ----- 22:13:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:13:54 DEBUG --- http response --- 22:13:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:13:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:13:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:13:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:13:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:13:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:13:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:13:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:13:54 DEBUG ----- output ----- 22:13:54 DEBUG --- http response --- 22:13:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:13:54 DEBUG --- http response --- 22:13:54 DEBUG --- http response --- 22:13:54 DEBUG ----- output ----- 22:13:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:13:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:13:54 DEBUG ----- output ----- 22:13:54 DEBUG ----- output ----- 22:13:54 DEBUG ----- output ----- 22:13:54 DEBUG --- status code --- 22:13:54 DEBUG ----- output ----- 22:13:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:13:54 DEBUG { 22:13:54 DEBUG ----- output ----- 22:13:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:13:54 DEBUG ----- output ----- 22:13:54 DEBUG ----- output ----- 22:13:54 DEBUG ----- output ----- 22:13:54 DEBUG ----- output ----- 22:13:54 DEBUG ----- output ----- 22:13:54 DEBUG ----- output ----- 22:13:54 DEBUG ----- output ----- 22:13:54 DEBUG ----- output ----- 22:13:54 DEBUG { 22:13:54 DEBUG ----- output ----- 22:13:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:13:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:13:54 DEBUG { 22:13:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:13:54 DEBUG ----- output ----- 22:13:54 DEBUG ----- output ----- 22:13:54 DEBUG { 22:13:54 DEBUG { 22:13:54 DEBUG { 22:13:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:13:54 DEBUG { 22:13:54 DEBUG ----- output ----- 22:13:54 DEBUG "status": "success", 22:13:54 DEBUG { 22:13:54 DEBUG ----- output ----- 22:13:54 DEBUG { 22:13:54 DEBUG { 22:13:54 DEBUG { 22:13:54 DEBUG { 22:13:54 DEBUG { 22:13:54 DEBUG { 22:13:54 DEBUG { 22:13:54 DEBUG { 22:13:54 DEBUG "status": "success", 22:13:54 DEBUG "data": { 22:13:54 DEBUG ----- output ----- 22:13:54 DEBUG ----- output ----- 22:13:54 DEBUG "status": "success", 22:13:54 DEBUG ----- output ----- 22:13:54 DEBUG { 22:13:54 DEBUG { 22:13:54 DEBUG "status": "success", 22:13:54 DEBUG "status": "success", 22:13:54 DEBUG "status": "success", 22:13:54 DEBUG --- http response --- 22:13:54 DEBUG "status": "success", 22:13:54 DEBUG { 22:13:54 DEBUG "data": { 22:13:54 DEBUG "status": "success", 22:13:54 DEBUG { 22:13:54 DEBUG "status": "success", 22:13:54 DEBUG "status": "success", 22:13:54 DEBUG "status": "success", 22:13:54 DEBUG "status": "success", 22:13:54 DEBUG "status": "success", 22:13:54 DEBUG "status": "success", 22:13:54 DEBUG "status": "success", 22:13:54 DEBUG "status": "success", 22:13:54 DEBUG "data": { 22:13:54 DEBUG "resultType": "vector", 22:13:54 DEBUG { 22:13:54 DEBUG { 22:13:54 DEBUG "data": { 22:13:54 DEBUG { 22:13:54 DEBUG "status": "success", 22:13:54 DEBUG "status": "success", 22:13:54 DEBUG "data": { 22:13:54 DEBUG "data": { 22:13:54 DEBUG "data": { 22:13:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:13:54 DEBUG "data": { 22:13:54 DEBUG "status": "success", 22:13:54 DEBUG "resultType": "vector", 22:13:54 DEBUG "data": { 22:13:54 DEBUG "status": "success", 22:13:54 DEBUG "data": { 22:13:54 DEBUG "data": { 22:13:54 DEBUG "data": { 22:13:54 DEBUG "data": { 22:13:54 DEBUG "data": { 22:13:54 DEBUG "data": { 22:13:54 DEBUG "data": { 22:13:54 DEBUG "resultType": "vector", 22:13:54 DEBUG { 22:13:54 DEBUG "result": [ 22:13:54 DEBUG "status": "success", 22:13:54 DEBUG "status": "success", 22:13:54 DEBUG "resultType": "vector", 22:13:54 DEBUG "status": "success", 22:13:54 DEBUG "data": { 22:13:54 DEBUG "data": { 22:13:54 DEBUG "resultType": "vector", 22:13:54 DEBUG "resultType": "vector", 22:13:54 DEBUG "resultType": "vector", 22:13:54 DEBUG ----- output ----- 22:13:54 DEBUG "resultType": "vector", 22:13:54 DEBUG "data": { 22:13:54 DEBUG "result": [ 22:13:54 DEBUG "resultType": "vector", 22:13:54 DEBUG "data": { 22:13:54 DEBUG "resultType": "vector", 22:13:54 DEBUG "resultType": "vector", 22:13:54 DEBUG "resultType": "vector", 22:13:54 DEBUG "resultType": "vector", 22:13:54 DEBUG "resultType": "vector", 22:13:54 DEBUG "resultType": "vector", 22:13:54 DEBUG "resultType": "vector", 22:13:54 DEBUG "result": [ 22:13:54 DEBUG "status": "success", 22:13:54 DEBUG { 22:13:54 DEBUG "data": { 22:13:54 DEBUG "data": { 22:13:54 DEBUG "result": [ 22:13:54 DEBUG "data": { 22:13:54 DEBUG "resultType": "vector", 22:13:54 DEBUG "resultType": "vector", 22:13:54 DEBUG "result": [ 22:13:54 DEBUG "result": [ 22:13:54 DEBUG "result": [ 22:13:54 DEBUG { 22:13:54 DEBUG "result": [ 22:13:54 DEBUG "resultType": "vector", 22:13:54 DEBUG { 22:13:54 DEBUG "result": [ 22:13:54 DEBUG "resultType": "vector", 22:13:54 DEBUG "result": [ 22:13:54 DEBUG "result": [ 22:13:54 DEBUG "result": [ 22:13:54 DEBUG "result": [ 22:13:54 DEBUG "result": [ 22:13:54 DEBUG "result": [ 22:13:54 DEBUG "result": [ 22:13:54 DEBUG { 22:13:54 DEBUG "data": { 22:13:54 DEBUG "metric": { 22:13:54 DEBUG "resultType": "vector", 22:13:54 DEBUG "resultType": "vector", 22:13:54 DEBUG { 22:13:54 DEBUG "resultType": "vector", 22:13:54 DEBUG "result": [ 22:13:54 DEBUG "result": [ 22:13:54 DEBUG { 22:13:54 DEBUG { 22:13:54 DEBUG { 22:13:54 DEBUG "status": "success", 22:13:54 DEBUG { 22:13:54 DEBUG "result": [ 22:13:54 DEBUG "metric": { 22:13:54 DEBUG { 22:13:54 DEBUG "result": [ 22:13:54 DEBUG { 22:13:54 DEBUG { 22:13:54 DEBUG { 22:13:54 DEBUG { 22:13:54 DEBUG { 22:13:54 DEBUG { 22:13:54 DEBUG { 22:13:54 DEBUG "metric": { 22:13:54 DEBUG "resultType": "vector", 22:13:54 DEBUG "result": [ 22:13:54 DEBUG "result": [ 22:13:54 DEBUG "result": [ 22:13:54 DEBUG "metric": { 22:13:54 DEBUG "result": [ 22:13:54 DEBUG { 22:13:54 DEBUG { 22:13:54 DEBUG "metric": { 22:13:54 DEBUG "metric": { 22:13:54 DEBUG "metric": { 22:13:54 DEBUG "data": { 22:13:54 DEBUG "metric": { 22:13:54 DEBUG { 22:13:54 DEBUG "instance": "10.142.0.107:9100" 22:13:54 DEBUG "metric": { 22:13:54 DEBUG { 22:13:54 DEBUG "metric": { 22:13:54 DEBUG "metric": { 22:13:54 DEBUG "metric": { 22:13:54 DEBUG "metric": { 22:13:54 DEBUG "metric": { 22:13:54 DEBUG "metric": { 22:13:54 DEBUG "metric": { 22:13:54 DEBUG "metric": { 22:13:54 DEBUG "pod": "ds-idrepo-2" 22:13:54 DEBUG "instance": "10.142.0.107:9100" 22:13:54 DEBUG { 22:13:54 DEBUG { 22:13:54 DEBUG { 22:13:54 DEBUG "pod": "am-555f4db595-fz8sn" 22:13:54 DEBUG { 22:13:54 DEBUG "metric": { 22:13:54 DEBUG "metric": { 22:13:54 DEBUG "pod": "ds-idrepo-2" 22:13:54 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:13:54 DEBUG "pod": "am-555f4db595-fz8sn" 22:13:54 DEBUG "resultType": "vector", 22:13:54 DEBUG "pod": "ds-idrepo-2" 22:13:54 DEBUG "metric": { 22:13:54 DEBUG }, 22:13:54 DEBUG "pod": "am-555f4db595-fz8sn" 22:13:54 INFO [loop_until]: OK (rc = 0) 22:13:54 DEBUG "pod": "ds-idrepo-2" 22:13:54 DEBUG "pod": "am-555f4db595-fz8sn" 22:13:54 DEBUG "instance": "10.142.0.107:9100" 22:13:54 DEBUG "pod": "ds-idrepo-2" 22:13:54 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:13:54 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:13:54 DEBUG "pod": "am-555f4db595-fz8sn" 22:13:54 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:13:54 DEBUG }, 22:13:54 DEBUG }, 22:13:54 DEBUG "metric": { 22:13:54 DEBUG "metric": { 22:13:54 DEBUG "metric": { 22:13:54 DEBUG }, 22:13:54 DEBUG "metric": { 22:13:54 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:13:54 DEBUG "instance": "10.142.0.107:9100" 22:13:54 DEBUG }, 22:13:54 DEBUG }, 22:13:54 DEBUG }, 22:13:54 DEBUG "result": [ 22:13:54 DEBUG }, 22:13:54 DEBUG "pod": "am-555f4db595-fz8sn" 22:13:54 DEBUG "value": [ 22:13:54 DEBUG }, 22:13:54 DEBUG --- stdout --- 22:13:54 DEBUG }, 22:13:54 DEBUG }, 22:13:54 DEBUG }, 22:13:54 DEBUG }, 22:13:54 DEBUG }, 22:13:54 DEBUG }, 22:13:54 DEBUG }, 22:13:54 DEBUG }, 22:13:54 DEBUG "value": [ 22:13:54 DEBUG "value": [ 22:13:54 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:13:54 DEBUG "pod": "am-555f4db595-fz8sn" 22:13:54 DEBUG "pod": "ds-idrepo-2" 22:13:54 DEBUG "value": [ 22:13:54 DEBUG "pod": "ds-idrepo-2" 22:13:54 DEBUG }, 22:13:54 DEBUG }, 22:13:54 DEBUG "value": [ 22:13:54 DEBUG "value": [ 22:13:54 DEBUG "value": [ 22:13:54 DEBUG { 22:13:54 DEBUG "value": [ 22:13:54 DEBUG }, 22:13:54 DEBUG 22:13:54 DEBUG "value": [ 22:13:54 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 62m 0% 5092Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5158m 17% 9480Mi 8% gke-xlou-cdm-default-pool-1b0769c8-nhlf 9096m 30% 6883Mi 6% gke-xlou-cdm-default-pool-1b0769c8-qkns 89m 0% 2654Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 4676m 15% 6206Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 6056m 20% 11651Mi 10% gke-xlou-cdm-default-pool-1b0769c8-xnmp 62m 0% 3149Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 9446m 31% 6486Mi 5% gke-xlou-cdm-ds-0f601751-hmm7 4685m 15% 5559Mi 4% gke-xlou-cdm-ds-0f601751-j488 5081m 17% 8782Mi 7% gke-xlou-cdm-ds-0f601751-mc5d 10648m 35% 3349Mi 3% gke-xlou-cdm-ds-0f601751-nj6q 9659m 32% 6768Mi 6% gke-xlou-cdm-ds-0f601751-nmxm 4684m 15% 7362Mi 6% gke-xlou-cdm-frontend-a8771548-i82n 6474m 40% 5940Mi 10% 22:13:54 DEBUG "value": [ 22:13:54 DEBUG "value": [ 22:13:54 DEBUG "value": [ 22:13:54 DEBUG "value": [ 22:13:54 DEBUG "value": [ 22:13:54 DEBUG "value": [ 22:13:54 DEBUG 22:13:54 DEBUG "value": [ 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG }, 22:13:54 DEBUG }, 22:13:54 DEBUG }, 22:13:54 DEBUG 22:13:54 DEBUG }, 22:13:54 DEBUG "value": [ 22:13:54 DEBUG "value": [ 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG "metric": { 22:13:54 DEBUG 22:13:54 DEBUG "value": [ 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG --- stderr --- 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG "value": [ 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG "value": [ 22:13:54 DEBUG "value": [ 22:13:54 DEBUG "value": [ 22:13:54 DEBUG 22:13:54 DEBUG "value": [ 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG "instance": "10.142.0.107:9100" 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG [...] 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG [...] 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG [...] 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG [...] 22:13:54 DEBUG 22:13:54 DEBUG [...] 22:13:54 DEBUG [...] 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG [...] 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG [...] 22:13:54 DEBUG [...] 22:13:54 DEBUG }, 22:13:54 DEBUG 22:13:54 DEBUG [...] 22:13:54 DEBUG [...] 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG [...] 22:13:54 DEBUG [...] 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG [...] 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG [...] 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG "value": [ 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG [...] 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG [...] 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG [...] 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG [...] 22:13:54 DEBUG [...] 22:13:54 DEBUG [...] 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG [...] 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG [...] 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG "pod": "am-555f4db595-kzxsp" 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG [...] 22:13:54 DEBUG "pod": "am-555f4db595-kzxsp" 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG "instance": "10.142.0.43:9100" 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG "pod": "ds-idrepo-0" 22:13:54 DEBUG 22:13:54 DEBUG }, 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG "pod": "overseer-0-7c9b667974-9mr2s" 22:13:54 DEBUG 22:13:54 DEBUG "pod": "ds-idrepo-0" 22:13:54 DEBUG "instance": "10.142.0.43:9100" 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG }, 22:13:54 DEBUG 22:13:54 DEBUG "pod": "overseer-0-7c9b667974-9mr2s" 22:13:54 DEBUG "pod": "ds-idrepo-0" 22:13:54 DEBUG 22:13:54 DEBUG }, 22:13:54 DEBUG "pod": "am-555f4db595-kzxsp" 22:13:54 DEBUG "pod": "ds-idrepo-0" 22:13:54 DEBUG 22:13:54 DEBUG }, 22:13:54 DEBUG "pod": "am-555f4db595-kzxsp" 22:13:54 DEBUG 22:13:54 DEBUG }, 22:13:54 DEBUG 22:13:54 DEBUG "value": [ 22:13:54 DEBUG "pod": "overseer-0-7c9b667974-9mr2s" 22:13:54 DEBUG 22:13:54 DEBUG }, 22:13:54 DEBUG "pod": "overseer-0-7c9b667974-9mr2s" 22:13:54 DEBUG }, 22:13:54 DEBUG }, 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG 22:13:54 DEBUG "value": [ 22:13:54 DEBUG 22:13:54 DEBUG }, 22:13:54 DEBUG }, 22:13:54 DEBUG [...] 22:13:54 DEBUG "value": [ 22:13:54 DEBUG }, 22:13:54 DEBUG }, 22:13:54 DEBUG "pod": "am-555f4db595-kzxsp" 22:13:54 DEBUG "value": [ 22:13:54 DEBUG }, 22:13:54 DEBUG 22:13:54 DEBUG "value": [ 22:13:54 DEBUG "pod": "ds-idrepo-0" 22:13:54 DEBUG 1713302034, 22:13:54 DEBUG }, 22:13:54 DEBUG "pod": "am-555f4db595-kzxsp" 22:13:54 DEBUG "value": [ 22:13:54 DEBUG }, 22:13:54 DEBUG "value": [ 22:13:54 DEBUG "value": [ 22:13:54 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:13:54 DEBUG "pod": "am-555f4db595-kzxsp" 22:13:54 DEBUG "pod": "ds-idrepo-0" 22:13:54 DEBUG 22:13:54 DEBUG 1713302034, 22:13:54 DEBUG "instance": "10.142.0.43:9100" 22:13:54 DEBUG "value": [ 22:13:54 DEBUG "value": [ 22:13:54 DEBUG 22:13:54 DEBUG 1713302034, 22:13:54 DEBUG "value": [ 22:13:54 DEBUG "value": [ 22:13:54 DEBUG }, 22:13:54 DEBUG 1713302034, 22:13:54 DEBUG "value": [ 22:13:54 DEBUG "instance": "10.142.0.43:9100" 22:13:54 DEBUG 1713302034, 22:13:54 DEBUG }, 22:13:54 DEBUG "0" 22:13:54 DEBUG "value": [ 22:13:54 DEBUG }, 22:13:54 DEBUG 1713302034, 22:13:54 DEBUG "value": [ 22:13:54 DEBUG 1713302034, 22:13:54 DEBUG 1713302034, 22:13:54 DEBUG }, 22:13:54 DEBUG }, 22:13:54 DEBUG }, 22:13:54 DEBUG "pod": "ds-cts-2" 22:13:54 DEBUG "0" 22:13:54 DEBUG }, 22:13:54 DEBUG 1713302034, 22:13:54 DEBUG 1713302034, 22:13:54 DEBUG 22:13:54 DEBUG "6.3339908179588" 22:13:54 DEBUG 1713302034, 22:13:54 DEBUG 1713302034, 22:13:54 DEBUG "value": [ 22:13:54 DEBUG "45308834.3" 22:13:54 DEBUG 1713302034, 22:13:54 DEBUG }, 22:13:54 DEBUG "0" 22:13:54 DEBUG "value": [ 22:13:54 DEBUG ] 22:13:54 DEBUG 1713302034, 22:13:54 DEBUG "value": [ 22:13:54 DEBUG "6.281021688981102" 22:13:54 DEBUG 1713302034, 22:13:54 DEBUG "0" 22:13:54 DEBUG "45299128.3" 22:13:54 DEBUG "value": [ 22:13:54 DEBUG "value": [ 22:13:54 DEBUG "value": [ 22:13:54 DEBUG }, 22:13:54 DEBUG ] 22:13:54 DEBUG "value": [ 22:13:54 DEBUG "0" 22:13:54 DEBUG "0" 22:13:54 DEBUG 22:13:54 DEBUG ] 22:13:54 DEBUG } 22:13:54 DEBUG "0" 22:13:54 DEBUG 1713302034, 22:13:54 DEBUG ] 22:13:54 DEBUG "1995.3999999999999" 22:13:54 DEBUG "value": [ 22:13:54 DEBUG ] 22:13:54 DEBUG 1713302034, 22:13:54 DEBUG } 22:13:54 DEBUG "47760.21125553003" 22:13:54 DEBUG 1713302034, 22:13:54 DEBUG ] 22:13:54 DEBUG "0" 22:13:54 DEBUG ] 22:13:54 DEBUG ] 22:13:54 DEBUG 1713302034, 22:13:54 DEBUG 1713302034, 22:13:54 DEBUG 1713302034, 22:13:54 DEBUG "value": [ 22:13:54 DEBUG } 22:13:54 DEBUG 1713302034, 22:13:54 DEBUG ] 22:13:54 DEBUG ] 22:13:54 DEBUG "instance": "10.142.0.43:9100" 22:13:54 DEBUG "1995.2" 22:13:54 DEBUG ] 22:13:54 DEBUG ] 22:13:54 DEBUG "1995.2" 22:13:54 DEBUG } 22:13:54 DEBUG ] 22:13:54 DEBUG 1713302034, 22:13:54 DEBUG } 22:13:54 DEBUG "0" 22:13:54 DEBUG ] 22:13:54 DEBUG ] 22:13:54 DEBUG "200846.5" 22:13:54 DEBUG } 22:13:54 DEBUG ] 22:13:54 DEBUG } 22:13:54 DEBUG } 22:13:54 DEBUG "626290688" 22:13:54 DEBUG "0.023" 22:13:54 DEBUG "0" 22:13:54 DEBUG 1713302034, 22:13:54 DEBUG ] 22:13:54 DEBUG "0" 22:13:54 DEBUG } 22:13:54 DEBUG } 22:13:54 DEBUG }, 22:13:54 DEBUG ] 22:13:54 DEBUG } 22:13:54 DEBUG } 22:13:54 DEBUG ] 22:13:54 DEBUG ] 22:13:54 DEBUG } 22:13:54 DEBUG "19947164672" 22:13:54 DEBUG ] 22:13:54 DEBUG ] 22:13:54 DEBUG } 22:13:54 DEBUG } 22:13:54 DEBUG ] 22:13:54 DEBUG } 22:13:54 DEBUG } 22:13:54 DEBUG ] 22:13:54 DEBUG ] 22:13:54 DEBUG ] 22:13:54 DEBUG ] 22:13:54 DEBUG ] 22:13:54 DEBUG "0" 22:13:54 DEBUG } 22:13:54 DEBUG ] 22:13:54 DEBUG ] 22:13:54 DEBUG ] 22:13:54 DEBUG "value": [ 22:13:54 DEBUG } 22:13:54 DEBUG } 22:13:54 DEBUG ] 22:13:54 DEBUG } 22:13:54 DEBUG } 22:13:54 DEBUG ] 22:13:54 DEBUG ] 22:13:54 DEBUG } 22:13:54 DEBUG } 22:13:54 DEBUG } 22:13:54 DEBUG ] 22:13:54 DEBUG ] 22:13:54 DEBUG ] 22:13:54 DEBUG ] 22:13:54 DEBUG } 22:13:54 DEBUG } 22:13:54 DEBUG } 22:13:54 DEBUG } 22:13:54 DEBUG } 22:13:54 DEBUG ] 22:13:54 DEBUG } 22:13:54 DEBUG } 22:13:54 DEBUG } 22:13:54 DEBUG } 22:13:54 DEBUG 1713302034, 22:13:54 DEBUG ] 22:13:54 DEBUG } 22:13:54 DEBUG ] 22:13:54 DEBUG } 22:13:54 DEBUG } 22:13:54 DEBUG } 22:13:54 DEBUG } 22:13:54 DEBUG ] 22:13:54 DEBUG } 22:13:54 DEBUG } 22:13:54 DEBUG } 22:13:54 DEBUG } 22:13:54 DEBUG } 22:13:54 DEBUG } 22:13:54 DEBUG ] 22:13:54 DEBUG ] 22:13:54 DEBUG ] 22:13:54 DEBUG } 22:13:54 DEBUG ] 22:13:54 DEBUG } 22:13:54 DEBUG } 22:13:54 DEBUG "0.0003333333333330302" 22:13:54 DEBUG } 22:13:54 DEBUG } 22:13:54 DEBUG } 22:13:54 DEBUG } 22:13:54 DEBUG ] 22:13:54 DEBUG } 22:13:54 DEBUG } 22:13:54 DEBUG } 22:13:54 DEBUG } 22:13:54 DEBUG } 22:13:54 DEBUG } 22:13:54 DEBUG } 22:13:54 DEBUG } 22:13:54 DEBUG ] 22:13:54 DEBUG } 22:13:54 DEBUG ] 22:13:54 DEBUG } 22:13:54 DEBUG } 22:13:54 DEBUG } 22:13:54 DEBUG } 22:13:54 DEBUG } 22:13:54 DEBUG } 22:13:54 DEBUG } 22:13:54 DEBUG } 22:13:54 DEBUG } 22:13:54 DEBUG } 22:13:54 DEBUG } 22:13:54 DEBUG } 22:13:54 DEBUG ] 22:13:54 DEBUG } 22:13:54 DEBUG } 22:14:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713302094" 22:14:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302094" 22:14:54 ERROR [http_cmd]: ERROR 22:14:54 ERROR --- status code --- 22:14:54 ERROR http status code is 400 (expected [200, 401, 403, 404]) 22:14:54 ERROR --- http response --- 22:14:54 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 22:14:54 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 22:14:54 INFO [http_cmd]: http status code OK 22:14:54 DEBUG --- status code --- 22:14:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:14:54 DEBUG --- http response --- 22:14:54 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 22:14:54 INFO 22:14:54 INFO [loop_until]: kubectl --namespace=xlou top pods 22:14:54 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:14:54 INFO 22:14:54 INFO [loop_until]: kubectl --namespace=xlou top node 22:14:54 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:14:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713302094" 22:14:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302094" 22:14:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302094" 22:14:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302094" 22:14:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302094" 22:14:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302094" 22:14:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302094" 22:14:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302094" 22:14:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302094" 22:14:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302094" 22:14:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302094" 22:14:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302094" 22:14:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302094" 22:14:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302094" 22:14:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302094" 22:14:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302094" 22:14:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302094" 22:14:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302094" 22:14:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302094" 22:14:54 INFO [loop_until]: OK (rc = 0) 22:14:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302094" 22:14:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713302094" 22:14:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302094" 22:14:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302094" 22:14:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713302094" 22:14:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713302094" 22:14:54 INFO [http_cmd]: http status code OK 22:14:54 INFO [http_cmd]: http status code OK 22:14:54 INFO [http_cmd]: http status code OK 22:14:54 INFO [http_cmd]: http status code OK 22:14:54 INFO [http_cmd]: http status code OK 22:14:54 INFO [http_cmd]: http status code OK 22:14:54 INFO [http_cmd]: http status code OK 22:14:54 INFO [http_cmd]: http status code OK 22:14:54 INFO [http_cmd]: http status code OK 22:14:54 INFO [http_cmd]: http status code OK 22:14:54 INFO [http_cmd]: http status code OK 22:14:54 INFO [http_cmd]: http status code OK 22:14:54 DEBUG --- stdout --- 22:14:54 INFO [http_cmd]: http status code OK 22:14:54 INFO [http_cmd]: http status code OK 22:14:54 INFO [http_cmd]: http status code OK 22:14:54 INFO [http_cmd]: http status code OK 22:14:54 DEBUG --- status code --- 22:14:54 INFO [http_cmd]: http status code OK 22:14:54 DEBUG --- status code --- 22:14:54 INFO [http_cmd]: http status code OK 22:14:54 INFO [http_cmd]: http status code OK 22:14:54 DEBUG --- status code --- 22:14:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:14:54 DEBUG --- status code --- 22:14:54 DEBUG --- status code --- 22:14:54 DEBUG --- status code --- 22:14:54 DEBUG --- status code --- 22:14:54 DEBUG --- status code --- 22:14:54 DEBUG --- status code --- 22:14:54 DEBUG --- status code --- 22:14:54 DEBUG --- status code --- 22:14:54 DEBUG --- status code --- 22:14:54 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 9Mi am-555f4db595-4lbss 9115m 6980Mi am-555f4db595-fz8sn 9396m 7039Mi am-555f4db595-kzxsp 9356m 7180Mi ds-cts-0 5693m 7222Mi ds-cts-1 5174m 7013Mi ds-cts-2 5202m 6893Mi ds-idrepo-0 4749m 5032Mi ds-idrepo-1 4441m 2833Mi ds-idrepo-2 4642m 2883Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 3m 1208Mi idm-b85f9d978-rl76f 3m 1225Mi lodemon-bc756c647-cr7x6 23m 62Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-7c9b667974-9mr2s 6242m 507Mi 22:14:54 DEBUG --- status code --- 22:14:54 DEBUG --- status code --- 22:14:54 DEBUG --- status code --- 22:14:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:14:54 DEBUG --- status code --- 22:14:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:14:54 DEBUG --- status code --- 22:14:54 INFO [http_cmd]: http status code OK 22:14:54 DEBUG --- status code --- 22:14:54 INFO [http_cmd]: http status code OK 22:14:54 INFO [http_cmd]: http status code OK 22:14:54 DEBUG --- http response --- 22:14:54 DEBUG --- status code --- 22:14:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:14:54 DEBUG --- http response --- 22:14:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:14:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:14:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:14:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:14:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:14:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:14:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:14:54 DEBUG --- stderr --- 22:14:54 INFO [http_cmd]: http status code OK 22:14:54 INFO [http_cmd]: http status code OK 22:14:54 INFO [http_cmd]: http status code OK 22:14:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:14:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:14:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:14:54 DEBUG --- http response --- 22:14:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:14:54 DEBUG --- http response --- 22:14:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:14:54 DEBUG --- status code --- 22:14:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:14:54 DEBUG --- status code --- 22:14:54 DEBUG --- status code --- 22:14:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:14:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:14:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:14:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:14:54 DEBUG --- http response --- 22:14:54 DEBUG --- http response --- 22:14:54 DEBUG --- http response --- 22:14:54 DEBUG --- http response --- 22:14:54 DEBUG --- http response --- 22:14:54 DEBUG --- http response --- 22:14:54 DEBUG --- http response --- 22:14:54 DEBUG 22:14:54 DEBUG --- status code --- 22:14:54 DEBUG --- status code --- 22:14:54 DEBUG --- status code --- 22:14:54 DEBUG --- http response --- 22:14:54 DEBUG --- http response --- 22:14:54 DEBUG --- http response --- 22:14:54 DEBUG --- http response --- 22:14:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:14:54 DEBUG --- http response --- 22:14:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:14:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:14:54 DEBUG --- http response --- 22:14:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:14:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:14:54 DEBUG ----- output ----- 22:14:54 DEBUG --- http response --- 22:14:54 DEBUG --- http response --- 22:14:54 DEBUG ----- output ----- 22:14:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 22:14:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:14:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:14:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:14:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:14:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:14:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:14:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:14:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:14:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:14:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:14:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:14:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:14:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:14:54 DEBUG ----- output ----- 22:14:54 DEBUG ----- output ----- 22:14:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 22:14:54 DEBUG --- http response --- 22:14:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 22:14:54 DEBUG --- http response --- 22:14:54 DEBUG --- http response --- 22:14:54 DEBUG { 22:14:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:14:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 22:14:54 DEBUG { 22:14:54 DEBUG ----- output ----- 22:14:54 DEBUG ----- output ----- 22:14:54 DEBUG ----- output ----- 22:14:54 DEBUG ----- output ----- 22:14:54 DEBUG ----- output ----- 22:14:54 DEBUG ----- output ----- 22:14:54 DEBUG ----- output ----- 22:14:54 DEBUG --- http response --- 22:14:54 DEBUG --- http response --- 22:14:54 DEBUG --- http response --- 22:14:54 DEBUG ----- output ----- 22:14:54 DEBUG ----- output ----- 22:14:54 DEBUG ----- output ----- 22:14:54 DEBUG ----- output ----- 22:14:54 DEBUG { 22:14:54 DEBUG { 22:14:54 DEBUG ----- output ----- 22:14:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:14:54 DEBUG ----- output ----- 22:14:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:14:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:14:54 DEBUG "status": "success", 22:14:54 DEBUG ----- output ----- 22:14:54 DEBUG ----- output ----- 22:14:54 DEBUG "status": "success", 22:14:54 DEBUG { 22:14:54 DEBUG { 22:14:54 DEBUG { 22:14:54 DEBUG { 22:14:54 DEBUG { 22:14:54 DEBUG { 22:14:54 DEBUG { 22:14:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:14:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:14:54 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:14:54 DEBUG { 22:14:54 DEBUG { 22:14:54 DEBUG { 22:14:54 DEBUG { 22:14:54 DEBUG "status": "success", 22:14:54 DEBUG "status": "success", 22:14:54 DEBUG { 22:14:54 DEBUG ----- output ----- 22:14:54 DEBUG { 22:14:54 DEBUG ----- output ----- 22:14:54 DEBUG ----- output ----- 22:14:54 DEBUG "data": { 22:14:54 DEBUG { 22:14:54 DEBUG { 22:14:54 DEBUG "data": { 22:14:54 DEBUG "status": "success", 22:14:54 DEBUG "status": "success", 22:14:54 DEBUG "status": "success", 22:14:54 DEBUG "status": "success", 22:14:54 DEBUG "status": "success", 22:14:54 DEBUG "status": "success", 22:14:54 DEBUG "status": "success", 22:14:54 DEBUG ----- output ----- 22:14:54 DEBUG ----- output ----- 22:14:54 DEBUG ----- output ----- 22:14:54 DEBUG "status": "success", 22:14:54 DEBUG "status": "success", 22:14:54 DEBUG "status": "success", 22:14:54 DEBUG "status": "success", 22:14:54 DEBUG "data": { 22:14:54 DEBUG "data": { 22:14:54 DEBUG "status": "success", 22:14:54 DEBUG { 22:14:54 DEBUG "status": "success", 22:14:54 DEBUG { 22:14:54 DEBUG { 22:14:54 DEBUG "resultType": "vector", 22:14:54 DEBUG "status": "success", 22:14:54 DEBUG "status": "success", 22:14:54 DEBUG "resultType": "vector", 22:14:54 DEBUG "data": { 22:14:54 DEBUG "data": { 22:14:54 DEBUG "data": { 22:14:54 DEBUG "data": { 22:14:54 DEBUG "data": { 22:14:54 DEBUG "data": { 22:14:54 DEBUG "data": { 22:14:54 DEBUG { 22:14:54 DEBUG { 22:14:54 DEBUG { 22:14:54 DEBUG "data": { 22:14:54 DEBUG "data": { 22:14:54 DEBUG "data": { 22:14:54 DEBUG "data": { 22:14:54 DEBUG "resultType": "vector", 22:14:54 DEBUG "resultType": "vector", 22:14:54 DEBUG "data": { 22:14:54 DEBUG "status": "success", 22:14:54 DEBUG "data": { 22:14:54 DEBUG "status": "success", 22:14:54 DEBUG "status": "success", 22:14:54 DEBUG "result": [ 22:14:54 DEBUG "data": { 22:14:54 DEBUG "data": { 22:14:54 DEBUG "result": [ 22:14:54 DEBUG { 22:14:54 DEBUG "resultType": "vector", 22:14:54 DEBUG "resultType": "vector", 22:14:54 DEBUG "resultType": "vector", 22:14:54 DEBUG "resultType": "vector", 22:14:54 DEBUG "resultType": "vector", 22:14:54 DEBUG "resultType": "vector", 22:14:54 DEBUG "status": "success", 22:14:54 DEBUG "status": "success", 22:14:54 DEBUG "status": "success", 22:14:54 DEBUG "resultType": "vector", 22:14:54 DEBUG "resultType": "vector", 22:14:54 DEBUG "result": [ 22:14:54 DEBUG "resultType": "vector", 22:14:54 DEBUG "result": [ 22:14:54 DEBUG "result": [ 22:14:54 DEBUG "resultType": "vector", 22:14:54 DEBUG "data": { 22:14:54 DEBUG "resultType": "vector", 22:14:54 DEBUG "data": { 22:14:54 DEBUG "data": { 22:14:54 DEBUG { 22:14:54 DEBUG "resultType": "vector", 22:14:54 DEBUG "resultType": "vector", 22:14:54 DEBUG "metric": { 22:14:54 DEBUG "resultType": "vector", 22:14:54 DEBUG "result": [ 22:14:54 DEBUG "result": [ 22:14:54 DEBUG "result": [ 22:14:54 DEBUG "result": [ 22:14:54 DEBUG "result": [ 22:14:54 DEBUG "result": [ 22:14:54 DEBUG "data": { 22:14:54 DEBUG "data": { 22:14:54 DEBUG "data": { 22:14:54 DEBUG "result": [ 22:14:54 DEBUG { 22:14:54 DEBUG "resultType": "vector", 22:14:54 DEBUG "result": [ 22:14:54 DEBUG { 22:14:54 DEBUG { 22:14:54 DEBUG "result": [ 22:14:54 DEBUG "resultType": "vector", 22:14:54 DEBUG "result": [ 22:14:54 DEBUG "resultType": "vector", 22:14:54 DEBUG "resultType": "vector", 22:14:54 DEBUG "metric": { 22:14:54 DEBUG "result": [ 22:14:54 DEBUG "result": [ 22:14:54 DEBUG "instance": "10.142.0.107:9100" 22:14:54 DEBUG "result": [ 22:14:55 DEBUG { 22:14:54 DEBUG { 22:14:55 DEBUG { 22:14:55 DEBUG { 22:14:55 DEBUG { 22:14:55 DEBUG { 22:14:55 DEBUG "resultType": "vector", 22:14:55 DEBUG "resultType": "vector", 22:14:55 DEBUG "resultType": "vector", 22:14:55 DEBUG { 22:14:55 DEBUG "metric": { 22:14:55 DEBUG "result": [ 22:14:55 DEBUG { 22:14:55 INFO [loop_until]: OK (rc = 0) 22:14:55 DEBUG "metric": { 22:14:55 DEBUG "metric": { 22:14:55 DEBUG { 22:14:55 DEBUG "result": [ 22:14:55 DEBUG { 22:14:55 DEBUG "result": [ 22:14:55 DEBUG "result": [ 22:14:55 DEBUG "pod": "ds-idrepo-2" 22:14:55 DEBUG { 22:14:55 DEBUG { 22:14:55 DEBUG }, 22:14:54 DEBUG { 22:14:55 DEBUG "metric": { 22:14:55 DEBUG "metric": { 22:14:55 DEBUG "metric": { 22:14:55 DEBUG "metric": { 22:14:55 DEBUG "metric": { 22:14:55 DEBUG "metric": { 22:14:55 DEBUG "result": [ 22:14:55 DEBUG "result": [ 22:14:55 DEBUG "result": [ 22:14:55 DEBUG "metric": { 22:14:55 DEBUG "pod": "am-555f4db595-fz8sn" 22:14:55 DEBUG { 22:14:55 DEBUG "metric": { 22:14:55 DEBUG --- stdout --- 22:14:55 DEBUG "pod": "am-555f4db595-fz8sn" 22:14:55 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:14:55 DEBUG "metric": { 22:14:55 DEBUG { 22:14:55 DEBUG "metric": { 22:14:55 DEBUG { 22:14:55 DEBUG { 22:14:55 DEBUG }, 22:14:55 DEBUG "metric": { 22:14:55 DEBUG "metric": { 22:14:55 DEBUG "value": [ 22:14:55 DEBUG "metric": { 22:14:55 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:14:55 DEBUG "instance": "10.142.0.107:9100" 22:14:55 DEBUG "pod": "ds-idrepo-2" 22:14:55 DEBUG "pod": "am-555f4db595-fz8sn" 22:14:55 DEBUG "pod": "am-555f4db595-fz8sn" 22:14:55 DEBUG "pod": "ds-idrepo-2" 22:14:55 DEBUG { 22:14:55 DEBUG { 22:14:55 DEBUG { 22:14:55 DEBUG "pod": "am-555f4db595-fz8sn" 22:14:55 DEBUG }, 22:14:55 DEBUG "metric": { 22:14:55 DEBUG "pod": "ds-idrepo-2" 22:14:55 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 62m 0% 5082Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5299m 17% 9520Mi 8% gke-xlou-cdm-default-pool-1b0769c8-nhlf 9295m 31% 9233Mi 8% gke-xlou-cdm-default-pool-1b0769c8-qkns 89m 0% 2651Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 4785m 16% 6211Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 6233m 20% 11662Mi 10% gke-xlou-cdm-default-pool-1b0769c8-xnmp 64m 0% 3147Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 9273m 31% 8754Mi 7% gke-xlou-cdm-ds-0f601751-hmm7 4659m 15% 5565Mi 4% gke-xlou-cdm-ds-0f601751-j488 5274m 17% 8795Mi 7% gke-xlou-cdm-ds-0f601751-mc5d 10694m 35% 3338Mi 2% gke-xlou-cdm-ds-0f601751-nj6q 9521m 31% 9039Mi 8% gke-xlou-cdm-ds-0f601751-nmxm 4681m 15% 7352Mi 6% gke-xlou-cdm-frontend-a8771548-i82n 6473m 40% 5949Mi 10% 22:14:55 DEBUG }, 22:14:55 DEBUG }, 22:14:55 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:14:55 DEBUG "metric": { 22:14:55 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:14:55 DEBUG "metric": { 22:14:55 DEBUG "metric": { 22:14:55 DEBUG "value": [ 22:14:55 DEBUG "pod": "ds-idrepo-2" 22:14:55 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:14:55 DEBUG 22:14:55 DEBUG "pod": "ds-idrepo-2" 22:14:55 DEBUG }, 22:14:55 DEBUG }, 22:14:55 DEBUG }, 22:14:55 DEBUG }, 22:14:55 DEBUG }, 22:14:55 DEBUG }, 22:14:55 DEBUG "metric": { 22:14:55 DEBUG "metric": { 22:14:55 DEBUG "metric": { 22:14:55 DEBUG }, 22:14:55 DEBUG "value": [ 22:14:55 DEBUG "pod": "am-555f4db595-fz8sn" 22:14:55 DEBUG }, 22:14:55 DEBUG --- stderr --- 22:14:55 DEBUG "value": [ 22:14:55 DEBUG "value": [ 22:14:55 DEBUG }, 22:14:55 DEBUG "instance": "10.142.0.107:9100" 22:14:55 DEBUG }, 22:14:55 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:14:55 DEBUG "instance": "10.142.0.107:9100" 22:14:55 DEBUG 22:14:55 DEBUG }, 22:14:55 DEBUG }, 22:14:55 DEBUG 22:14:55 DEBUG }, 22:14:55 DEBUG "value": [ 22:14:55 DEBUG "value": [ 22:14:55 DEBUG "value": [ 22:14:55 DEBUG "value": [ 22:14:55 DEBUG "value": [ 22:14:55 DEBUG "value": [ 22:14:55 DEBUG "pod": "am-555f4db595-fz8sn" 22:14:55 DEBUG "pod": "ds-idrepo-2" 22:14:55 DEBUG "instance": "10.142.0.107:9100" 22:14:55 DEBUG "value": [ 22:14:55 DEBUG 22:14:55 DEBUG }, 22:14:55 DEBUG "value": [ 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG "value": [ 22:14:55 DEBUG }, 22:14:55 DEBUG "value": [ 22:14:55 DEBUG }, 22:14:55 DEBUG }, 22:14:55 DEBUG 22:14:55 DEBUG "value": [ 22:14:55 DEBUG "value": [ 22:14:55 DEBUG 22:14:55 DEBUG "value": [ 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG }, 22:14:55 DEBUG }, 22:14:55 DEBUG }, 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG "value": [ 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG "value": [ 22:14:55 DEBUG 22:14:55 DEBUG "value": [ 22:14:55 DEBUG "value": [ 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG [...] 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG "value": [ 22:14:55 DEBUG "value": [ 22:14:55 DEBUG "value": [ 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG [...] 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG [...] 22:14:55 DEBUG [...] 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG [...] 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG [...] 22:14:55 DEBUG [...] 22:14:55 DEBUG [...] 22:14:55 DEBUG [...] 22:14:55 DEBUG [...] 22:14:55 DEBUG [...] 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG [...] 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG [...] 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG [...] 22:14:55 DEBUG 22:14:55 DEBUG [...] 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG [...] 22:14:55 DEBUG [...] 22:14:55 DEBUG [...] 22:14:55 DEBUG 22:14:55 DEBUG [...] 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG [...] 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG [...] 22:14:55 DEBUG 22:14:55 DEBUG [...] 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG "instance": "10.142.0.43:9100" 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG [...] 22:14:55 DEBUG [...] 22:14:55 DEBUG [...] 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG }, 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG "instance": "10.142.0.43:9100" 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG "pod": "am-555f4db595-kzxsp" 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG "pod": "am-555f4db595-kzxsp" 22:14:55 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG "pod": "ds-idrepo-0" 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG "pod": "ds-idrepo-0" 22:14:55 DEBUG "value": [ 22:14:55 DEBUG 22:14:55 DEBUG "pod": "overseer-0-7c9b667974-9mr2s" 22:14:55 DEBUG 22:14:55 DEBUG }, 22:14:55 DEBUG "pod": "am-555f4db595-kzxsp" 22:14:55 DEBUG "pod": "am-555f4db595-kzxsp" 22:14:55 DEBUG "pod": "ds-idrepo-0" 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG "pod": "am-555f4db595-kzxsp" 22:14:55 DEBUG }, 22:14:55 DEBUG 22:14:55 DEBUG "pod": "ds-idrepo-0" 22:14:55 DEBUG }, 22:14:55 DEBUG }, 22:14:55 DEBUG "pod": "ds-idrepo-1" 22:14:55 DEBUG 22:14:55 DEBUG "pod": "ds-idrepo-1" 22:14:55 DEBUG 22:14:55 DEBUG }, 22:14:55 DEBUG "instance": "10.142.0.43:9100" 22:14:55 DEBUG 22:14:55 DEBUG }, 22:14:55 DEBUG 1713302094, 22:14:55 DEBUG "pod": "ds-idrepo-0" 22:14:55 DEBUG }, 22:14:55 DEBUG "pod": "ds-idrepo-0" 22:14:55 DEBUG "value": [ 22:14:55 DEBUG }, 22:14:55 DEBUG }, 22:14:55 DEBUG }, 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG 22:14:55 DEBUG }, 22:14:55 DEBUG "value": [ 22:14:55 DEBUG "pod": "am-555f4db595-kzxsp" 22:14:55 DEBUG }, 22:14:55 DEBUG "value": [ 22:14:55 DEBUG "value": [ 22:14:55 DEBUG }, 22:14:55 DEBUG "instance": "10.142.0.43:9100" 22:14:55 DEBUG }, 22:14:55 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:14:55 DEBUG "value": [ 22:14:55 DEBUG }, 22:14:55 DEBUG "pod": "overseer-0-7c9b667974-9mr2s" 22:14:55 DEBUG "value": [ 22:14:55 DEBUG "45484828.633333325" 22:14:55 DEBUG }, 22:14:55 DEBUG "value": [ 22:14:55 DEBUG }, 22:14:55 DEBUG 1713302094, 22:14:55 DEBUG "value": [ 22:14:55 DEBUG "value": [ 22:14:55 DEBUG "value": [ 22:14:55 DEBUG "pod": "am-555f4db595-kzxsp" 22:14:55 DEBUG "pod": "ds-cts-2" 22:14:55 DEBUG "instance": "10.142.0.43:9100" 22:14:55 DEBUG "value": [ 22:14:55 DEBUG 1713302094, 22:14:55 DEBUG }, 22:14:55 DEBUG "value": [ 22:14:55 DEBUG 1713302094, 22:14:55 DEBUG 1713302094, 22:14:55 DEBUG "value": [ 22:14:55 DEBUG }, 22:14:55 DEBUG "value": [ 22:14:55 DEBUG }, 22:14:55 DEBUG 1713302094, 22:14:55 DEBUG "value": [ 22:14:55 DEBUG }, 22:14:55 DEBUG 1713302094, 22:14:55 DEBUG ] 22:14:55 DEBUG "value": [ 22:14:55 DEBUG 1713302094, 22:14:55 DEBUG "value": [ 22:14:55 DEBUG "45494749.699999996" 22:14:55 DEBUG 1713302094, 22:14:55 DEBUG 1713302094, 22:14:55 DEBUG 1713302094, 22:14:55 DEBUG }, 22:14:55 DEBUG }, 22:14:55 DEBUG }, 22:14:55 DEBUG 1713302094, 22:14:55 DEBUG "1998.6" 22:14:55 DEBUG "value": [ 22:14:55 DEBUG 1713302094, 22:14:55 DEBUG "0" 22:14:55 DEBUG "6.333178497304753" 22:14:55 DEBUG 1713302094, 22:14:55 DEBUG "value": [ 22:14:55 DEBUG 1713302094, 22:14:55 DEBUG "value": [ 22:14:55 DEBUG "0" 22:14:55 DEBUG 1713302094, 22:14:55 DEBUG "value": [ 22:14:55 DEBUG "0" 22:14:55 DEBUG } 22:14:55 DEBUG 1713302094, 22:14:55 DEBUG "0" 22:14:55 DEBUG 1713302094, 22:14:55 DEBUG ] 22:14:55 DEBUG "0" 22:14:55 DEBUG "1998.5666666666666" 22:14:55 DEBUG "0" 22:14:55 DEBUG "value": [ 22:14:55 DEBUG "value": [ 22:14:55 DEBUG "value": [ 22:14:55 DEBUG "320794.5" 22:14:55 DEBUG ] 22:14:55 DEBUG 1713302094, 22:14:55 DEBUG "0" 22:14:55 DEBUG ] 22:14:55 DEBUG ] 22:14:55 DEBUG "0" 22:14:55 DEBUG 1713302094, 22:14:55 DEBUG "0" 22:14:55 DEBUG 1713302094, 22:14:55 DEBUG ] 22:14:55 DEBUG "0" 22:14:55 DEBUG 1713302094, 22:14:55 DEBUG ] 22:14:55 DEBUG ] 22:14:55 DEBUG "0" 22:14:55 DEBUG ] 22:14:55 DEBUG "0" 22:14:55 DEBUG } 22:14:55 DEBUG ] 22:14:55 DEBUG ] 22:14:55 DEBUG ] 22:14:55 DEBUG 1713302094, 22:14:55 DEBUG 1713302094, 22:14:55 DEBUG 1713302094, 22:14:55 DEBUG ] 22:14:55 DEBUG } 22:14:55 DEBUG "1998.6333333333332" 22:14:55 DEBUG ] 22:14:55 DEBUG } 22:14:55 DEBUG } 22:14:55 DEBUG ] 22:14:55 DEBUG "0.00040000000000001514" 22:14:55 DEBUG ] 22:14:55 DEBUG "629628928" 22:14:55 DEBUG } 22:14:55 DEBUG ] 22:14:55 DEBUG "0" 22:14:55 DEBUG } 22:14:55 DEBUG } 22:14:55 DEBUG ] 22:14:55 DEBUG } 22:14:55 DEBUG ] 22:14:55 DEBUG ] 22:14:55 DEBUG } 22:14:55 DEBUG } 22:14:55 DEBUG } 22:14:55 DEBUG "0.023" 22:14:55 DEBUG "0" 22:14:55 DEBUG "19947648000" 22:14:55 DEBUG } 22:14:55 DEBUG ] 22:14:55 DEBUG ] 22:14:55 DEBUG } 22:14:55 DEBUG ] 22:14:55 DEBUG ] 22:14:55 DEBUG } 22:14:55 DEBUG ] 22:14:55 DEBUG } 22:14:55 DEBUG ] 22:14:55 DEBUG ] 22:14:55 DEBUG } 22:14:55 DEBUG ] 22:14:55 DEBUG ] 22:14:55 DEBUG } 22:14:55 DEBUG } 22:14:55 DEBUG ] 22:14:55 DEBUG } 22:14:55 DEBUG } 22:14:55 DEBUG ] 22:14:55 DEBUG ] 22:14:55 DEBUG ] 22:14:55 DEBUG ] 22:14:55 DEBUG ] 22:14:55 DEBUG ] 22:14:55 DEBUG ] 22:14:55 DEBUG } 22:14:55 DEBUG } 22:14:55 DEBUG ] 22:14:55 DEBUG } 22:14:55 DEBUG } 22:14:55 DEBUG ] 22:14:55 DEBUG } 22:14:55 DEBUG ] 22:14:55 DEBUG } 22:14:55 DEBUG } 22:14:55 DEBUG ] 22:14:55 DEBUG } 22:14:55 DEBUG } 22:14:55 DEBUG ] 22:14:55 DEBUG } 22:14:55 DEBUG ] 22:14:55 DEBUG } 22:14:55 DEBUG } 22:14:55 DEBUG } 22:14:55 DEBUG } 22:14:55 DEBUG } 22:14:55 DEBUG } 22:14:55 DEBUG } 22:14:55 DEBUG } 22:14:55 DEBUG } 22:14:55 DEBUG ] 22:14:55 DEBUG } 22:14:55 DEBUG } 22:14:55 DEBUG } 22:14:55 DEBUG } 22:14:55 DEBUG ] 22:14:55 DEBUG } 22:14:55 DEBUG ] 22:14:55 DEBUG } 22:14:55 DEBUG } 22:14:55 DEBUG ] 22:14:55 DEBUG } 22:14:55 DEBUG } 22:14:55 DEBUG } 22:14:55 DEBUG } 22:14:55 DEBUG } 22:14:55 DEBUG } 22:14:55 DEBUG } 22:14:55 DEBUG ] 22:14:55 DEBUG ] 22:14:55 DEBUG ] 22:14:55 DEBUG } 22:14:55 DEBUG } 22:14:55 DEBUG } 22:14:55 DEBUG } 22:14:55 DEBUG } 22:14:55 DEBUG } 22:14:55 DEBUG } 22:14:55 DEBUG } 22:14:55 DEBUG } 22:14:55 DEBUG } 22:14:55 DEBUG } 22:14:55 DEBUG } 22:14:55 DEBUG } 22:14:55 DEBUG } 22:14:55 DEBUG } 22:14:55 DEBUG } 22:14:55 DEBUG } 22:14:55 DEBUG } 22:14:55 DEBUG } 22:14:55 DEBUG } 22:14:55 DEBUG } 22:15:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713302154" 22:15:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302154" 22:15:54 INFO [http_cmd]: http status code OK 22:15:54 DEBUG --- status code --- 22:15:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:15:54 DEBUG --- http response --- 22:15:54 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 22:15:54 ERROR [http_cmd]: ERROR 22:15:54 ERROR --- status code --- 22:15:54 ERROR http status code is 400 (expected [200, 401, 403, 404]) 22:15:54 ERROR --- http response --- 22:15:54 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 22:15:54 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 22:15:55 INFO 22:15:55 INFO [loop_until]: kubectl --namespace=xlou top pods 22:15:55 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:15:55 INFO 22:15:55 INFO [loop_until]: kubectl --namespace=xlou top node 22:15:55 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:15:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302155" 22:15:55 INFO [loop_until]: OK (rc = 0) 22:15:55 DEBUG --- stdout --- 22:15:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302155" 22:15:55 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 9Mi am-555f4db595-4lbss 9136m 6993Mi am-555f4db595-fz8sn 9068m 7050Mi am-555f4db595-kzxsp 9446m 7216Mi ds-cts-0 5946m 7729Mi ds-cts-1 5578m 7709Mi ds-cts-2 5409m 7414Mi ds-idrepo-0 4642m 5032Mi ds-idrepo-1 4476m 2833Mi ds-idrepo-2 4550m 2972Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 3m 1208Mi idm-b85f9d978-rl76f 3m 1225Mi lodemon-bc756c647-cr7x6 14m 62Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-7c9b667974-9mr2s 6199m 510Mi 22:15:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713302155" 22:15:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302155" 22:15:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302155" 22:15:55 DEBUG --- stderr --- 22:15:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302155" 22:15:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302155" 22:15:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302155" 22:15:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302155" 22:15:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302155" 22:15:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302155" 22:15:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302155" 22:15:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302155" 22:15:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302155" 22:15:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302155" 22:15:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302155" 22:15:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302155" 22:15:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302155" 22:15:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302155" 22:15:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713302155" 22:15:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302155" 22:15:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302155" 22:15:55 DEBUG 22:15:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713302155" 22:15:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302155" 22:15:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713302155" 22:15:55 INFO [http_cmd]: http status code OK 22:15:55 INFO [http_cmd]: http status code OK 22:15:55 INFO [http_cmd]: http status code OK 22:15:55 INFO [http_cmd]: http status code OK 22:15:55 INFO [http_cmd]: http status code OK 22:15:55 INFO [http_cmd]: http status code OK 22:15:55 INFO [http_cmd]: http status code OK 22:15:55 INFO [http_cmd]: http status code OK 22:15:55 INFO [http_cmd]: http status code OK 22:15:55 INFO [http_cmd]: http status code OK 22:15:55 INFO [http_cmd]: http status code OK 22:15:55 INFO [http_cmd]: http status code OK 22:15:55 INFO [http_cmd]: http status code OK 22:15:55 INFO [http_cmd]: http status code OK 22:15:55 INFO [http_cmd]: http status code OK 22:15:55 DEBUG --- status code --- 22:15:55 INFO [http_cmd]: http status code OK 22:15:55 INFO [http_cmd]: http status code OK 22:15:55 INFO [http_cmd]: http status code OK 22:15:55 DEBUG --- status code --- 22:15:55 INFO [http_cmd]: http status code OK 22:15:55 DEBUG --- status code --- 22:15:55 DEBUG --- status code --- 22:15:55 DEBUG --- status code --- 22:15:55 DEBUG --- status code --- 22:15:55 DEBUG --- status code --- 22:15:55 DEBUG --- status code --- 22:15:55 DEBUG --- status code --- 22:15:55 DEBUG --- status code --- 22:15:55 INFO [http_cmd]: http status code OK 22:15:55 DEBUG --- status code --- 22:15:55 INFO [http_cmd]: http status code OK 22:15:55 DEBUG --- status code --- 22:15:55 INFO [http_cmd]: http status code OK 22:15:55 DEBUG --- status code --- 22:15:55 DEBUG --- status code --- 22:15:55 DEBUG --- status code --- 22:15:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:15:55 DEBUG --- status code --- 22:15:55 DEBUG --- status code --- 22:15:55 INFO [http_cmd]: http status code OK 22:15:55 INFO [http_cmd]: http status code OK 22:15:55 DEBUG --- status code --- 22:15:55 INFO [http_cmd]: http status code OK 22:15:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:15:55 DEBUG --- status code --- 22:15:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:15:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:15:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:15:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:15:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:15:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:15:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:15:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:15:55 DEBUG --- status code --- 22:15:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:15:55 DEBUG --- status code --- 22:15:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:15:55 DEBUG --- http response --- 22:15:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:15:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:15:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:15:55 DEBUG --- http response --- 22:15:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:15:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:15:55 DEBUG --- http response --- 22:15:55 DEBUG --- status code --- 22:15:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:15:55 DEBUG --- status code --- 22:15:55 DEBUG --- http response --- 22:15:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:15:55 DEBUG --- http response --- 22:15:55 DEBUG --- http response --- 22:15:55 DEBUG --- http response --- 22:15:55 DEBUG --- http response --- 22:15:55 DEBUG --- http response --- 22:15:55 DEBUG --- http response --- 22:15:55 DEBUG --- http response --- 22:15:55 DEBUG --- http response --- 22:15:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:15:55 DEBUG --- http response --- 22:15:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:15:55 DEBUG --- status code --- 22:15:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 22:15:55 DEBUG --- http response --- 22:15:55 DEBUG --- http response --- 22:15:55 DEBUG --- http response --- 22:15:55 DEBUG --- http response --- 22:15:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:15:55 DEBUG --- status code --- 22:15:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:15:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:15:55 DEBUG --- http response --- 22:15:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:15:55 DEBUG --- http response --- 22:15:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:15:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:15:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:15:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:15:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 22:15:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:15:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 22:15:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:15:55 DEBUG --- http response --- 22:15:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:15:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:15:55 DEBUG --- http response --- 22:15:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:15:55 DEBUG ----- output ----- 22:15:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:15:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:15:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:15:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:15:55 DEBUG ----- output ----- 22:15:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:15:55 DEBUG ----- output ----- 22:15:55 DEBUG --- http response --- 22:15:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:15:55 DEBUG --- http response --- 22:15:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:15:55 DEBUG ----- output ----- 22:15:55 DEBUG ----- output ----- 22:15:55 DEBUG ----- output ----- 22:15:55 DEBUG ----- output ----- 22:15:55 DEBUG ----- output ----- 22:15:55 DEBUG ----- output ----- 22:15:55 DEBUG ----- output ----- 22:15:55 DEBUG ----- output ----- 22:15:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:15:55 DEBUG ----- output ----- 22:15:55 DEBUG ----- output ----- 22:15:55 DEBUG --- http response --- 22:15:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 22:15:55 DEBUG { 22:15:55 DEBUG ----- output ----- 22:15:55 DEBUG ----- output ----- 22:15:55 DEBUG ----- output ----- 22:15:55 DEBUG ----- output ----- 22:15:55 DEBUG { 22:15:55 DEBUG --- http response --- 22:15:55 DEBUG { 22:15:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:15:55 DEBUG ----- output ----- 22:15:55 DEBUG ----- output ----- 22:15:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:15:55 DEBUG { 22:15:55 DEBUG { 22:15:55 DEBUG { 22:15:55 DEBUG { 22:15:55 DEBUG { 22:15:55 DEBUG { 22:15:55 DEBUG { 22:15:55 DEBUG { 22:15:55 DEBUG ----- output ----- 22:15:55 DEBUG { 22:15:55 DEBUG { 22:15:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:15:55 DEBUG ----- output ----- 22:15:55 DEBUG "status": "success", 22:15:55 DEBUG { 22:15:55 DEBUG { 22:15:55 DEBUG { 22:15:55 DEBUG { 22:15:55 DEBUG "status": "success", 22:15:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:15:55 DEBUG "status": "success", 22:15:55 DEBUG ----- output ----- 22:15:55 DEBUG { 22:15:55 DEBUG { 22:15:55 DEBUG ----- output ----- 22:15:55 DEBUG "status": "success", 22:15:55 DEBUG "status": "success", 22:15:55 DEBUG "status": "success", 22:15:55 DEBUG "status": "success", 22:15:55 DEBUG "status": "success", 22:15:55 DEBUG "status": "success", 22:15:55 INFO [loop_until]: OK (rc = 0) 22:15:55 DEBUG "status": "success", 22:15:55 DEBUG "status": "success", 22:15:55 DEBUG { 22:15:55 DEBUG "status": "success", 22:15:55 DEBUG "status": "success", 22:15:55 DEBUG ----- output ----- 22:15:55 DEBUG { 22:15:55 DEBUG "data": { 22:15:55 DEBUG "status": "success", 22:15:55 DEBUG "status": "success", 22:15:55 DEBUG "status": "success", 22:15:55 DEBUG "status": "success", 22:15:55 DEBUG "data": { 22:15:55 DEBUG ----- output ----- 22:15:55 DEBUG "data": { 22:15:55 DEBUG { 22:15:55 DEBUG "status": "success", 22:15:55 DEBUG "status": "success", 22:15:55 DEBUG { 22:15:55 DEBUG "data": { 22:15:55 DEBUG "data": { 22:15:55 DEBUG "data": { 22:15:55 DEBUG "data": { 22:15:55 DEBUG "data": { 22:15:55 DEBUG "data": { 22:15:55 DEBUG --- stdout --- 22:15:55 DEBUG "data": { 22:15:55 DEBUG "data": { 22:15:55 DEBUG "status": "success", 22:15:55 DEBUG "data": { 22:15:55 DEBUG "data": { 22:15:55 DEBUG { 22:15:55 DEBUG "status": "success", 22:15:55 DEBUG "resultType": "vector", 22:15:55 DEBUG "data": { 22:15:55 DEBUG "data": { 22:15:55 DEBUG "data": { 22:15:55 DEBUG "data": { 22:15:55 DEBUG "resultType": "vector", 22:15:55 DEBUG { 22:15:55 DEBUG "resultType": "vector", 22:15:55 DEBUG "status": "success", 22:15:55 DEBUG "data": { 22:15:55 DEBUG "data": { 22:15:55 DEBUG "status": "success", 22:15:55 DEBUG "resultType": "vector", 22:15:55 DEBUG "resultType": "vector", 22:15:55 DEBUG "resultType": "vector", 22:15:55 DEBUG "resultType": "vector", 22:15:55 DEBUG "resultType": "vector", 22:15:55 DEBUG "resultType": "vector", 22:15:55 DEBUG "resultType": "vector", 22:15:55 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 61m 0% 5094Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5525m 18% 10077Mi 9% gke-xlou-cdm-default-pool-1b0769c8-nhlf 9223m 30% 9272Mi 8% gke-xlou-cdm-default-pool-1b0769c8-qkns 86m 0% 2655Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 4743m 15% 6304Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 6511m 21% 12214Mi 10% gke-xlou-cdm-default-pool-1b0769c8-xnmp 68m 0% 3155Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 9296m 31% 8777Mi 7% gke-xlou-cdm-ds-0f601751-hmm7 4684m 15% 5567Mi 4% gke-xlou-cdm-ds-0f601751-j488 5488m 18% 9448Mi 8% gke-xlou-cdm-ds-0f601751-mc5d 10646m 35% 3343Mi 2% gke-xlou-cdm-ds-0f601751-nj6q 9483m 31% 9073Mi 8% gke-xlou-cdm-ds-0f601751-nmxm 4688m 15% 7353Mi 6% gke-xlou-cdm-frontend-a8771548-i82n 6436m 40% 5948Mi 10% 22:15:55 DEBUG "resultType": "vector", 22:15:55 DEBUG "data": { 22:15:55 DEBUG "resultType": "vector", 22:15:55 DEBUG "resultType": "vector", 22:15:55 DEBUG "status": "success", 22:15:55 DEBUG "data": { 22:15:55 DEBUG "result": [ 22:15:55 DEBUG "resultType": "vector", 22:15:55 DEBUG "resultType": "vector", 22:15:55 DEBUG "resultType": "vector", 22:15:55 DEBUG "resultType": "vector", 22:15:55 DEBUG "result": [ 22:15:55 DEBUG "status": "success", 22:15:55 DEBUG "result": [ 22:15:55 DEBUG "data": { 22:15:55 DEBUG "resultType": "vector", 22:15:55 DEBUG "resultType": "vector", 22:15:55 DEBUG "data": { 22:15:55 DEBUG "result": [ 22:15:55 DEBUG "result": [ 22:15:55 DEBUG "result": [ 22:15:55 DEBUG "result": [ 22:15:55 DEBUG "result": [ 22:15:55 DEBUG "result": [ 22:15:55 DEBUG "result": [ 22:15:55 DEBUG --- stderr --- 22:15:55 DEBUG "result": [ 22:15:55 DEBUG "resultType": "vector", 22:15:55 DEBUG "result": [ 22:15:55 DEBUG "result": [ 22:15:55 DEBUG "data": { 22:15:55 DEBUG "resultType": "vector", 22:15:55 DEBUG { 22:15:55 DEBUG "result": [ 22:15:55 DEBUG "result": [ 22:15:55 DEBUG "result": [ 22:15:55 DEBUG "result": [ 22:15:55 DEBUG { 22:15:55 DEBUG "data": { 22:15:55 DEBUG { 22:15:55 DEBUG "resultType": "vector", 22:15:55 DEBUG "result": [ 22:15:55 DEBUG "result": [ 22:15:55 DEBUG "resultType": "vector", 22:15:55 DEBUG { 22:15:55 DEBUG { 22:15:55 DEBUG { 22:15:55 DEBUG { 22:15:55 DEBUG { 22:15:55 DEBUG { 22:15:55 DEBUG { 22:15:55 DEBUG 22:15:55 DEBUG { 22:15:55 DEBUG "result": [ 22:15:55 DEBUG { 22:15:55 DEBUG { 22:15:55 DEBUG "resultType": "vector", 22:15:55 DEBUG "result": [ 22:15:55 DEBUG "metric": { 22:15:55 DEBUG { 22:15:55 DEBUG { 22:15:55 DEBUG { 22:15:55 DEBUG { 22:15:55 DEBUG "metric": { 22:15:55 DEBUG "resultType": "vector", 22:15:55 DEBUG "metric": { 22:15:55 DEBUG "result": [ 22:15:55 DEBUG { 22:15:55 DEBUG { 22:15:55 DEBUG "result": [ 22:15:55 DEBUG "metric": { 22:15:55 DEBUG "metric": { 22:15:55 DEBUG "metric": { 22:15:55 DEBUG "metric": { 22:15:55 DEBUG "metric": { 22:15:55 DEBUG "metric": { 22:15:55 DEBUG "metric": { 22:15:55 DEBUG "metric": { 22:15:55 DEBUG { 22:15:55 DEBUG "metric": { 22:15:55 DEBUG "metric": { 22:15:55 DEBUG "result": [ 22:15:55 DEBUG { 22:15:55 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:15:55 DEBUG "metric": { 22:15:55 DEBUG "metric": { 22:15:55 DEBUG "metric": { 22:15:55 DEBUG "metric": { 22:15:55 DEBUG "instance": "10.142.0.107:9100" 22:15:55 DEBUG "result": [ 22:15:55 DEBUG "pod": "ds-idrepo-2" 22:15:55 DEBUG { 22:15:55 DEBUG "metric": { 22:15:55 DEBUG "metric": { 22:15:55 DEBUG { 22:15:55 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:15:55 DEBUG "pod": "am-555f4db595-fz8sn" 22:15:55 DEBUG "instance": "10.142.0.107:9100" 22:15:55 DEBUG "pod": "ds-idrepo-2" 22:15:55 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:15:55 DEBUG "pod": "am-555f4db595-fz8sn" 22:15:55 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:15:55 DEBUG "pod": "ds-idrepo-2" 22:15:55 DEBUG "metric": { 22:15:55 DEBUG "pod": "ds-idrepo-2" 22:15:55 DEBUG "pod": "ds-idrepo-2" 22:15:55 DEBUG { 22:15:55 DEBUG "metric": { 22:15:55 DEBUG }, 22:15:55 DEBUG "pod": "am-555f4db595-fz8sn" 22:15:55 DEBUG "pod": "am-555f4db595-fz8sn" 22:15:55 DEBUG "pod": "am-555f4db595-fz8sn" 22:15:55 DEBUG "instance": "10.142.0.107:9100" 22:15:55 DEBUG }, 22:15:55 DEBUG { 22:15:55 DEBUG }, 22:15:55 DEBUG "metric": { 22:15:55 DEBUG "pod": "ds-idrepo-2" 22:15:55 DEBUG "pod": "am-555f4db595-fz8sn" 22:15:55 DEBUG "metric": { 22:15:55 DEBUG }, 22:15:55 DEBUG }, 22:15:55 DEBUG }, 22:15:55 DEBUG }, 22:15:55 DEBUG }, 22:15:55 DEBUG }, 22:15:55 DEBUG }, 22:15:55 DEBUG }, 22:15:55 DEBUG "instance": "10.142.0.107:9100" 22:15:55 DEBUG }, 22:15:55 DEBUG }, 22:15:55 DEBUG "metric": { 22:15:55 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:15:55 DEBUG "value": [ 22:15:55 DEBUG }, 22:15:55 DEBUG }, 22:15:55 DEBUG }, 22:15:55 DEBUG }, 22:15:55 DEBUG "value": [ 22:15:55 DEBUG "metric": { 22:15:55 DEBUG "value": [ 22:15:55 DEBUG "pod": "ds-idrepo-2" 22:15:55 DEBUG }, 22:15:55 DEBUG }, 22:15:55 DEBUG "instance": "10.142.0.107:9100" 22:15:55 DEBUG "value": [ 22:15:55 DEBUG "value": [ 22:15:55 DEBUG "value": [ 22:15:55 DEBUG "value": [ 22:15:55 DEBUG "value": [ 22:15:55 DEBUG "value": [ 22:15:55 DEBUG "value": [ 22:15:55 DEBUG "value": [ 22:15:55 DEBUG 22:15:55 DEBUG "value": [ 22:15:55 DEBUG "value": [ 22:15:55 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:15:55 DEBUG }, 22:15:55 DEBUG 22:15:55 DEBUG "value": [ 22:15:55 DEBUG "value": [ 22:15:55 DEBUG "value": [ 22:15:55 DEBUG "value": [ 22:15:55 DEBUG 22:15:55 DEBUG "pod": "am-555f4db595-fz8sn" 22:15:55 DEBUG 22:15:55 DEBUG }, 22:15:55 DEBUG "value": [ 22:15:55 DEBUG "value": [ 22:15:55 DEBUG }, 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG }, 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG }, 22:15:55 DEBUG "value": [ 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG }, 22:15:55 DEBUG 22:15:55 DEBUG "value": [ 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG "value": [ 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG "value": [ 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG "value": [ 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG "value": [ 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG [...] 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG [...] 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG [...] 22:15:55 DEBUG [...] 22:15:55 DEBUG 22:15:55 DEBUG [...] 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG [...] 22:15:55 DEBUG [...] 22:15:55 DEBUG [...] 22:15:55 DEBUG [...] 22:15:55 DEBUG [...] 22:15:55 DEBUG [...] 22:15:55 DEBUG [...] 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG [...] 22:15:55 DEBUG [...] 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG [...] 22:15:55 DEBUG [...] 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG [...] 22:15:55 DEBUG [...] 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG [...] 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG [...] 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG [...] 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG [...] 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG [...] 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG [...] 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG [...] 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG "instance": "10.142.0.43:9100" 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG "pod": "ds-idrepo-0" 22:15:55 DEBUG }, 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG "pod": "idm-b85f9d978-j5pbm" 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG "pod": "am-555f4db595-kzxsp" 22:15:55 DEBUG "instance": "10.142.0.43:9100" 22:15:55 DEBUG 22:15:55 DEBUG "pod": "ds-idrepo-0" 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:15:55 DEBUG "pod": "am-555f4db595-kzxsp" 22:15:55 DEBUG 22:15:55 DEBUG }, 22:15:55 DEBUG "pod": "idm-b85f9d978-j5pbm" 22:15:55 DEBUG }, 22:15:55 DEBUG "pod": "idm-b85f9d978-j5pbm" 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG "value": [ 22:15:55 DEBUG "pod": "ds-idrepo-0" 22:15:55 DEBUG 22:15:55 DEBUG 22:15:55 DEBUG }, 22:15:55 DEBUG "pod": "am-555f4db595-kzxsp" 22:15:55 DEBUG "pod": "am-555f4db595-kzxsp" 22:15:55 DEBUG 22:15:55 DEBUG }, 22:15:55 DEBUG }, 22:15:55 DEBUG 22:15:55 DEBUG }, 22:15:55 DEBUG 22:15:55 DEBUG "pod": "ds-idrepo-0" 22:15:55 DEBUG "pod": "am-555f4db595-kzxsp" 22:15:55 DEBUG 22:15:55 DEBUG }, 22:15:55 DEBUG }, 22:15:55 DEBUG "pod": "ds-idrepo-0" 22:15:55 DEBUG "value": [ 22:15:55 DEBUG "value": [ 22:15:55 DEBUG "pod": "am-555f4db595-kzxsp" 22:15:55 DEBUG }, 22:15:55 DEBUG 22:15:55 DEBUG "pod": "ds-idrepo-0" 22:15:55 DEBUG 1713302155, 22:15:55 DEBUG }, 22:15:55 DEBUG 22:15:55 DEBUG "pod": "idm-b85f9d978-j5pbm" 22:15:55 DEBUG "value": [ 22:15:55 DEBUG }, 22:15:55 DEBUG }, 22:15:55 DEBUG "instance": "10.142.0.43:9100" 22:15:55 DEBUG "value": [ 22:15:55 DEBUG "value": [ 22:15:55 DEBUG 22:15:55 DEBUG "value": [ 22:15:55 DEBUG "pod": "ds-cts-2" 22:15:55 DEBUG }, 22:15:55 DEBUG }, 22:15:55 DEBUG "instance": "10.142.0.43:9100" 22:15:55 DEBUG "value": [ 22:15:55 DEBUG "value": [ 22:15:55 DEBUG }, 22:15:55 DEBUG 1713302155, 22:15:55 DEBUG 1713302155, 22:15:55 DEBUG }, 22:15:55 DEBUG "value": [ 22:15:55 DEBUG "instance": "10.142.0.43:9100" 22:15:55 DEBUG }, 22:15:55 DEBUG "0" 22:15:55 DEBUG "value": [ 22:15:55 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:15:55 DEBUG }, 22:15:55 DEBUG 1713302155, 22:15:55 DEBUG "value": [ 22:15:55 DEBUG "value": [ 22:15:55 DEBUG }, 22:15:55 DEBUG 1713302155, 22:15:55 DEBUG 1713302155, 22:15:55 DEBUG "pod": "am-555f4db595-kzxsp" 22:15:55 DEBUG 1713302155, 22:15:55 DEBUG }, 22:15:55 DEBUG "value": [ 22:15:55 DEBUG "value": [ 22:15:55 DEBUG }, 22:15:55 DEBUG 1713302155, 22:15:55 DEBUG 1713302155, 22:15:55 DEBUG "value": [ 22:15:55 DEBUG "45374002.99999999" 22:15:55 DEBUG "915.3622468658747" 22:15:55 DEBUG "value": [ 22:15:55 DEBUG 1713302155, 22:15:55 DEBUG }, 22:15:55 DEBUG "value": [ 22:15:55 DEBUG ] 22:15:55 DEBUG 1713302155, 22:15:55 DEBUG }, 22:15:55 DEBUG "value": [ 22:15:55 DEBUG "20118.11256161863" 22:15:55 DEBUG 1713302155, 22:15:55 DEBUG 1713302155, 22:15:55 DEBUG "value": [ 22:15:55 DEBUG "0" 22:15:55 DEBUG "45364721.83333333" 22:15:55 DEBUG }, 22:15:55 DEBUG "0" 22:15:55 DEBUG "value": [ 22:15:55 DEBUG 1713302155, 22:15:55 DEBUG 1713302155, 22:15:55 DEBUG "value": [ 22:15:55 DEBUG "6.291224268335213" 22:15:55 DEBUG "0" 22:15:55 DEBUG 1713302155, 22:15:55 DEBUG ] 22:15:55 DEBUG ] 22:15:55 DEBUG 1713302155, 22:15:55 DEBUG "1992.5333333333333" 22:15:55 DEBUG "value": [ 22:15:55 DEBUG 1713302155, 22:15:55 DEBUG } 22:15:55 DEBUG "0" 22:15:55 DEBUG "value": [ 22:15:55 DEBUG 1713302155, 22:15:55 DEBUG ] 22:15:55 DEBUG "440619" 22:15:55 DEBUG "1992.7333333333333" 22:15:55 DEBUG 1713302155, 22:15:55 DEBUG ] 22:15:55 DEBUG ] 22:15:55 DEBUG "value": [ 22:15:55 DEBUG ] 22:15:55 DEBUG 1713302155, 22:15:55 DEBUG "0" 22:15:55 DEBUG "1992.4333333333334" 22:15:55 DEBUG 1713302155, 22:15:55 DEBUG ] 22:15:55 DEBUG ] 22:15:55 DEBUG "0" 22:15:55 DEBUG } 22:15:55 DEBUG } 22:15:55 DEBUG "0.006258226958643941" 22:15:55 DEBUG ] 22:15:55 DEBUG 1713302155, 22:15:55 DEBUG "0" 22:15:55 DEBUG ] 22:15:55 DEBUG ] 22:15:55 DEBUG 1713302155, 22:15:55 DEBUG "0" 22:15:55 DEBUG } 22:15:55 DEBUG ] 22:15:55 DEBUG ] 22:15:55 DEBUG "0" 22:15:55 DEBUG } 22:15:55 DEBUG } 22:15:55 DEBUG 1713302155, 22:15:55 DEBUG } 22:15:55 DEBUG "0" 22:15:55 DEBUG ] 22:15:55 DEBUG ] 22:15:55 DEBUG "19948311552" 22:15:55 DEBUG } 22:15:55 DEBUG } 22:15:55 DEBUG ] 22:15:55 DEBUG ] 22:15:55 DEBUG ] 22:15:55 DEBUG ] 22:15:55 DEBUG } 22:15:55 DEBUG "0.0005000000000004927" 22:15:55 DEBUG ] 22:15:55 DEBUG } 22:15:55 DEBUG } 22:15:55 DEBUG "633548800" 22:15:55 DEBUG ] 22:15:55 DEBUG ] 22:15:55 DEBUG } 22:15:55 DEBUG } 22:15:55 DEBUG ] 22:15:55 DEBUG ] 22:15:55 DEBUG ] 22:15:55 DEBUG "0.023" 22:15:55 DEBUG ] 22:15:55 DEBUG ] 22:15:55 DEBUG } 22:15:55 DEBUG } 22:15:55 DEBUG ] 22:15:55 DEBUG ] 22:15:55 DEBUG ] 22:15:55 DEBUG } 22:15:55 DEBUG } 22:15:55 DEBUG } 22:15:55 DEBUG } 22:15:55 DEBUG ] 22:15:55 DEBUG ] 22:15:55 DEBUG } 22:15:55 DEBUG } 22:15:55 DEBUG ] 22:15:55 DEBUG ] 22:15:55 DEBUG } 22:15:55 DEBUG } 22:15:55 DEBUG ] 22:15:55 DEBUG ] 22:15:55 DEBUG } 22:15:55 DEBUG } 22:15:55 DEBUG } 22:15:55 DEBUG ] 22:15:55 DEBUG } 22:15:55 DEBUG } 22:15:55 DEBUG ] 22:15:55 DEBUG ] 22:15:55 DEBUG } 22:15:55 DEBUG } 22:15:55 DEBUG } 22:15:55 DEBUG ] 22:15:55 DEBUG } 22:15:55 DEBUG } 22:15:55 DEBUG ] 22:15:55 DEBUG } 22:15:55 DEBUG } 22:15:55 DEBUG ] 22:15:55 DEBUG } 22:15:55 DEBUG } 22:15:55 DEBUG ] 22:15:55 DEBUG } 22:15:55 DEBUG } 22:15:55 DEBUG } 22:15:55 DEBUG ] 22:15:55 DEBUG } 22:15:55 DEBUG } 22:15:55 DEBUG } 22:15:55 DEBUG } 22:15:55 DEBUG ] 22:15:55 DEBUG } 22:15:55 DEBUG } 22:15:55 DEBUG ] 22:15:55 DEBUG } 22:15:55 DEBUG } 22:15:55 DEBUG } 22:15:55 DEBUG } 22:15:55 DEBUG } 22:15:55 DEBUG ] 22:15:55 DEBUG } 22:15:55 DEBUG } 22:15:55 DEBUG ] 22:15:55 DEBUG } 22:15:55 DEBUG } 22:15:55 DEBUG } 22:15:55 DEBUG } 22:15:55 DEBUG ] 22:15:55 DEBUG } 22:15:55 DEBUG } 22:15:55 DEBUG } 22:15:55 DEBUG } 22:15:55 DEBUG } 22:15:55 DEBUG } 22:15:55 DEBUG } 22:15:55 DEBUG } 22:15:55 DEBUG } 22:15:55 DEBUG } 22:15:55 DEBUG } 22:15:55 DEBUG } 22:15:55 DEBUG } 22:15:55 DEBUG } 22:15:55 DEBUG } 22:15:55 DEBUG } 22:15:55 DEBUG } 22:16:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302214" 22:16:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713302214" 22:16:54 ERROR [http_cmd]: ERROR 22:16:54 ERROR --- status code --- 22:16:54 ERROR http status code is 400 (expected [200, 401, 403, 404]) 22:16:54 ERROR --- http response --- 22:16:54 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 22:16:54 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 22:16:54 INFO [http_cmd]: http status code OK 22:16:54 DEBUG --- status code --- 22:16:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:16:54 DEBUG --- http response --- 22:16:54 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 22:16:55 INFO 22:16:55 INFO [loop_until]: kubectl --namespace=xlou top pods 22:16:55 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:16:55 INFO 22:16:55 INFO [loop_until]: kubectl --namespace=xlou top node 22:16:55 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:16:55 INFO [loop_until]: OK (rc = 0) 22:16:55 DEBUG --- stdout --- 22:16:55 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 9Mi am-555f4db595-4lbss 9030m 6995Mi am-555f4db595-fz8sn 9215m 7052Mi am-555f4db595-kzxsp 8930m 7216Mi ds-cts-0 5986m 9149Mi ds-cts-1 5647m 9207Mi ds-cts-2 5463m 9129Mi ds-idrepo-0 4619m 5032Mi ds-idrepo-1 4289m 2833Mi ds-idrepo-2 4439m 3059Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 4m 1208Mi idm-b85f9d978-rl76f 3m 1226Mi lodemon-bc756c647-cr7x6 14m 62Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-7c9b667974-9mr2s 6232m 511Mi 22:16:55 DEBUG --- stderr --- 22:16:55 DEBUG 22:16:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302215" 22:16:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302215" 22:16:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302215" 22:16:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302215" 22:16:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713302215" 22:16:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302215" 22:16:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302215" 22:16:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302215" 22:16:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302215" 22:16:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302215" 22:16:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302215" 22:16:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302215" 22:16:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302215" 22:16:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302215" 22:16:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302215" 22:16:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302215" 22:16:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302215" 22:16:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713302215" 22:16:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302215" 22:16:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302215" 22:16:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713302215" 22:16:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713302215" 22:16:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302215" 22:16:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302215" 22:16:55 INFO [http_cmd]: http status code OK 22:16:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302215" 22:16:55 INFO [http_cmd]: http status code OK 22:16:55 INFO [http_cmd]: http status code OK 22:16:55 INFO [http_cmd]: http status code OK 22:16:55 INFO [http_cmd]: http status code OK 22:16:55 INFO [http_cmd]: http status code OK 22:16:55 INFO [http_cmd]: http status code OK 22:16:55 INFO [loop_until]: OK (rc = 0) 22:16:55 INFO [http_cmd]: http status code OK 22:16:55 INFO [http_cmd]: http status code OK 22:16:55 INFO [http_cmd]: http status code OK 22:16:55 INFO [http_cmd]: http status code OK 22:16:55 INFO [http_cmd]: http status code OK 22:16:55 INFO [http_cmd]: http status code OK 22:16:55 INFO [http_cmd]: http status code OK 22:16:55 INFO [http_cmd]: http status code OK 22:16:55 INFO [http_cmd]: http status code OK 22:16:55 INFO [http_cmd]: http status code OK 22:16:55 DEBUG --- status code --- 22:16:55 INFO [http_cmd]: http status code OK 22:16:55 INFO [http_cmd]: http status code OK 22:16:55 DEBUG --- status code --- 22:16:55 INFO [http_cmd]: http status code OK 22:16:55 DEBUG --- status code --- 22:16:55 DEBUG --- status code --- 22:16:55 DEBUG --- status code --- 22:16:55 DEBUG --- status code --- 22:16:55 DEBUG --- status code --- 22:16:55 DEBUG --- stdout --- 22:16:55 DEBUG --- status code --- 22:16:55 DEBUG --- status code --- 22:16:55 DEBUG --- status code --- 22:16:55 DEBUG --- status code --- 22:16:55 INFO [http_cmd]: http status code OK 22:16:55 DEBUG --- status code --- 22:16:55 INFO [http_cmd]: http status code OK 22:16:55 INFO [http_cmd]: http status code OK 22:16:55 DEBUG --- status code --- 22:16:55 DEBUG --- status code --- 22:16:55 DEBUG --- status code --- 22:16:55 DEBUG --- status code --- 22:16:55 DEBUG --- status code --- 22:16:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:16:55 DEBUG --- status code --- 22:16:55 DEBUG --- status code --- 22:16:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:16:55 DEBUG --- status code --- 22:16:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:16:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:16:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:16:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:16:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:16:55 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 61m 0% 5083Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5640m 18% 11765Mi 10% gke-xlou-cdm-default-pool-1b0769c8-nhlf 9363m 31% 9264Mi 8% gke-xlou-cdm-default-pool-1b0769c8-qkns 87m 0% 2649Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 4673m 15% 6397Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 6609m 22% 13854Mi 12% gke-xlou-cdm-default-pool-1b0769c8-xnmp 69m 0% 3143Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 9328m 31% 8777Mi 7% gke-xlou-cdm-ds-0f601751-hmm7 4675m 15% 5564Mi 4% gke-xlou-cdm-ds-0f601751-j488 5599m 18% 11043Mi 9% gke-xlou-cdm-ds-0f601751-mc5d 10669m 35% 3331Mi 2% gke-xlou-cdm-ds-0f601751-nj6q 9546m 31% 9077Mi 8% gke-xlou-cdm-ds-0f601751-nmxm 4673m 15% 7360Mi 6% gke-xlou-cdm-frontend-a8771548-i82n 6427m 40% 5949Mi 10% 22:16:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:16:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:16:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:16:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:16:55 DEBUG --- status code --- 22:16:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:16:55 DEBUG --- status code --- 22:16:55 DEBUG --- status code --- 22:16:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:16:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:16:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:16:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:16:55 INFO [http_cmd]: http status code OK 22:16:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:16:55 DEBUG --- http response --- 22:16:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:16:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:16:55 DEBUG --- http response --- 22:16:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:16:55 DEBUG --- http response --- 22:16:55 DEBUG --- http response --- 22:16:55 DEBUG --- http response --- 22:16:55 DEBUG --- http response --- 22:16:55 DEBUG --- http response --- 22:16:55 DEBUG --- stderr --- 22:16:55 DEBUG --- http response --- 22:16:55 DEBUG --- http response --- 22:16:55 DEBUG --- http response --- 22:16:55 INFO [http_cmd]: http status code OK 22:16:55 DEBUG --- http response --- 22:16:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:16:55 DEBUG --- http response --- 22:16:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:16:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:16:55 DEBUG --- http response --- 22:16:55 DEBUG --- http response --- 22:16:55 DEBUG --- http response --- 22:16:55 DEBUG --- http response --- 22:16:55 DEBUG --- status code --- 22:16:55 DEBUG --- http response --- 22:16:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:16:55 DEBUG --- http response --- 22:16:55 DEBUG --- http response --- 22:16:55 DEBUG --- http response --- 22:16:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:16:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 22:16:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 22:16:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:16:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:16:55 DEBUG 22:16:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:16:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:16:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:16:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:16:55 DEBUG --- status code --- 22:16:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:16:55 DEBUG --- http response --- 22:16:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:16:55 DEBUG --- http response --- 22:16:55 DEBUG --- http response --- 22:16:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:16:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:16:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:16:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:16:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:16:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 22:16:55 DEBUG ----- output ----- 22:16:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:16:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:16:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:16:55 DEBUG ----- output ----- 22:16:55 DEBUG ----- output ----- 22:16:55 DEBUG ----- output ----- 22:16:55 DEBUG ----- output ----- 22:16:55 DEBUG ----- output ----- 22:16:55 DEBUG ----- output ----- 22:16:55 DEBUG ----- output ----- 22:16:55 DEBUG ----- output ----- 22:16:55 DEBUG ----- output ----- 22:16:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:16:55 DEBUG ----- output ----- 22:16:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:16:55 DEBUG ----- output ----- 22:16:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:16:55 DEBUG ----- output ----- 22:16:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:16:55 DEBUG ----- output ----- 22:16:55 DEBUG ----- output ----- 22:16:55 DEBUG { 22:16:55 DEBUG --- http response --- 22:16:55 DEBUG ----- output ----- 22:16:55 DEBUG { 22:16:55 DEBUG ----- output ----- 22:16:55 DEBUG ----- output ----- 22:16:55 DEBUG ----- output ----- 22:16:55 DEBUG { 22:16:55 DEBUG { 22:16:55 DEBUG { 22:16:55 DEBUG { 22:16:55 DEBUG { 22:16:55 DEBUG { 22:16:55 DEBUG { 22:16:55 DEBUG { 22:16:55 DEBUG { 22:16:55 DEBUG --- http response --- 22:16:55 DEBUG { 22:16:55 DEBUG ----- output ----- 22:16:55 DEBUG { 22:16:55 DEBUG ----- output ----- 22:16:55 DEBUG { 22:16:55 DEBUG ----- output ----- 22:16:55 DEBUG { 22:16:55 DEBUG ----- output ----- 22:16:55 DEBUG "status": "success", 22:16:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 22:16:55 DEBUG { 22:16:55 DEBUG "status": "success", 22:16:55 DEBUG { 22:16:55 DEBUG { 22:16:55 DEBUG { 22:16:55 DEBUG "status": "success", 22:16:55 DEBUG "status": "success", 22:16:55 DEBUG "status": "success", 22:16:55 DEBUG "status": "success", 22:16:55 DEBUG "status": "success", 22:16:55 DEBUG "status": "success", 22:16:55 DEBUG "status": "success", 22:16:55 DEBUG "status": "success", 22:16:55 DEBUG "status": "success", 22:16:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:16:55 DEBUG "status": "success", 22:16:55 DEBUG { 22:16:55 DEBUG "status": "success", 22:16:55 DEBUG { 22:16:55 DEBUG "status": "success", 22:16:55 DEBUG { 22:16:55 DEBUG "status": "success", 22:16:55 DEBUG { 22:16:55 DEBUG "data": { 22:16:55 DEBUG ----- output ----- 22:16:55 DEBUG "status": "success", 22:16:55 DEBUG "data": { 22:16:55 DEBUG "status": "success", 22:16:55 DEBUG "status": "success", 22:16:55 DEBUG "status": "success", 22:16:55 DEBUG "data": { 22:16:55 DEBUG "data": { 22:16:55 DEBUG "data": { 22:16:55 DEBUG "data": { 22:16:55 DEBUG "data": { 22:16:55 DEBUG "data": { 22:16:55 DEBUG "data": { 22:16:55 DEBUG "data": { 22:16:55 DEBUG "data": { 22:16:55 DEBUG ----- output ----- 22:16:55 DEBUG "data": { 22:16:55 DEBUG "status": "success", 22:16:55 DEBUG "data": { 22:16:55 DEBUG "status": "success", 22:16:55 DEBUG "data": { 22:16:55 DEBUG "status": "success", 22:16:55 DEBUG "data": { 22:16:55 DEBUG "status": "success", 22:16:55 DEBUG "resultType": "vector", 22:16:55 DEBUG { 22:16:55 DEBUG "data": { 22:16:55 DEBUG "resultType": "vector", 22:16:55 DEBUG "data": { 22:16:55 DEBUG "data": { 22:16:55 DEBUG "data": { 22:16:55 DEBUG "resultType": "vector", 22:16:55 DEBUG "resultType": "vector", 22:16:55 DEBUG "resultType": "vector", 22:16:55 DEBUG "resultType": "vector", 22:16:55 DEBUG "resultType": "vector", 22:16:55 DEBUG "resultType": "vector", 22:16:55 DEBUG "resultType": "vector", 22:16:55 DEBUG "resultType": "vector", 22:16:55 DEBUG "resultType": "vector", 22:16:55 DEBUG { 22:16:55 DEBUG "resultType": "vector", 22:16:55 DEBUG "data": { 22:16:55 DEBUG "resultType": "vector", 22:16:55 DEBUG "data": { 22:16:55 DEBUG "resultType": "vector", 22:16:55 DEBUG "data": { 22:16:55 DEBUG "resultType": "vector", 22:16:55 DEBUG "data": { 22:16:55 DEBUG "result": [ 22:16:55 DEBUG "status": "success", 22:16:55 DEBUG "resultType": "vector", 22:16:55 DEBUG "result": [ 22:16:55 DEBUG "resultType": "vector", 22:16:55 DEBUG "resultType": "vector", 22:16:55 DEBUG "resultType": "vector", 22:16:55 DEBUG "result": [ 22:16:55 DEBUG "result": [ 22:16:55 DEBUG "result": [ 22:16:55 DEBUG "result": [ 22:16:55 DEBUG "result": [ 22:16:55 DEBUG "result": [ 22:16:55 DEBUG "result": [ 22:16:55 DEBUG "result": [ 22:16:55 DEBUG "result": [ 22:16:55 DEBUG "status": "success", 22:16:55 DEBUG "result": [ 22:16:55 DEBUG "resultType": "vector", 22:16:55 DEBUG "result": [ 22:16:55 DEBUG "resultType": "vector", 22:16:55 DEBUG "result": [ 22:16:55 DEBUG "resultType": "vector", 22:16:55 DEBUG "result": [ 22:16:55 DEBUG "resultType": "vector", 22:16:55 DEBUG { 22:16:55 DEBUG "data": { 22:16:55 DEBUG "result": [ 22:16:55 DEBUG { 22:16:55 DEBUG "result": [ 22:16:55 DEBUG "result": [ 22:16:55 DEBUG "result": [ 22:16:55 DEBUG { 22:16:55 DEBUG { 22:16:55 DEBUG { 22:16:55 DEBUG { 22:16:55 DEBUG "metric": { 22:16:55 DEBUG { 22:16:55 DEBUG { 22:16:55 DEBUG { 22:16:55 DEBUG { 22:16:55 DEBUG "data": { 22:16:55 DEBUG { 22:16:55 DEBUG "result": [ 22:16:55 DEBUG { 22:16:55 DEBUG "result": [ 22:16:55 DEBUG { 22:16:55 DEBUG "result": [ 22:16:55 DEBUG { 22:16:55 DEBUG "result": [ 22:16:55 DEBUG "metric": { 22:16:55 DEBUG "resultType": "vector", 22:16:55 DEBUG "result": [ 22:16:55 DEBUG "metric": { 22:16:55 DEBUG { 22:16:55 DEBUG { 22:16:55 DEBUG { 22:16:55 DEBUG "metric": { 22:16:55 DEBUG "metric": { 22:16:55 DEBUG "metric": { 22:16:55 DEBUG { 22:16:55 DEBUG "pod": "ds-idrepo-2" 22:16:55 DEBUG "metric": { 22:16:55 DEBUG "metric": { 22:16:55 DEBUG "metric": { 22:16:55 DEBUG "metric": { 22:16:55 DEBUG "resultType": "vector", 22:16:55 DEBUG "metric": { 22:16:55 DEBUG { 22:16:55 DEBUG "metric": { 22:16:55 DEBUG { 22:16:55 DEBUG "metric": { 22:16:55 DEBUG { 22:16:55 DEBUG "metric": { 22:16:55 DEBUG { 22:16:55 DEBUG "pod": "ds-idrepo-2" 22:16:55 DEBUG { 22:16:55 DEBUG { 22:16:55 DEBUG "pod": "ds-idrepo-2" 22:16:55 DEBUG "metric": { 22:16:55 DEBUG "metric": { 22:16:55 DEBUG "metric": { 22:16:55 DEBUG "pod": "am-555f4db595-fz8sn" 22:16:55 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:16:55 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:16:55 DEBUG "metric": { 22:16:55 DEBUG }, 22:16:55 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:16:55 DEBUG "pod": "am-555f4db595-fz8sn" 22:16:55 DEBUG "pod": "am-555f4db595-fz8sn" 22:16:55 DEBUG "instance": "10.142.0.107:9100" 22:16:55 DEBUG "result": [ 22:16:55 DEBUG "pod": "am-555f4db595-fz8sn" 22:16:55 DEBUG "metric": { 22:16:55 DEBUG "pod": "ds-idrepo-2" 22:16:55 DEBUG "metric": { 22:16:55 DEBUG "pod": "am-555f4db595-fz8sn" 22:16:55 DEBUG "metric": { 22:16:55 DEBUG "pod": "am-555f4db595-fz8sn" 22:16:55 DEBUG "metric": { 22:16:55 DEBUG }, 22:16:55 DEBUG "metric": { 22:16:55 DEBUG "metric": { 22:16:55 DEBUG }, 22:16:55 DEBUG "pod": "ds-idrepo-2" 22:16:55 DEBUG "instance": "10.142.0.107:9100" 22:16:55 DEBUG "pod": "ds-idrepo-2" 22:16:55 DEBUG }, 22:16:55 DEBUG }, 22:16:55 DEBUG }, 22:16:55 DEBUG "pod": "ds-idrepo-2" 22:16:55 DEBUG "value": [ 22:16:55 DEBUG }, 22:16:55 DEBUG }, 22:16:55 DEBUG }, 22:16:55 DEBUG }, 22:16:55 DEBUG { 22:16:55 DEBUG }, 22:16:55 DEBUG "pod": "am-555f4db595-fz8sn" 22:16:55 DEBUG }, 22:16:55 DEBUG "instance": "10.142.0.107:9100" 22:16:55 DEBUG }, 22:16:55 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:16:55 DEBUG }, 22:16:55 DEBUG "instance": "10.142.0.107:9100" 22:16:55 DEBUG "value": [ 22:16:55 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:16:55 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:16:55 DEBUG "value": [ 22:16:55 DEBUG }, 22:16:55 DEBUG }, 22:16:55 DEBUG }, 22:16:55 DEBUG "value": [ 22:16:55 DEBUG "value": [ 22:16:55 DEBUG "value": [ 22:16:55 DEBUG }, 22:16:55 DEBUG 22:16:55 DEBUG "value": [ 22:16:55 DEBUG "value": [ 22:16:55 DEBUG "value": [ 22:16:55 DEBUG "value": [ 22:16:55 DEBUG "metric": { 22:16:55 DEBUG "value": [ 22:16:55 DEBUG }, 22:16:55 DEBUG "value": [ 22:16:55 DEBUG }, 22:16:55 DEBUG "value": [ 22:16:55 DEBUG }, 22:16:55 DEBUG "value": [ 22:16:55 DEBUG }, 22:16:55 DEBUG 22:16:55 DEBUG }, 22:16:55 DEBUG }, 22:16:55 DEBUG 22:16:55 DEBUG "value": [ 22:16:55 DEBUG "value": [ 22:16:55 DEBUG "value": [ 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG "value": [ 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG "instance": "10.142.0.107:9100" 22:16:55 DEBUG 22:16:55 DEBUG "value": [ 22:16:55 DEBUG 22:16:55 DEBUG "value": [ 22:16:55 DEBUG 22:16:55 DEBUG "value": [ 22:16:55 DEBUG 22:16:55 DEBUG "value": [ 22:16:55 DEBUG 22:16:55 DEBUG "value": [ 22:16:55 DEBUG "value": [ 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG }, 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG [...] 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG "value": [ 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG [...] 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG [...] 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG [...] 22:16:55 DEBUG [...] 22:16:55 DEBUG [...] 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG [...] 22:16:55 DEBUG [...] 22:16:55 DEBUG [...] 22:16:55 DEBUG [...] 22:16:55 DEBUG 22:16:55 DEBUG [...] 22:16:55 DEBUG 22:16:55 DEBUG [...] 22:16:55 DEBUG 22:16:55 DEBUG [...] 22:16:55 DEBUG 22:16:55 DEBUG [...] 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG [...] 22:16:55 DEBUG [...] 22:16:55 DEBUG [...] 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG [...] 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG [...] 22:16:55 DEBUG 22:16:55 DEBUG [...] 22:16:55 DEBUG 22:16:55 DEBUG [...] 22:16:55 DEBUG 22:16:55 DEBUG [...] 22:16:55 DEBUG 22:16:55 DEBUG [...] 22:16:55 DEBUG [...] 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG "pod": "ds-idrepo-0" 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG [...] 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG "pod": "ds-idrepo-0" 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG "pod": "ds-idrepo-0" 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG "pod": "am-555f4db595-kzxsp" 22:16:55 DEBUG "pod": "ds-idrepo-2" 22:16:55 DEBUG "pod": "ds-idrepo-2" 22:16:55 DEBUG 22:16:55 DEBUG }, 22:16:55 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:16:55 DEBUG "pod": "am-555f4db595-kzxsp" 22:16:55 DEBUG "pod": "am-555f4db595-kzxsp" 22:16:55 DEBUG "instance": "10.142.0.43:9100" 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG "pod": "am-555f4db595-kzxsp" 22:16:55 DEBUG "pod": "ds-idrepo-0" 22:16:55 DEBUG 22:16:55 DEBUG "pod": "am-555f4db595-kzxsp" 22:16:55 DEBUG 22:16:55 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:16:55 DEBUG 22:16:55 DEBUG }, 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG }, 22:16:55 DEBUG "pod": "ds-cts-2" 22:16:55 DEBUG "instance": "10.142.0.43:9100" 22:16:55 DEBUG "pod": "ds-idrepo-0" 22:16:55 DEBUG }, 22:16:55 DEBUG }, 22:16:55 DEBUG }, 22:16:55 DEBUG "pod": "ds-idrepo-0" 22:16:55 DEBUG "value": [ 22:16:55 DEBUG }, 22:16:55 DEBUG }, 22:16:55 DEBUG }, 22:16:55 DEBUG }, 22:16:55 DEBUG 22:16:55 DEBUG 22:16:55 DEBUG }, 22:16:55 DEBUG }, 22:16:55 DEBUG "instance": "10.142.0.43:9100" 22:16:55 DEBUG }, 22:16:55 DEBUG "pod": "am-555f4db595-kzxsp" 22:16:55 DEBUG }, 22:16:55 DEBUG "instance": "10.142.0.43:9100" 22:16:55 DEBUG "value": [ 22:16:55 DEBUG "pod": "ds-idrepo-2" 22:16:55 DEBUG "pod": "ds-idrepo-2" 22:16:55 DEBUG }, 22:16:55 DEBUG }, 22:16:55 DEBUG }, 22:16:55 DEBUG }, 22:16:55 DEBUG "value": [ 22:16:55 DEBUG "value": [ 22:16:55 DEBUG "value": [ 22:16:55 DEBUG }, 22:16:55 DEBUG 1713302215, 22:16:55 DEBUG "value": [ 22:16:55 DEBUG "value": [ 22:16:55 DEBUG "value": [ 22:16:55 DEBUG "value": [ 22:16:55 DEBUG "instance": "10.142.0.43:9100" 22:16:55 DEBUG "pod": "am-555f4db595-kzxsp" 22:16:55 DEBUG "value": [ 22:16:55 DEBUG "value": [ 22:16:55 DEBUG }, 22:16:55 DEBUG "value": [ 22:16:55 DEBUG }, 22:16:55 DEBUG "value": [ 22:16:55 DEBUG }, 22:16:55 DEBUG 1713302215, 22:16:55 DEBUG }, 22:16:55 DEBUG "value": [ 22:16:55 DEBUG "value": [ 22:16:55 DEBUG "value": [ 22:16:55 DEBUG "value": [ 22:16:55 DEBUG "value": [ 22:16:55 DEBUG 1713302215, 22:16:55 DEBUG 1713302215, 22:16:55 DEBUG 1713302215, 22:16:55 DEBUG "value": [ 22:16:55 DEBUG "0" 22:16:55 DEBUG 1713302215, 22:16:55 DEBUG 1713302215, 22:16:55 DEBUG 1713302215, 22:16:55 DEBUG "0" 22:16:55 DEBUG }, 22:16:55 DEBUG }, 22:16:55 DEBUG 1713302215, 22:16:55 DEBUG 1713302215, 22:16:55 DEBUG "value": [ 22:16:55 DEBUG 1713302215, 22:16:55 DEBUG "value": [ 22:16:55 DEBUG 1713302215, 22:16:55 DEBUG "value": [ 22:16:55 DEBUG "0" 22:16:55 DEBUG "value": [ 22:16:55 DEBUG 1713302215, 22:16:55 DEBUG 1713302215, 22:16:55 DEBUG 1713302215, 22:16:55 DEBUG 1713302215, 22:16:55 DEBUG 1713302215, 22:16:55 DEBUG "0" 22:16:55 DEBUG "3614.6286060055972" 22:16:55 DEBUG "233.20184554874822" 22:16:55 DEBUG 1713302215, 22:16:55 DEBUG ] 22:16:55 DEBUG "6.297168583178237" 22:16:55 DEBUG "1991.3" 22:16:55 DEBUG 1713302215, 22:16:55 DEBUG ] 22:16:55 DEBUG "value": [ 22:16:55 DEBUG "value": [ 22:16:55 DEBUG "1991.3333333333333" 22:16:55 DEBUG "0" 22:16:55 DEBUG 1713302215, 22:16:55 DEBUG "1991.2333333333333" 22:16:55 DEBUG 1713302215, 22:16:55 DEBUG "634257408" 22:16:55 DEBUG 1713302215, 22:16:55 DEBUG ] 22:16:55 DEBUG 1713302215, 22:16:55 DEBUG "0" 22:16:55 DEBUG "349.80276832312234" 22:16:55 DEBUG "0" 22:16:55 DEBUG "19948936192" 22:16:55 DEBUG "0" 22:16:55 DEBUG ] 22:16:55 DEBUG ] 22:16:55 DEBUG ] 22:16:55 DEBUG "0" 22:16:55 DEBUG } 22:16:55 DEBUG ] 22:16:55 DEBUG ] 22:16:55 DEBUG "45238095.5" 22:16:55 DEBUG } 22:16:55 DEBUG 1713302215, 22:16:55 DEBUG 1713302215, 22:16:55 DEBUG ] 22:16:55 DEBUG ] 22:16:55 DEBUG "0.00046666666666700016" 22:16:55 DEBUG ] 22:16:55 DEBUG "560476.5" 22:16:55 DEBUG ] 22:16:55 DEBUG "45228767.43333333" 22:16:55 DEBUG } 22:16:55 DEBUG "2.2076182233823025" 22:16:55 DEBUG ] 22:16:55 DEBUG ] 22:16:55 DEBUG ] 22:16:55 DEBUG ] 22:16:55 DEBUG ] 22:16:55 DEBUG } 22:16:55 DEBUG ] 22:16:55 DEBUG } 22:16:55 DEBUG ] 22:16:55 DEBUG ] 22:16:55 DEBUG } 22:16:55 DEBUG } 22:16:55 DEBUG ] 22:16:55 DEBUG ] 22:16:55 DEBUG "0.0003333333333333262" 22:16:55 DEBUG "0.023" 22:16:55 DEBUG } 22:16:55 DEBUG } 22:16:55 DEBUG ] 22:16:55 DEBUG } 22:16:55 DEBUG ] 22:16:55 DEBUG } 22:16:55 DEBUG ] 22:16:55 DEBUG ] 22:16:55 DEBUG ] 22:16:55 DEBUG } 22:16:55 DEBUG } 22:16:55 DEBUG } 22:16:55 DEBUG } 22:16:55 DEBUG } 22:16:55 DEBUG } 22:16:55 DEBUG } 22:16:55 DEBUG ] 22:16:55 DEBUG } 22:16:55 DEBUG } 22:16:55 DEBUG ] 22:16:55 DEBUG ] 22:16:55 DEBUG } 22:16:55 DEBUG } 22:16:55 DEBUG ] 22:16:55 DEBUG ] 22:16:55 DEBUG ] 22:16:55 DEBUG ] 22:16:55 DEBUG } 22:16:55 DEBUG ] 22:16:55 DEBUG } 22:16:55 DEBUG ] 22:16:55 DEBUG } 22:16:55 DEBUG } 22:16:55 DEBUG } 22:16:55 DEBUG ] 22:16:55 DEBUG ] 22:16:55 DEBUG ] 22:16:55 DEBUG ] 22:16:55 DEBUG ] 22:16:55 DEBUG ] 22:16:55 DEBUG } 22:16:55 DEBUG } 22:16:55 DEBUG ] 22:16:55 DEBUG } 22:16:55 DEBUG } 22:16:55 DEBUG } 22:16:55 DEBUG ] 22:16:55 DEBUG } 22:16:55 DEBUG } 22:16:55 DEBUG } 22:16:55 DEBUG } 22:16:55 DEBUG } 22:16:55 DEBUG ] 22:16:55 DEBUG } 22:16:55 DEBUG ] 22:16:55 DEBUG } 22:16:55 DEBUG ] 22:16:55 DEBUG } 22:16:55 DEBUG ] 22:16:55 DEBUG } 22:16:55 DEBUG } 22:16:55 DEBUG } 22:16:55 DEBUG } 22:16:55 DEBUG } 22:16:55 DEBUG } 22:16:55 DEBUG } 22:16:55 DEBUG } 22:16:55 DEBUG } 22:16:55 DEBUG } 22:16:55 DEBUG } 22:16:55 DEBUG ] 22:16:55 DEBUG ] 22:16:55 DEBUG } 22:16:55 DEBUG } 22:16:55 DEBUG } 22:16:55 DEBUG } 22:16:55 DEBUG } 22:16:55 DEBUG } 22:16:55 DEBUG } 22:16:55 DEBUG } 22:16:55 DEBUG } 22:16:55 DEBUG } 22:16:55 DEBUG } 22:16:55 DEBUG } 22:16:55 DEBUG } 22:16:55 DEBUG } 22:16:55 DEBUG } 22:16:55 DEBUG } 22:16:55 DEBUG } 22:16:55 DEBUG } 22:16:55 DEBUG } 22:16:55 DEBUG } 22:16:55 DEBUG } 22:16:55 DEBUG } 22:16:55 DEBUG } 22:16:55 DEBUG } 22:17:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713302274" 22:17:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302274" 22:17:54 ERROR [http_cmd]: ERROR 22:17:54 INFO [http_cmd]: http status code OK 22:17:54 ERROR --- status code --- 22:17:54 DEBUG --- status code --- 22:17:54 ERROR http status code is 400 (expected [200, 401, 403, 404]) 22:17:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:17:54 ERROR --- http response --- 22:17:54 DEBUG --- http response --- 22:17:54 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 22:17:54 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 22:17:54 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 22:17:55 INFO 22:17:55 INFO [loop_until]: kubectl --namespace=xlou top pods 22:17:55 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:17:55 INFO [loop_until]: OK (rc = 0) 22:17:55 DEBUG --- stdout --- 22:17:55 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 9Mi am-555f4db595-4lbss 9271m 8733Mi am-555f4db595-fz8sn 9232m 8732Mi am-555f4db595-kzxsp 9355m 8798Mi ds-cts-0 6166m 10881Mi ds-cts-1 5665m 10468Mi ds-cts-2 5416m 10748Mi ds-idrepo-0 4873m 5032Mi ds-idrepo-1 4455m 2833Mi ds-idrepo-2 4525m 3067Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 4m 1208Mi idm-b85f9d978-rl76f 4m 1226Mi lodemon-bc756c647-cr7x6 11m 62Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-7c9b667974-9mr2s 6280m 514Mi 22:17:55 DEBUG --- stderr --- 22:17:55 DEBUG 22:17:55 INFO 22:17:55 INFO [loop_until]: kubectl --namespace=xlou top node 22:17:55 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:17:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302275" 22:17:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302275" 22:17:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302275" 22:17:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302275" 22:17:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302275" 22:17:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713302275" 22:17:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302275" 22:17:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302275" 22:17:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302275" 22:17:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302275" 22:17:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713302275" 22:17:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302275" 22:17:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302275" 22:17:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302275" 22:17:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302275" 22:17:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302275" 22:17:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302275" 22:17:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713302275" 22:17:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302275" 22:17:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302275" 22:17:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713302275" 22:17:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302275" 22:17:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302275" 22:17:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302275" 22:17:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302275" 22:17:55 INFO [loop_until]: OK (rc = 0) 22:17:55 INFO [http_cmd]: http status code OK 22:17:55 INFO [http_cmd]: http status code OK 22:17:55 INFO [http_cmd]: http status code OK 22:17:55 INFO [http_cmd]: http status code OK 22:17:55 INFO [http_cmd]: http status code OK 22:17:55 INFO [http_cmd]: http status code OK 22:17:55 INFO [http_cmd]: http status code OK 22:17:55 INFO [http_cmd]: http status code OK 22:17:55 INFO [http_cmd]: http status code OK 22:17:55 INFO [http_cmd]: http status code OK 22:17:55 INFO [http_cmd]: http status code OK 22:17:55 INFO [http_cmd]: http status code OK 22:17:55 INFO [http_cmd]: http status code OK 22:17:55 INFO [http_cmd]: http status code OK 22:17:55 INFO [http_cmd]: http status code OK 22:17:55 INFO [http_cmd]: http status code OK 22:17:55 DEBUG --- stdout --- 22:17:55 DEBUG --- status code --- 22:17:55 INFO [http_cmd]: http status code OK 22:17:55 INFO [http_cmd]: http status code OK 22:17:55 INFO [http_cmd]: http status code OK 22:17:55 DEBUG --- status code --- 22:17:55 INFO [http_cmd]: http status code OK 22:17:55 DEBUG --- status code --- 22:17:55 DEBUG --- status code --- 22:17:55 DEBUG --- status code --- 22:17:55 DEBUG --- status code --- 22:17:55 DEBUG --- status code --- 22:17:55 INFO [http_cmd]: http status code OK 22:17:55 DEBUG --- status code --- 22:17:55 INFO [http_cmd]: http status code OK 22:17:55 DEBUG --- status code --- 22:17:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:17:55 INFO [http_cmd]: http status code OK 22:17:55 DEBUG --- status code --- 22:17:55 DEBUG --- status code --- 22:17:55 DEBUG --- status code --- 22:17:55 DEBUG --- status code --- 22:17:55 DEBUG --- status code --- 22:17:55 DEBUG --- status code --- 22:17:55 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 62m 0% 5086Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5673m 19% 13235Mi 11% gke-xlou-cdm-default-pool-1b0769c8-nhlf 9496m 31% 10961Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 83m 0% 2635Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 4662m 15% 6406Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 6671m 22% 15410Mi 13% gke-xlou-cdm-default-pool-1b0769c8-xnmp 67m 0% 3146Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 9278m 31% 10499Mi 9% gke-xlou-cdm-ds-0f601751-hmm7 4702m 15% 5569Mi 4% gke-xlou-cdm-ds-0f601751-j488 5498m 18% 12457Mi 11% gke-xlou-cdm-ds-0f601751-mc5d 10701m 35% 3347Mi 2% gke-xlou-cdm-ds-0f601751-nj6q 9756m 32% 10655Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 4750m 15% 7358Mi 6% gke-xlou-cdm-frontend-a8771548-i82n 6435m 40% 5948Mi 10% 22:17:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:17:55 DEBUG --- status code --- 22:17:55 DEBUG --- status code --- 22:17:55 INFO [http_cmd]: http status code OK 22:17:55 DEBUG --- status code --- 22:17:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:17:55 DEBUG --- status code --- 22:17:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:17:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:17:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:17:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:17:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:17:55 DEBUG --- status code --- 22:17:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:17:55 DEBUG --- status code --- 22:17:55 DEBUG --- status code --- 22:17:55 DEBUG --- http response --- 22:17:55 DEBUG --- status code --- 22:17:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:17:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:17:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:17:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:17:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:17:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:17:55 DEBUG --- stderr --- 22:17:55 DEBUG --- http response --- 22:17:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:17:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:17:55 DEBUG --- status code --- 22:17:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:17:55 DEBUG --- http response --- 22:17:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:17:55 DEBUG --- http response --- 22:17:55 DEBUG --- http response --- 22:17:55 DEBUG --- http response --- 22:17:55 INFO [http_cmd]: http status code OK 22:17:55 DEBUG --- http response --- 22:17:55 DEBUG --- http response --- 22:17:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:17:55 DEBUG --- http response --- 22:17:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:17:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:17:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:17:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:17:55 DEBUG --- http response --- 22:17:55 DEBUG --- http response --- 22:17:55 DEBUG --- http response --- 22:17:55 DEBUG --- http response --- 22:17:55 DEBUG --- http response --- 22:17:55 DEBUG --- http response --- 22:17:55 DEBUG 22:17:55 DEBUG --- http response --- 22:17:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:17:55 DEBUG --- http response --- 22:17:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:17:55 DEBUG --- http response --- 22:17:55 DEBUG --- http response --- 22:17:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:17:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:17:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:17:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:17:55 DEBUG --- status code --- 22:17:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:17:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:17:55 DEBUG --- http response --- 22:17:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:17:55 DEBUG --- http response --- 22:17:55 DEBUG --- http response --- 22:17:55 DEBUG --- http response --- 22:17:55 DEBUG ----- output ----- 22:17:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:17:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:17:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:17:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:17:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:17:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 22:17:55 DEBUG ----- output ----- 22:17:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:17:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:17:55 DEBUG --- http response --- 22:17:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:17:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 22:17:55 DEBUG ----- output ----- 22:17:55 DEBUG ----- output ----- 22:17:55 DEBUG ----- output ----- 22:17:55 DEBUG ----- output ----- 22:17:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:17:55 DEBUG ----- output ----- 22:17:55 DEBUG ----- output ----- 22:17:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:17:55 DEBUG ----- output ----- 22:17:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:17:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:17:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:17:55 DEBUG { 22:17:55 DEBUG ----- output ----- 22:17:55 DEBUG ----- output ----- 22:17:55 DEBUG ----- output ----- 22:17:55 DEBUG ----- output ----- 22:17:55 DEBUG ----- output ----- 22:17:55 DEBUG ----- output ----- 22:17:55 DEBUG { 22:17:55 DEBUG ----- output ----- 22:17:55 DEBUG ----- output ----- 22:17:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:17:55 DEBUG ----- output ----- 22:17:55 DEBUG ----- output ----- 22:17:55 DEBUG { 22:17:55 DEBUG { 22:17:55 DEBUG { 22:17:55 DEBUG { 22:17:55 DEBUG --- http response --- 22:17:55 DEBUG { 22:17:55 DEBUG { 22:17:55 DEBUG ----- output ----- 22:17:55 DEBUG { 22:17:55 DEBUG ----- output ----- 22:17:55 DEBUG ----- output ----- 22:17:55 DEBUG ----- output ----- 22:17:55 DEBUG "status": "success", 22:17:55 DEBUG { 22:17:55 DEBUG { 22:17:55 DEBUG { 22:17:55 DEBUG { 22:17:55 DEBUG { 22:17:55 DEBUG { 22:17:55 DEBUG "status": "success", 22:17:55 DEBUG { 22:17:55 DEBUG { 22:17:55 DEBUG ----- output ----- 22:17:55 DEBUG { 22:17:55 DEBUG { 22:17:55 DEBUG "status": "success", 22:17:55 DEBUG "status": "success", 22:17:55 DEBUG "status": "success", 22:17:55 DEBUG "status": "success", 22:17:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:17:55 DEBUG "status": "success", 22:17:55 DEBUG "status": "success", 22:17:55 DEBUG { 22:17:55 DEBUG "status": "success", 22:17:55 DEBUG { 22:17:55 DEBUG "status": "success", 22:17:55 DEBUG { 22:17:55 DEBUG "data": { 22:17:55 DEBUG "status": "success", 22:17:55 DEBUG "status": "success", 22:17:55 DEBUG "status": "success", 22:17:55 DEBUG "status": "success", 22:17:55 DEBUG "status": "success", 22:17:55 DEBUG "status": "success", 22:17:55 DEBUG "data": { 22:17:55 DEBUG "status": "success", 22:17:55 DEBUG "status": "success", 22:17:55 DEBUG { 22:17:55 DEBUG "status": "success", 22:17:55 DEBUG "status": "success", 22:17:55 DEBUG "data": { 22:17:55 DEBUG "data": { 22:17:55 DEBUG "data": { 22:17:55 DEBUG "data": { 22:17:55 DEBUG ----- output ----- 22:17:55 DEBUG "data": { 22:17:55 DEBUG "data": { 22:17:55 DEBUG "status": "success", 22:17:55 DEBUG "data": { 22:17:55 DEBUG "data": { 22:17:55 DEBUG { 22:17:55 DEBUG "status": "success", 22:17:55 DEBUG "resultType": "vector", 22:17:55 DEBUG "data": { 22:17:55 DEBUG "data": { 22:17:55 DEBUG "data": { 22:17:55 DEBUG "data": { 22:17:55 DEBUG "data": { 22:17:55 DEBUG "data": { 22:17:55 DEBUG "resultType": "vector", 22:17:55 DEBUG "data": { 22:17:55 DEBUG "data": { 22:17:55 DEBUG "status": "success", 22:17:55 DEBUG "data": { 22:17:55 DEBUG "data": { 22:17:55 DEBUG "resultType": "vector", 22:17:55 DEBUG "resultType": "vector", 22:17:55 DEBUG "resultType": "vector", 22:17:55 DEBUG "resultType": "vector", 22:17:55 DEBUG { 22:17:55 DEBUG "resultType": "vector", 22:17:55 DEBUG "resultType": "vector", 22:17:55 DEBUG "data": { 22:17:55 DEBUG "resultType": "vector", 22:17:55 DEBUG "resultType": "vector", 22:17:55 DEBUG "status": "success", 22:17:55 DEBUG "data": { 22:17:55 DEBUG "result": [ 22:17:55 DEBUG "resultType": "vector", 22:17:55 DEBUG "resultType": "vector", 22:17:55 DEBUG "resultType": "vector", 22:17:55 DEBUG "resultType": "vector", 22:17:55 DEBUG "resultType": "vector", 22:17:55 DEBUG "resultType": "vector", 22:17:55 DEBUG "result": [ 22:17:55 DEBUG "resultType": "vector", 22:17:55 DEBUG "resultType": "vector", 22:17:55 DEBUG "resultType": "vector", 22:17:55 DEBUG "data": { 22:17:55 DEBUG "resultType": "vector", 22:17:55 DEBUG "result": [ 22:17:55 DEBUG "result": [ 22:17:55 DEBUG "result": [ 22:17:55 DEBUG "result": [ 22:17:55 DEBUG "status": "success", 22:17:55 DEBUG "result": [ 22:17:55 DEBUG "result": [ 22:17:55 DEBUG "resultType": "vector", 22:17:55 DEBUG "result": [ 22:17:55 DEBUG "result": [ 22:17:55 DEBUG "data": { 22:17:55 DEBUG "resultType": "vector", 22:17:55 DEBUG { 22:17:55 DEBUG "result": [ 22:17:55 DEBUG "result": [ 22:17:55 DEBUG "result": [ 22:17:55 DEBUG "result": [ 22:17:55 DEBUG "result": [ 22:17:55 DEBUG "result": [ 22:17:55 DEBUG { 22:17:55 DEBUG "result": [ 22:17:55 DEBUG "result": [ 22:17:55 DEBUG "result": [ 22:17:55 DEBUG "resultType": "vector", 22:17:55 DEBUG "result": [ 22:17:55 DEBUG { 22:17:55 DEBUG { 22:17:55 DEBUG { 22:17:55 DEBUG { 22:17:55 DEBUG "data": { 22:17:55 DEBUG { 22:17:55 DEBUG { 22:17:55 DEBUG "result": [ 22:17:55 DEBUG { 22:17:55 DEBUG { 22:17:55 DEBUG "resultType": "vector", 22:17:55 DEBUG "result": [ 22:17:55 DEBUG "metric": { 22:17:55 DEBUG { 22:17:55 DEBUG { 22:17:55 DEBUG { 22:17:55 DEBUG { 22:17:55 DEBUG { 22:17:55 DEBUG { 22:17:55 DEBUG "metric": { 22:17:55 DEBUG { 22:17:55 DEBUG { 22:17:55 DEBUG { 22:17:55 DEBUG "result": [ 22:17:55 DEBUG { 22:17:55 DEBUG "metric": { 22:17:55 DEBUG "metric": { 22:17:55 DEBUG "metric": { 22:17:55 DEBUG "metric": { 22:17:55 DEBUG "resultType": "vector", 22:17:55 DEBUG "metric": { 22:17:55 DEBUG "metric": { 22:17:55 DEBUG { 22:17:55 DEBUG "metric": { 22:17:55 DEBUG "metric": { 22:17:55 DEBUG "result": [ 22:17:55 DEBUG { 22:17:55 DEBUG "pod": "ds-idrepo-2" 22:17:55 DEBUG "metric": { 22:17:55 DEBUG "metric": { 22:17:55 DEBUG "metric": { 22:17:55 DEBUG "metric": { 22:17:55 DEBUG "metric": { 22:17:55 DEBUG "metric": { 22:17:55 DEBUG "pod": "ds-idrepo-2" 22:17:55 DEBUG "metric": { 22:17:55 DEBUG "metric": { 22:17:55 DEBUG "metric": { 22:17:55 DEBUG { 22:17:55 DEBUG "metric": { 22:17:55 DEBUG "pod": "am-555f4db595-fz8sn" 22:17:55 DEBUG "pod": "am-555f4db595-fz8sn" 22:17:55 DEBUG "pod": "am-555f4db595-fz8sn" 22:17:55 DEBUG "pod": "ds-idrepo-2" 22:17:55 DEBUG "result": [ 22:17:55 DEBUG "pod": "am-555f4db595-fz8sn" 22:17:55 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:17:55 DEBUG "metric": { 22:17:55 DEBUG "pod": "am-555f4db595-fz8sn" 22:17:55 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:17:55 DEBUG { 22:17:55 DEBUG "metric": { 22:17:55 DEBUG }, 22:17:55 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:17:55 DEBUG "pod": "ds-idrepo-2" 22:17:55 DEBUG "pod": "ds-idrepo-2" 22:17:55 DEBUG "instance": "10.142.0.107:9100" 22:17:55 DEBUG "pod": "ds-idrepo-2" 22:17:55 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:17:55 DEBUG }, 22:17:55 DEBUG "instance": "10.142.0.107:9100" 22:17:55 DEBUG }, 22:17:55 DEBUG "pod": "am-555f4db595-fz8sn" 22:17:55 DEBUG "metric": { 22:17:55 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:17:55 DEBUG }, 22:17:55 DEBUG }, 22:17:55 DEBUG }, 22:17:55 DEBUG }, 22:17:55 DEBUG { 22:17:55 DEBUG }, 22:17:55 DEBUG }, 22:17:55 DEBUG "pod": "am-555f4db595-fz8sn" 22:17:55 DEBUG }, 22:17:55 DEBUG }, 22:17:55 DEBUG "metric": { 22:17:55 DEBUG "instance": "10.142.0.107:9100" 22:17:55 DEBUG "value": [ 22:17:55 DEBUG }, 22:17:55 DEBUG }, 22:17:55 DEBUG }, 22:17:55 DEBUG }, 22:17:55 DEBUG }, 22:17:55 DEBUG }, 22:17:55 DEBUG "value": [ 22:17:55 DEBUG "pod": "ds-idrepo-2" 22:17:55 DEBUG "value": [ 22:17:55 DEBUG }, 22:17:55 DEBUG "instance": "10.142.0.107:9100" 22:17:55 DEBUG }, 22:17:55 DEBUG "value": [ 22:17:55 DEBUG "value": [ 22:17:55 DEBUG "value": [ 22:17:55 DEBUG "value": [ 22:17:55 DEBUG "metric": { 22:17:55 DEBUG "value": [ 22:17:55 DEBUG "value": [ 22:17:55 DEBUG }, 22:17:55 DEBUG "value": [ 22:17:55 DEBUG "value": [ 22:17:55 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:17:55 DEBUG }, 22:17:55 DEBUG 22:17:55 DEBUG "value": [ 22:17:55 DEBUG "value": [ 22:17:55 DEBUG "value": [ 22:17:55 DEBUG "value": [ 22:17:55 DEBUG "value": [ 22:17:55 DEBUG "value": [ 22:17:55 DEBUG 22:17:55 DEBUG }, 22:17:55 DEBUG 22:17:55 DEBUG "value": [ 22:17:55 DEBUG }, 22:17:55 DEBUG "value": [ 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG "instance": "10.142.0.107:9100" 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG "value": [ 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG }, 22:17:55 DEBUG "value": [ 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG "value": [ 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG "value": [ 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG }, 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG "value": [ 22:17:55 DEBUG 22:17:55 DEBUG [...] 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG "value": [ 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG [...] 22:17:55 DEBUG 22:17:55 DEBUG [...] 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG [...] 22:17:55 DEBUG [...] 22:17:55 DEBUG [...] 22:17:55 DEBUG [...] 22:17:55 DEBUG 22:17:55 DEBUG [...] 22:17:55 DEBUG [...] 22:17:55 DEBUG 22:17:55 DEBUG [...] 22:17:55 DEBUG [...] 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG [...] 22:17:55 DEBUG [...] 22:17:55 DEBUG [...] 22:17:55 DEBUG [...] 22:17:55 DEBUG [...] 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG [...] 22:17:55 DEBUG 22:17:55 DEBUG [...] 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG [...] 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG [...] 22:17:55 DEBUG 22:17:55 DEBUG [...] 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG [...] 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG [...] 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG [...] 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG [...] 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG "pod": "ds-idrepo-0" 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG "pod": "ds-idrepo-0" 22:17:55 DEBUG 22:17:55 DEBUG "instance": "10.142.0.43:9100" 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG 22:17:55 DEBUG "pod": "am-555f4db595-kzxsp" 22:17:55 DEBUG "pod": "am-555f4db595-kzxsp" 22:17:55 DEBUG }, 22:17:55 DEBUG "pod": "ds-idrepo-0" 22:17:55 DEBUG 22:17:55 DEBUG "pod": "am-555f4db595-kzxsp" 22:17:55 DEBUG "pod": "overseer-0-7c9b667974-9mr2s" 22:17:55 DEBUG 22:17:55 DEBUG "pod": "am-555f4db595-kzxsp" 22:17:55 DEBUG "pod": "overseer-0-7c9b667974-9mr2s" 22:17:55 DEBUG 22:17:55 DEBUG }, 22:17:55 DEBUG "instance": "10.142.0.43:9100" 22:17:55 DEBUG 22:17:55 DEBUG "pod": "ds-idrepo-0" 22:17:55 DEBUG "pod": "ds-idrepo-0" 22:17:55 DEBUG }, 22:17:55 DEBUG "pod": "ds-idrepo-0" 22:17:55 DEBUG "pod": "overseer-0-7c9b667974-9mr2s" 22:17:55 DEBUG }, 22:17:55 DEBUG 22:17:55 DEBUG }, 22:17:55 DEBUG "pod": "am-555f4db595-kzxsp" 22:17:55 DEBUG 22:17:55 DEBUG "pod": "overseer-0-7c9b667974-9mr2s" 22:17:55 DEBUG }, 22:17:55 DEBUG "value": [ 22:17:55 DEBUG "pod": "am-555f4db595-kzxsp" 22:17:55 DEBUG }, 22:17:55 DEBUG 22:17:55 DEBUG }, 22:17:55 DEBUG }, 22:17:55 DEBUG "pod": "am-555f4db595-kzxsp" 22:17:55 DEBUG }, 22:17:55 DEBUG }, 22:17:55 DEBUG 22:17:55 DEBUG "value": [ 22:17:55 DEBUG }, 22:17:55 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:17:55 DEBUG }, 22:17:55 DEBUG "instance": "10.142.0.43:9100" 22:17:55 DEBUG "value": [ 22:17:55 DEBUG }, 22:17:55 DEBUG }, 22:17:55 DEBUG "value": [ 22:17:55 DEBUG "pod": "ds-cts-2" 22:17:55 DEBUG "value": [ 22:17:55 DEBUG }, 22:17:55 DEBUG "instance": "10.142.0.43:9100" 22:17:55 DEBUG }, 22:17:55 DEBUG "value": [ 22:17:55 DEBUG 1713302275, 22:17:55 DEBUG }, 22:17:55 DEBUG "value": [ 22:17:55 DEBUG 22:17:55 DEBUG "value": [ 22:17:55 DEBUG "value": [ 22:17:55 DEBUG }, 22:17:55 DEBUG "value": [ 22:17:55 DEBUG "value": [ 22:17:55 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:17:55 DEBUG 1713302275, 22:17:55 DEBUG "value": [ 22:17:55 DEBUG }, 22:17:55 DEBUG "value": [ 22:17:55 DEBUG }, 22:17:55 DEBUG 1713302275, 22:17:55 DEBUG "value": [ 22:17:55 DEBUG "value": [ 22:17:55 DEBUG 1713302275, 22:17:55 DEBUG }, 22:17:55 DEBUG 1713302275, 22:17:55 DEBUG "value": [ 22:17:55 DEBUG }, 22:17:55 DEBUG "value": [ 22:17:55 DEBUG 1713302275, 22:17:55 DEBUG "2005.2333333333333" 22:17:55 DEBUG "value": [ 22:17:55 DEBUG 1713302275, 22:17:55 DEBUG "instance": "10.142.0.43:9100" 22:17:55 DEBUG 1713302275, 22:17:55 DEBUG 1713302275, 22:17:55 DEBUG "value": [ 22:17:55 DEBUG 1713302275, 22:17:55 DEBUG 1713302275, 22:17:55 DEBUG }, 22:17:55 DEBUG "value": [ 22:17:55 DEBUG 1713302275, 22:17:55 DEBUG "value": [ 22:17:55 DEBUG 1713302275, 22:17:55 DEBUG "value": [ 22:17:55 DEBUG "0" 22:17:55 DEBUG 1713302275, 22:17:55 DEBUG 1713302275, 22:17:55 DEBUG "0" 22:17:55 DEBUG "value": [ 22:17:55 DEBUG "0.0004999999999995453" 22:17:55 DEBUG 1713302275, 22:17:55 DEBUG "value": [ 22:17:55 DEBUG 1713302275, 22:17:55 DEBUG "0" 22:17:55 DEBUG ] 22:17:55 DEBUG 1713302275, 22:17:55 DEBUG "0" 22:17:55 DEBUG }, 22:17:55 DEBUG "0" 22:17:55 DEBUG "34880.778588807785" 22:17:55 DEBUG 1713302275, 22:17:55 DEBUG "0.023" 22:17:55 DEBUG "6.28178663695942" 22:17:55 DEBUG 1713302275, 22:17:55 DEBUG "45517908.4" 22:17:55 DEBUG "0" 22:17:55 DEBUG 1713302275, 22:17:55 DEBUG "0" 22:17:55 DEBUG 1713302275, 22:17:55 DEBUG ] 22:17:55 DEBUG "0" 22:17:55 DEBUG "0" 22:17:55 DEBUG ] 22:17:55 DEBUG } 22:17:55 DEBUG ] 22:17:55 DEBUG "680278" 22:17:55 DEBUG 1713302275, 22:17:55 DEBUG "0" 22:17:55 DEBUG ] 22:17:55 DEBUG } 22:17:55 DEBUG "2005.3333333333333" 22:17:55 DEBUG ] 22:17:55 DEBUG "value": [ 22:17:55 DEBUG ] 22:17:55 DEBUG ] 22:17:55 DEBUG "2004.8999999999999" 22:17:55 DEBUG ] 22:17:55 DEBUG ] 22:17:55 DEBUG "6.296376657622266" 22:17:55 DEBUG ] 22:17:55 DEBUG ] 22:17:55 DEBUG } 22:17:55 DEBUG ] 22:17:55 DEBUG "19949544448" 22:17:55 DEBUG } 22:17:55 DEBUG ] 22:17:55 DEBUG ] 22:17:55 DEBUG 1713302275, 22:17:55 DEBUG ] 22:17:55 DEBUG } 22:17:55 DEBUG ] 22:17:55 DEBUG "45526810.133333325" 22:17:55 DEBUG ] 22:17:55 DEBUG } 22:17:55 DEBUG ] 22:17:55 DEBUG ] 22:17:55 DEBUG } 22:17:55 DEBUG 1713302275, 22:17:55 DEBUG } 22:17:55 DEBUG } 22:17:55 DEBUG ] 22:17:55 DEBUG } 22:17:55 DEBUG } 22:17:55 DEBUG ] 22:17:55 DEBUG } 22:17:55 DEBUG "636518400" 22:17:55 DEBUG ] 22:17:55 DEBUG } 22:17:55 DEBUG ] 22:17:55 DEBUG ] 22:17:55 DEBUG } 22:17:55 DEBUG } 22:17:55 DEBUG "0" 22:17:55 DEBUG } 22:17:55 DEBUG ] 22:17:55 DEBUG } 22:17:55 DEBUG ] 22:17:55 DEBUG } 22:17:55 DEBUG ] 22:17:55 DEBUG } 22:17:55 DEBUG } 22:17:55 DEBUG ] 22:17:55 DEBUG "0" 22:17:55 DEBUG ] 22:17:55 DEBUG ] 22:17:55 DEBUG } 22:17:55 DEBUG ] 22:17:55 DEBUG ] 22:17:55 DEBUG } 22:17:55 DEBUG ] 22:17:55 DEBUG ] 22:17:55 DEBUG } 22:17:55 DEBUG ] 22:17:55 DEBUG } 22:17:55 DEBUG } 22:17:55 DEBUG ] 22:17:55 DEBUG ] 22:17:55 DEBUG ] 22:17:55 DEBUG } 22:17:55 DEBUG } 22:17:55 DEBUG ] 22:17:55 DEBUG } 22:17:55 DEBUG ] 22:17:55 DEBUG } 22:17:55 DEBUG } 22:17:55 DEBUG ] 22:17:55 DEBUG } 22:17:55 DEBUG ] 22:17:55 DEBUG } 22:17:55 DEBUG } 22:17:55 DEBUG ] 22:17:55 DEBUG } 22:17:55 DEBUG } 22:17:55 DEBUG ] 22:17:55 DEBUG } 22:17:55 DEBUG } 22:17:55 DEBUG } 22:17:55 DEBUG } 22:17:55 DEBUG ] 22:17:55 DEBUG } 22:17:55 DEBUG } 22:17:55 DEBUG } 22:17:55 DEBUG } 22:17:55 DEBUG } 22:17:55 DEBUG } 22:17:55 DEBUG ] 22:17:55 DEBUG } 22:17:55 DEBUG } 22:17:55 DEBUG } 22:17:55 DEBUG } 22:17:55 DEBUG } 22:17:55 DEBUG } 22:17:55 DEBUG } 22:17:55 DEBUG } 22:17:55 DEBUG } 22:17:55 DEBUG } 22:17:55 DEBUG } 22:17:55 DEBUG } 22:17:55 DEBUG ] 22:17:55 DEBUG } 22:17:55 DEBUG } 22:17:55 DEBUG } 22:17:55 DEBUG ] 22:17:55 DEBUG } 22:17:55 DEBUG } 22:17:55 DEBUG } 22:17:55 DEBUG } 22:17:55 DEBUG ] 22:17:55 DEBUG } 22:17:55 DEBUG } 22:17:55 DEBUG } 22:17:55 DEBUG } 22:17:55 DEBUG } 22:17:55 DEBUG } 22:17:55 DEBUG } 22:17:55 DEBUG } 22:17:55 DEBUG } 22:17:55 DEBUG } 22:17:55 DEBUG } 22:18:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302334" 22:18:54 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713302334" 22:18:54 INFO [http_cmd]: http status code OK 22:18:54 ERROR [http_cmd]: ERROR 22:18:54 DEBUG --- status code --- 22:18:54 ERROR --- status code --- 22:18:54 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:18:54 ERROR http status code is 400 (expected [200, 401, 403, 404]) 22:18:54 DEBUG --- http response --- 22:18:54 ERROR --- http response --- 22:18:54 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 22:18:54 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 22:18:54 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 22:18:55 INFO 22:18:55 INFO [loop_until]: kubectl --namespace=xlou top pods 22:18:55 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:18:55 INFO [loop_until]: OK (rc = 0) 22:18:55 DEBUG --- stdout --- 22:18:55 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 9Mi am-555f4db595-4lbss 9360m 8581Mi am-555f4db595-fz8sn 9055m 8589Mi am-555f4db595-kzxsp 9448m 8548Mi ds-cts-0 6136m 12106Mi ds-cts-1 5571m 12136Mi ds-cts-2 5522m 12097Mi ds-idrepo-0 4800m 5032Mi ds-idrepo-1 4370m 2833Mi ds-idrepo-2 4459m 3072Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 4m 1207Mi idm-b85f9d978-rl76f 3m 1226Mi lodemon-bc756c647-cr7x6 9m 63Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-7c9b667974-9mr2s 6200m 516Mi 22:18:55 DEBUG --- stderr --- 22:18:55 DEBUG 22:18:55 INFO 22:18:55 INFO [loop_until]: kubectl --namespace=xlou top node 22:18:55 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:18:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302335" 22:18:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302335" 22:18:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302335" 22:18:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302335" 22:18:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302335" 22:18:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302335" 22:18:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302335" 22:18:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302335" 22:18:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302335" 22:18:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302335" 22:18:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302335" 22:18:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713302335" 22:18:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302335" 22:18:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302335" 22:18:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302335" 22:18:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302335" 22:18:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713302335" 22:18:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302335" 22:18:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713302335" 22:18:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713302335" 22:18:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302335" 22:18:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302335" 22:18:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302335" 22:18:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302335" 22:18:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302335" 22:18:55 INFO [http_cmd]: http status code OK 22:18:55 INFO [http_cmd]: http status code OK 22:18:55 INFO [http_cmd]: http status code OK 22:18:55 INFO [loop_until]: OK (rc = 0) 22:18:55 INFO [http_cmd]: http status code OK 22:18:55 INFO [http_cmd]: http status code OK 22:18:55 INFO [http_cmd]: http status code OK 22:18:55 INFO [http_cmd]: http status code OK 22:18:55 INFO [http_cmd]: http status code OK 22:18:55 INFO [http_cmd]: http status code OK 22:18:55 INFO [http_cmd]: http status code OK 22:18:55 INFO [http_cmd]: http status code OK 22:18:55 INFO [http_cmd]: http status code OK 22:18:55 INFO [http_cmd]: http status code OK 22:18:55 INFO [http_cmd]: http status code OK 22:18:55 INFO [http_cmd]: http status code OK 22:18:55 INFO [http_cmd]: http status code OK 22:18:55 DEBUG --- status code --- 22:18:55 INFO [http_cmd]: http status code OK 22:18:55 DEBUG --- status code --- 22:18:55 DEBUG --- status code --- 22:18:55 INFO [http_cmd]: http status code OK 22:18:55 DEBUG --- stdout --- 22:18:55 DEBUG --- status code --- 22:18:55 INFO [http_cmd]: http status code OK 22:18:55 DEBUG --- status code --- 22:18:55 INFO [http_cmd]: http status code OK 22:18:55 INFO [http_cmd]: http status code OK 22:18:55 DEBUG --- status code --- 22:18:55 DEBUG --- status code --- 22:18:55 DEBUG --- status code --- 22:18:55 DEBUG --- status code --- 22:18:55 DEBUG --- status code --- 22:18:55 DEBUG --- status code --- 22:18:55 DEBUG --- status code --- 22:18:55 DEBUG --- status code --- 22:18:55 DEBUG --- status code --- 22:18:55 DEBUG --- status code --- 22:18:55 DEBUG --- status code --- 22:18:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:18:55 DEBUG --- status code --- 22:18:55 INFO [http_cmd]: http status code OK 22:18:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:18:55 INFO [http_cmd]: http status code OK 22:18:55 INFO [http_cmd]: http status code OK 22:18:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:18:55 DEBUG --- status code --- 22:18:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:18:55 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 63m 0% 5079Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5674m 19% 14712Mi 13% gke-xlou-cdm-default-pool-1b0769c8-nhlf 9380m 31% 10817Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 84m 0% 2643Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 4745m 15% 6408Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 6797m 22% 16658Mi 14% gke-xlou-cdm-default-pool-1b0769c8-xnmp 63m 0% 3148Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 9315m 31% 10353Mi 9% gke-xlou-cdm-ds-0f601751-hmm7 4547m 15% 5572Mi 4% gke-xlou-cdm-ds-0f601751-j488 5591m 18% 14052Mi 12% gke-xlou-cdm-ds-0f601751-mc5d 10716m 35% 3346Mi 2% gke-xlou-cdm-ds-0f601751-nj6q 9567m 32% 10419Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 4697m 15% 7359Mi 6% gke-xlou-cdm-frontend-a8771548-i82n 6460m 40% 5948Mi 10% 22:18:55 DEBUG --- status code --- 22:18:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:18:55 DEBUG --- status code --- 22:18:55 DEBUG --- status code --- 22:18:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:18:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:18:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:18:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:18:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:18:55 DEBUG --- http response --- 22:18:55 INFO [http_cmd]: http status code OK 22:18:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:18:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:18:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:18:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:18:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:18:55 DEBUG --- http response --- 22:18:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:18:55 DEBUG --- status code --- 22:18:55 DEBUG --- http response --- 22:18:55 DEBUG --- status code --- 22:18:55 DEBUG --- status code --- 22:18:55 DEBUG --- http response --- 22:18:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:18:55 DEBUG --- http response --- 22:18:55 DEBUG --- stderr --- 22:18:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:18:55 DEBUG --- http response --- 22:18:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:18:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:18:55 DEBUG --- http response --- 22:18:55 DEBUG --- http response --- 22:18:55 DEBUG --- http response --- 22:18:55 DEBUG --- http response --- 22:18:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:18:55 DEBUG --- status code --- 22:18:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:18:55 DEBUG --- http response --- 22:18:55 DEBUG --- http response --- 22:18:55 DEBUG --- http response --- 22:18:55 DEBUG --- http response --- 22:18:55 DEBUG --- http response --- 22:18:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:18:55 DEBUG --- http response --- 22:18:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:18:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:18:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:18:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:18:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:18:55 DEBUG --- http response --- 22:18:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:18:55 DEBUG 22:18:55 DEBUG --- http response --- 22:18:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:18:55 DEBUG --- http response --- 22:18:55 DEBUG --- http response --- 22:18:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:18:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:18:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:18:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:18:55 DEBUG --- http response --- 22:18:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:18:55 DEBUG ----- output ----- 22:18:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:18:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:18:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:18:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:18:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:18:55 DEBUG ----- output ----- 22:18:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:18:55 DEBUG --- http response --- 22:18:55 DEBUG ----- output ----- 22:18:55 DEBUG --- http response --- 22:18:55 DEBUG --- http response --- 22:18:55 DEBUG ----- output ----- 22:18:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:18:55 DEBUG ----- output ----- 22:18:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:18:55 DEBUG ----- output ----- 22:18:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:18:55 DEBUG ----- output ----- 22:18:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:18:55 DEBUG ----- output ----- 22:18:55 DEBUG ----- output ----- 22:18:55 DEBUG ----- output ----- 22:18:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:18:55 DEBUG --- http response --- 22:18:55 DEBUG { 22:18:55 DEBUG ----- output ----- 22:18:55 DEBUG ----- output ----- 22:18:55 DEBUG ----- output ----- 22:18:55 DEBUG ----- output ----- 22:18:55 DEBUG ----- output ----- 22:18:55 DEBUG { 22:18:55 DEBUG ----- output ----- 22:18:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:18:55 DEBUG { 22:18:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:18:55 DEBUG { 22:18:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:18:55 DEBUG ----- output ----- 22:18:55 DEBUG { 22:18:55 DEBUG ----- output ----- 22:18:55 DEBUG { 22:18:55 DEBUG "status": "success", 22:18:55 DEBUG { 22:18:55 DEBUG ----- output ----- 22:18:55 DEBUG { 22:18:55 DEBUG { 22:18:55 DEBUG { 22:18:55 DEBUG ----- output ----- 22:18:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:18:55 DEBUG "status": "success", 22:18:55 DEBUG { 22:18:55 DEBUG { 22:18:55 DEBUG { 22:18:55 DEBUG { 22:18:55 DEBUG { 22:18:55 DEBUG "status": "success", 22:18:55 DEBUG { 22:18:55 DEBUG ----- output ----- 22:18:55 DEBUG "status": "success", 22:18:55 DEBUG ----- output ----- 22:18:55 DEBUG "status": "success", 22:18:55 DEBUG ----- output ----- 22:18:55 DEBUG { 22:18:55 DEBUG "status": "success", 22:18:55 DEBUG { 22:18:55 DEBUG ----- output ----- 22:18:55 DEBUG "data": { 22:18:55 DEBUG "status": "success", 22:18:55 DEBUG { 22:18:55 DEBUG "status": "success", 22:18:55 DEBUG "status": "success", 22:18:55 DEBUG "status": "success", 22:18:55 DEBUG { 22:18:55 DEBUG ----- output ----- 22:18:55 DEBUG "data": { 22:18:55 DEBUG "status": "success", 22:18:55 DEBUG "status": "success", 22:18:55 DEBUG "data": { 22:18:55 DEBUG "status": "success", 22:18:55 DEBUG "status": "success", 22:18:55 DEBUG "data": { 22:18:55 DEBUG "status": "success", 22:18:55 DEBUG { 22:18:55 DEBUG "data": { 22:18:55 DEBUG { 22:18:55 DEBUG "data": { 22:18:55 DEBUG { 22:18:55 DEBUG "status": "success", 22:18:55 DEBUG "data": { 22:18:55 DEBUG "status": "success", 22:18:55 DEBUG { 22:18:55 DEBUG "resultType": "vector", 22:18:55 DEBUG "result": [ 22:18:55 DEBUG "status": "success", 22:18:55 DEBUG "data": { 22:18:55 DEBUG "data": { 22:18:55 DEBUG "data": { 22:18:55 DEBUG "status": "success", 22:18:55 DEBUG { 22:18:55 DEBUG "resultType": "vector", 22:18:55 DEBUG "data": { 22:18:55 DEBUG "status": "success", 22:18:55 DEBUG "resultType": "vector", 22:18:55 DEBUG "data": { 22:18:55 DEBUG "data": { 22:18:55 DEBUG "resultType": "vector", 22:18:55 DEBUG "data": { 22:18:55 DEBUG "status": "success", 22:18:55 DEBUG "resultType": "vector", 22:18:55 DEBUG "status": "success", 22:18:55 DEBUG "resultType": "vector", 22:18:55 DEBUG "status": "success", 22:18:55 DEBUG "data": { 22:18:55 DEBUG "resultType": "vector", 22:18:55 DEBUG "data": { 22:18:55 DEBUG "status": "success", 22:18:55 DEBUG { 22:18:55 DEBUG "data": { 22:18:55 DEBUG "data": { 22:18:55 DEBUG "resultType": "vector", 22:18:55 DEBUG "resultType": "vector", 22:18:55 DEBUG "resultType": "vector", 22:18:55 DEBUG "data": { 22:18:55 DEBUG "status": "success", 22:18:55 DEBUG "result": [ 22:18:55 DEBUG "resultType": "vector", 22:18:55 DEBUG "data": { 22:18:55 DEBUG "result": [ 22:18:55 DEBUG "resultType": "vector", 22:18:55 DEBUG "resultType": "vector", 22:18:55 DEBUG "result": [ 22:18:55 DEBUG "resultType": "vector", 22:18:55 DEBUG "data": { 22:18:55 DEBUG "result": [ 22:18:55 DEBUG "data": { 22:18:55 DEBUG "result": [ 22:18:55 DEBUG "data": { 22:18:55 DEBUG "resultType": "vector", 22:18:55 DEBUG "result": [ 22:18:55 DEBUG "resultType": "vector", 22:18:55 DEBUG "data": { 22:18:55 DEBUG "metric": { 22:18:55 DEBUG "resultType": "vector", 22:18:55 DEBUG "resultType": "vector", 22:18:55 DEBUG "result": [ 22:18:55 DEBUG "result": [ 22:18:55 DEBUG "result": [ 22:18:55 DEBUG "resultType": "vector", 22:18:55 DEBUG "data": { 22:18:55 DEBUG { 22:18:55 DEBUG "result": [ 22:18:55 DEBUG "resultType": "vector", 22:18:55 DEBUG { 22:18:55 DEBUG "result": [ 22:18:55 DEBUG "result": [ 22:18:55 DEBUG { 22:18:55 DEBUG "result": [ 22:18:55 DEBUG "resultType": "vector", 22:18:55 DEBUG { 22:18:55 DEBUG "resultType": "vector", 22:18:55 DEBUG { 22:18:55 DEBUG "resultType": "vector", 22:18:55 DEBUG "result": [ 22:18:55 DEBUG { 22:18:55 DEBUG "result": [ 22:18:55 DEBUG "resultType": "vector", 22:18:55 DEBUG "pod": "am-555f4db595-fz8sn" 22:18:55 DEBUG "result": [ 22:18:55 DEBUG "result": [ 22:18:55 DEBUG { 22:18:55 DEBUG { 22:18:55 DEBUG { 22:18:55 DEBUG "result": [ 22:18:55 DEBUG "resultType": "vector", 22:18:55 DEBUG "metric": { 22:18:55 DEBUG { 22:18:55 DEBUG "result": [ 22:18:55 DEBUG "metric": { 22:18:55 DEBUG { 22:18:55 DEBUG { 22:18:55 DEBUG "metric": { 22:18:55 DEBUG { 22:18:55 DEBUG "result": [ 22:18:55 DEBUG "metric": { 22:18:55 DEBUG "result": [ 22:18:55 DEBUG "metric": { 22:18:55 DEBUG "result": [ 22:18:55 DEBUG { 22:18:55 DEBUG "metric": { 22:18:55 DEBUG { 22:18:55 DEBUG "result": [ 22:18:55 DEBUG }, 22:18:55 DEBUG { 22:18:55 DEBUG { 22:18:55 DEBUG "metric": { 22:18:55 DEBUG "metric": { 22:18:55 DEBUG "metric": { 22:18:55 DEBUG { 22:18:55 DEBUG "result": [ 22:18:55 DEBUG "pod": "ds-idrepo-2" 22:18:55 DEBUG "metric": { 22:18:55 DEBUG { 22:18:55 DEBUG "instance": "10.142.0.107:9100" 22:18:55 DEBUG "metric": { 22:18:55 DEBUG "metric": { 22:18:55 DEBUG "pod": "ds-idrepo-2" 22:18:55 DEBUG "metric": { 22:18:55 DEBUG { 22:18:55 DEBUG "pod": "ds-idrepo-2" 22:18:55 DEBUG { 22:18:55 DEBUG "pod": "am-555f4db595-fz8sn" 22:18:55 DEBUG { 22:18:55 DEBUG "metric": { 22:18:55 DEBUG "pod": "ds-idrepo-2" 22:18:55 DEBUG "metric": { 22:18:55 DEBUG { 22:18:55 DEBUG "value": [ 22:18:55 DEBUG "metric": { 22:18:55 DEBUG "metric": { 22:18:55 DEBUG "pod": "am-555f4db595-fz8sn" 22:18:55 DEBUG "instance": "10.142.0.107:9100" 22:18:55 DEBUG "pod": "ds-idrepo-2" 22:18:55 DEBUG "metric": { 22:18:55 DEBUG { 22:18:55 DEBUG }, 22:18:55 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:18:55 DEBUG "metric": { 22:18:55 DEBUG }, 22:18:55 DEBUG "pod": "am-555f4db595-fz8sn" 22:18:55 DEBUG "pod": "am-555f4db595-fz8sn" 22:18:55 DEBUG }, 22:18:55 DEBUG "pod": "ds-idrepo-2" 22:18:55 DEBUG "metric": { 22:18:55 DEBUG }, 22:18:55 DEBUG "metric": { 22:18:55 DEBUG }, 22:18:55 DEBUG "metric": { 22:18:55 DEBUG "pod": "ds-idrepo-2" 22:18:55 DEBUG }, 22:18:55 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:18:55 DEBUG "metric": { 22:18:55 DEBUG 22:18:55 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:18:55 DEBUG "instance": "10.142.0.107:9100" 22:18:55 DEBUG }, 22:18:55 DEBUG }, 22:18:55 DEBUG }, 22:18:55 DEBUG "pod": "am-555f4db595-fz8sn" 22:18:55 DEBUG "metric": { 22:18:55 DEBUG "value": [ 22:18:55 DEBUG }, 22:18:55 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:18:55 DEBUG "value": [ 22:18:55 DEBUG }, 22:18:55 DEBUG }, 22:18:55 DEBUG "value": [ 22:18:55 DEBUG }, 22:18:55 DEBUG "pod": "am-555f4db595-fz8sn" 22:18:55 DEBUG "value": [ 22:18:55 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:18:55 DEBUG "value": [ 22:18:55 DEBUG "instance": "10.142.0.107:9100" 22:18:55 DEBUG }, 22:18:55 DEBUG "value": [ 22:18:55 DEBUG }, 22:18:55 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:18:55 DEBUG 22:18:55 DEBUG }, 22:18:55 DEBUG }, 22:18:55 DEBUG "value": [ 22:18:55 DEBUG "value": [ 22:18:55 DEBUG "value": [ 22:18:55 DEBUG }, 22:18:55 DEBUG "instance": "10.142.0.107:9100" 22:18:55 DEBUG 22:18:55 DEBUG "value": [ 22:18:55 DEBUG }, 22:18:55 DEBUG 22:18:55 DEBUG "value": [ 22:18:55 DEBUG "value": [ 22:18:55 DEBUG 22:18:55 DEBUG "value": [ 22:18:55 DEBUG }, 22:18:55 DEBUG 22:18:55 DEBUG }, 22:18:55 DEBUG 22:18:55 DEBUG }, 22:18:55 DEBUG "value": [ 22:18:55 DEBUG 22:18:55 DEBUG "value": [ 22:18:55 DEBUG }, 22:18:55 DEBUG 22:18:55 DEBUG "value": [ 22:18:55 DEBUG "value": [ 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG "value": [ 22:18:55 DEBUG }, 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG "value": [ 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG "value": [ 22:18:55 DEBUG 22:18:55 DEBUG "value": [ 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG "value": [ 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG "value": [ 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG "value": [ 22:18:55 DEBUG [...] 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG [...] 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG [...] 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG [...] 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG [...] 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG [...] 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG [...] 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG [...] 22:18:55 DEBUG [...] 22:18:55 DEBUG [...] 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG [...] 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG [...] 22:18:55 DEBUG [...] 22:18:55 DEBUG 22:18:55 DEBUG [...] 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG [...] 22:18:55 DEBUG 22:18:55 DEBUG [...] 22:18:55 DEBUG [...] 22:18:55 DEBUG 22:18:55 DEBUG [...] 22:18:55 DEBUG [...] 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG [...] 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG [...] 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG [...] 22:18:55 DEBUG 22:18:55 DEBUG [...] 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG [...] 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG [...] 22:18:55 DEBUG "pod": "am-555f4db595-kzxsp" 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG "pod": "am-555f4db595-kzxsp" 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG "instance": "10.142.0.43:9100" 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG "pod": "ds-idrepo-0" 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG "instance": "10.142.0.43:9100" 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG "pod": "ds-idrepo-0" 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG "pod": "ds-idrepo-0" 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG }, 22:18:55 DEBUG 22:18:55 DEBUG "pod": "ds-idrepo-0" 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG }, 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG "pod": "am-555f4db595-kzxsp" 22:18:55 DEBUG }, 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG }, 22:18:55 DEBUG "pod": "overseer-0-7c9b667974-9mr2s" 22:18:55 DEBUG 22:18:55 DEBUG }, 22:18:55 DEBUG "pod": "am-555f4db595-kzxsp" 22:18:55 DEBUG "pod": "am-555f4db595-kzxsp" 22:18:55 DEBUG }, 22:18:55 DEBUG "pod": "ds-idrepo-0" 22:18:55 DEBUG 22:18:55 DEBUG }, 22:18:55 DEBUG 22:18:55 DEBUG 22:18:55 DEBUG "value": [ 22:18:55 DEBUG "pod": "ds-cts-2" 22:18:55 DEBUG }, 22:18:55 DEBUG "pod": "overseer-0-7c9b667974-9mr2s" 22:18:55 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:18:55 DEBUG "value": [ 22:18:55 DEBUG "pod": "overseer-0-7c9b667974-9mr2s" 22:18:55 DEBUG "instance": "10.142.0.43:9100" 22:18:55 DEBUG }, 22:18:55 DEBUG "value": [ 22:18:55 DEBUG "pod": "ds-idrepo-0" 22:18:55 DEBUG "pod": "am-555f4db595-kzxsp" 22:18:55 DEBUG 22:18:55 DEBUG "value": [ 22:18:55 DEBUG }, 22:18:55 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:18:55 DEBUG "value": [ 22:18:55 DEBUG }, 22:18:55 DEBUG }, 22:18:55 DEBUG "value": [ 22:18:55 DEBUG }, 22:18:55 DEBUG "pod": "am-555f4db595-kzxsp" 22:18:55 DEBUG "value": [ 22:18:55 DEBUG "pod": "overseer-0-7c9b667974-9mr2s" 22:18:55 DEBUG 22:18:55 DEBUG 1713302335, 22:18:55 DEBUG }, 22:18:55 DEBUG "value": [ 22:18:55 DEBUG }, 22:18:55 DEBUG }, 22:18:55 DEBUG 1713302335, 22:18:55 DEBUG }, 22:18:55 DEBUG }, 22:18:55 DEBUG "value": [ 22:18:55 DEBUG 1713302335, 22:18:55 DEBUG }, 22:18:55 DEBUG }, 22:18:55 DEBUG "instance": "10.142.0.43:9100" 22:18:55 DEBUG 1713302335, 22:18:55 DEBUG "value": [ 22:18:55 DEBUG }, 22:18:55 DEBUG 1713302335, 22:18:55 DEBUG "value": [ 22:18:55 DEBUG "value": [ 22:18:55 DEBUG 1713302335, 22:18:55 DEBUG "value": [ 22:18:55 DEBUG }, 22:18:55 DEBUG 1713302335, 22:18:55 DEBUG }, 22:18:55 DEBUG "instance": "10.142.0.43:9100" 22:18:55 DEBUG "2008.4333333333334" 22:18:55 DEBUG "value": [ 22:18:55 DEBUG 1713302335, 22:18:55 DEBUG "value": [ 22:18:55 DEBUG "value": [ 22:18:55 DEBUG "0" 22:18:55 DEBUG "value": [ 22:18:55 DEBUG "value": [ 22:18:55 DEBUG 1713302335, 22:18:55 DEBUG "0.0004999999999995453" 22:18:55 DEBUG "value": [ 22:18:55 DEBUG "value": [ 22:18:55 DEBUG }, 22:18:55 DEBUG "0" 22:18:55 DEBUG 1713302335, 22:18:55 DEBUG "value": [ 22:18:55 DEBUG "45599226.13333333" 22:18:55 DEBUG 1713302335, 22:18:55 DEBUG 1713302335, 22:18:55 DEBUG "0" 22:18:55 DEBUG 1713302335, 22:18:55 DEBUG "value": [ 22:18:55 DEBUG "0" 22:18:55 DEBUG "value": [ 22:18:55 DEBUG }, 22:18:55 DEBUG ] 22:18:55 DEBUG 1713302335, 22:18:55 DEBUG "0" 22:18:55 DEBUG 1713302335, 22:18:55 DEBUG 1713302335, 22:18:55 DEBUG ] 22:18:55 DEBUG 1713302335, 22:18:55 DEBUG 1713302335, 22:18:55 DEBUG "0" 22:18:55 DEBUG ] 22:18:55 DEBUG 1713302335, 22:18:55 DEBUG 1713302335, 22:18:55 DEBUG "value": [ 22:18:55 DEBUG ] 22:18:55 DEBUG "4.524320847046647" 22:18:55 DEBUG 1713302335, 22:18:55 DEBUG ] 22:18:55 DEBUG "2008.3" 22:18:55 DEBUG "2008.3999999999999" 22:18:55 DEBUG ] 22:18:55 DEBUG "0" 22:18:55 DEBUG 1713302335, 22:18:55 DEBUG ] 22:18:55 DEBUG 1713302335, 22:18:55 DEBUG "value": [ 22:18:55 DEBUG } 22:18:55 DEBUG "0" 22:18:55 DEBUG ] 22:18:55 DEBUG "0" 22:18:55 DEBUG "635740160" 22:18:55 DEBUG } 22:18:55 DEBUG "32369.69857025227" 22:18:55 DEBUG "45608652.633333325" 22:18:55 DEBUG ] 22:18:55 DEBUG } 22:18:55 DEBUG "0" 22:18:55 DEBUG "0.023" 22:18:55 DEBUG 1713302335, 22:18:55 DEBUG } 22:18:55 DEBUG ] 22:18:55 DEBUG "6.322550797661424" 22:18:55 DEBUG } 22:18:55 DEBUG ] 22:18:55 DEBUG ] 22:18:55 DEBUG } 22:18:55 DEBUG ] 22:18:55 DEBUG "800150" 22:18:55 DEBUG } 22:18:55 DEBUG "0" 22:18:55 DEBUG 1713302335, 22:18:55 DEBUG ] 22:18:55 DEBUG ] 22:18:55 DEBUG } 22:18:55 DEBUG ] 22:18:55 DEBUG ] 22:18:55 DEBUG ] 22:18:55 DEBUG ] 22:18:55 DEBUG ] 22:18:55 DEBUG } 22:18:55 DEBUG ] 22:18:55 DEBUG ] 22:18:55 DEBUG ] 22:18:55 DEBUG "19950441472" 22:18:55 DEBUG ] 22:18:55 DEBUG } 22:18:55 DEBUG ] 22:18:55 DEBUG ] 22:18:55 DEBUG } 22:18:55 DEBUG } 22:18:55 DEBUG ] 22:18:55 DEBUG } 22:18:55 DEBUG ] 22:18:55 DEBUG ] 22:18:55 DEBUG ] 22:18:55 DEBUG "0" 22:18:55 DEBUG } 22:18:55 DEBUG } 22:18:55 DEBUG ] 22:18:55 DEBUG } 22:18:55 DEBUG } 22:18:55 DEBUG ] 22:18:55 DEBUG } 22:18:55 DEBUG } 22:18:55 DEBUG ] 22:18:55 DEBUG } 22:18:55 DEBUG } 22:18:55 DEBUG } 22:18:55 DEBUG ] 22:18:55 DEBUG } 22:18:55 DEBUG } 22:18:55 DEBUG } 22:18:55 DEBUG } 22:18:55 DEBUG ] 22:18:55 DEBUG ] 22:18:55 DEBUG } 22:18:55 DEBUG ] 22:18:55 DEBUG } 22:18:55 DEBUG } 22:18:55 DEBUG } 22:18:55 DEBUG ] 22:18:55 DEBUG } 22:18:55 DEBUG ] 22:18:55 DEBUG } 22:18:55 DEBUG } 22:18:55 DEBUG } 22:18:55 DEBUG } 22:18:55 DEBUG } 22:18:55 DEBUG } 22:18:55 DEBUG } 22:18:55 DEBUG ] 22:18:55 DEBUG ] 22:18:55 DEBUG } 22:18:55 DEBUG ] 22:18:55 DEBUG ] 22:18:55 DEBUG } 22:18:55 DEBUG } 22:18:55 DEBUG } 22:18:55 DEBUG } 22:18:55 DEBUG } 22:18:55 DEBUG ] 22:18:55 DEBUG } 22:18:55 DEBUG ] 22:18:55 DEBUG } 22:18:55 DEBUG } 22:18:55 DEBUG ] 22:18:55 DEBUG } 22:18:55 DEBUG ] 22:18:55 DEBUG ] 22:18:55 DEBUG } 22:18:55 DEBUG } 22:18:55 DEBUG } 22:18:55 DEBUG ] 22:18:55 DEBUG } 22:18:55 DEBUG } 22:18:55 DEBUG } 22:18:55 DEBUG } 22:18:55 DEBUG } 22:18:55 DEBUG } 22:18:55 DEBUG } 22:18:55 DEBUG ] 22:18:55 DEBUG } 22:18:55 DEBUG } 22:18:55 DEBUG } 22:18:55 DEBUG } 22:18:55 DEBUG } 22:18:55 DEBUG } 22:18:55 DEBUG } 22:18:55 DEBUG } 22:18:55 DEBUG } 22:18:55 DEBUG } 22:18:55 DEBUG } 22:18:55 DEBUG } 22:18:55 DEBUG } 22:18:55 DEBUG } 22:18:55 DEBUG } 22:18:55 DEBUG } 22:18:55 DEBUG } 22:19:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713302395" 22:19:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302395" 22:19:55 ERROR [http_cmd]: ERROR 22:19:55 INFO [http_cmd]: http status code OK 22:19:55 ERROR --- status code --- 22:19:55 DEBUG --- status code --- 22:19:55 ERROR http status code is 400 (expected [200, 401, 403, 404]) 22:19:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:19:55 ERROR --- http response --- 22:19:55 DEBUG --- http response --- 22:19:55 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 22:19:55 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 22:19:55 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 22:19:55 INFO 22:19:55 INFO [loop_until]: kubectl --namespace=xlou top pods 22:19:55 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:19:55 INFO [loop_until]: OK (rc = 0) 22:19:55 DEBUG --- stdout --- 22:19:55 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 9Mi am-555f4db595-4lbss 9427m 8584Mi am-555f4db595-fz8sn 9513m 8594Mi am-555f4db595-kzxsp 9328m 8551Mi ds-cts-0 6149m 12113Mi ds-cts-1 5637m 12143Mi ds-cts-2 5497m 12106Mi ds-idrepo-0 4847m 5032Mi ds-idrepo-1 4362m 2834Mi ds-idrepo-2 4441m 3079Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 4m 1208Mi idm-b85f9d978-rl76f 5m 1225Mi lodemon-bc756c647-cr7x6 14m 63Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-7c9b667974-9mr2s 6151m 517Mi 22:19:55 DEBUG --- stderr --- 22:19:55 DEBUG 22:19:55 INFO 22:19:55 INFO [loop_until]: kubectl --namespace=xlou top node 22:19:55 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:19:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713302395" 22:19:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302395" 22:19:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302395" 22:19:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302395" 22:19:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302395" 22:19:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302395" 22:19:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302395" 22:19:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302395" 22:19:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302395" 22:19:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302395" 22:19:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302395" 22:19:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713302395" 22:19:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713302395" 22:19:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302395" 22:19:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302395" 22:19:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302395" 22:19:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713302395" 22:19:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302395" 22:19:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302395" 22:19:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302395" 22:19:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302395" 22:19:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302395" 22:19:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302395" 22:19:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302395" 22:19:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302395" 22:19:55 INFO [http_cmd]: http status code OK 22:19:55 INFO [http_cmd]: http status code OK 22:19:55 INFO [http_cmd]: http status code OK 22:19:55 INFO [http_cmd]: http status code OK 22:19:55 INFO [http_cmd]: http status code OK 22:19:55 INFO [http_cmd]: http status code OK 22:19:55 INFO [http_cmd]: http status code OK 22:19:55 INFO [http_cmd]: http status code OK 22:19:55 INFO [http_cmd]: http status code OK 22:19:55 INFO [http_cmd]: http status code OK 22:19:55 INFO [http_cmd]: http status code OK 22:19:55 INFO [http_cmd]: http status code OK 22:19:55 INFO [http_cmd]: http status code OK 22:19:55 INFO [http_cmd]: http status code OK 22:19:55 INFO [http_cmd]: http status code OK 22:19:55 DEBUG --- status code --- 22:19:55 INFO [http_cmd]: http status code OK 22:19:55 DEBUG --- status code --- 22:19:55 INFO [http_cmd]: http status code OK 22:19:55 DEBUG --- status code --- 22:19:55 DEBUG --- status code --- 22:19:55 DEBUG --- status code --- 22:19:55 DEBUG --- status code --- 22:19:55 DEBUG --- status code --- 22:19:55 DEBUG --- status code --- 22:19:55 DEBUG --- status code --- 22:19:55 DEBUG --- status code --- 22:19:55 DEBUG --- status code --- 22:19:55 DEBUG --- status code --- 22:19:55 DEBUG --- status code --- 22:19:55 DEBUG --- status code --- 22:19:55 DEBUG --- status code --- 22:19:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:19:55 DEBUG --- status code --- 22:19:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:19:55 DEBUG --- status code --- 22:19:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:19:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:19:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:19:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:19:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:19:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:19:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:19:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:19:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:19:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:19:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:19:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:19:55 INFO [http_cmd]: http status code OK 22:19:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:19:55 INFO [http_cmd]: http status code OK 22:19:55 DEBUG --- http response --- 22:19:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:19:55 DEBUG --- http response --- 22:19:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:19:55 DEBUG --- http response --- 22:19:55 DEBUG --- http response --- 22:19:55 DEBUG --- http response --- 22:19:55 DEBUG --- http response --- 22:19:55 DEBUG --- http response --- 22:19:55 DEBUG --- http response --- 22:19:55 DEBUG --- http response --- 22:19:55 DEBUG --- http response --- 22:19:55 DEBUG --- http response --- 22:19:55 DEBUG --- http response --- 22:19:55 DEBUG --- http response --- 22:19:55 DEBUG --- http response --- 22:19:55 DEBUG --- status code --- 22:19:55 DEBUG --- http response --- 22:19:55 DEBUG --- status code --- 22:19:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:19:55 DEBUG --- http response --- 22:19:55 DEBUG --- http response --- 22:19:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:19:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:19:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:19:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:19:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:19:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:19:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:19:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:19:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:19:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:19:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:19:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:19:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:19:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:19:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:19:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:19:55 DEBUG ----- output ----- 22:19:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:19:55 DEBUG ----- output ----- 22:19:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:19:55 DEBUG ----- output ----- 22:19:55 DEBUG ----- output ----- 22:19:55 DEBUG ----- output ----- 22:19:55 DEBUG ----- output ----- 22:19:55 DEBUG ----- output ----- 22:19:55 DEBUG ----- output ----- 22:19:55 DEBUG ----- output ----- 22:19:55 DEBUG ----- output ----- 22:19:55 DEBUG ----- output ----- 22:19:55 DEBUG ----- output ----- 22:19:55 DEBUG ----- output ----- 22:19:55 DEBUG --- http response --- 22:19:55 DEBUG ----- output ----- 22:19:55 DEBUG --- http response --- 22:19:55 DEBUG ----- output ----- 22:19:55 DEBUG { 22:19:55 DEBUG ----- output ----- 22:19:55 DEBUG { 22:19:55 DEBUG ----- output ----- 22:19:55 DEBUG { 22:19:55 DEBUG { 22:19:55 DEBUG { 22:19:55 DEBUG { 22:19:55 DEBUG { 22:19:55 DEBUG { 22:19:55 DEBUG { 22:19:55 DEBUG { 22:19:55 DEBUG { 22:19:55 DEBUG { 22:19:55 DEBUG { 22:19:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:19:55 DEBUG { 22:19:55 DEBUG { 22:19:55 DEBUG "status": "success", 22:19:55 INFO [http_cmd]: http status code OK 22:19:55 DEBUG { 22:19:55 INFO [http_cmd]: http status code OK 22:19:55 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:19:55 INFO [http_cmd]: http status code OK 22:19:55 INFO [http_cmd]: http status code OK 22:19:55 DEBUG "status": "success", 22:19:55 INFO [http_cmd]: http status code OK 22:19:55 INFO [http_cmd]: http status code OK 22:19:55 DEBUG { 22:19:55 DEBUG "status": "success", 22:19:55 DEBUG "status": "success", 22:19:55 DEBUG "status": "success", 22:19:55 DEBUG "status": "success", 22:19:55 DEBUG "status": "success", 22:19:55 DEBUG "status": "success", 22:19:55 DEBUG "status": "success", 22:19:55 DEBUG "status": "success", 22:19:55 DEBUG "status": "success", 22:19:55 DEBUG "status": "success", 22:19:55 DEBUG "status": "success", 22:19:55 DEBUG ----- output ----- 22:19:55 DEBUG "status": "success", 22:19:55 DEBUG "status": "success", 22:19:55 DEBUG "data": { 22:19:55 DEBUG --- status code --- 22:19:55 DEBUG "status": "success", 22:19:55 DEBUG --- status code --- 22:19:55 DEBUG ----- output ----- 22:19:55 DEBUG --- status code --- 22:19:55 DEBUG --- status code --- 22:19:55 DEBUG "data": { 22:19:55 DEBUG --- status code --- 22:19:55 DEBUG --- status code --- 22:19:55 DEBUG "status": "success", 22:19:55 DEBUG "data": { 22:19:55 DEBUG "data": { 22:19:55 DEBUG "data": { 22:19:55 DEBUG "data": { 22:19:55 DEBUG "data": { 22:19:55 DEBUG "data": { 22:19:55 DEBUG "data": { 22:19:55 DEBUG "data": { 22:19:55 DEBUG "data": { 22:19:55 DEBUG "data": { 22:19:55 DEBUG "data": { 22:19:55 DEBUG { 22:19:55 DEBUG "data": { 22:19:55 DEBUG "data": { 22:19:55 DEBUG "resultType": "vector", 22:19:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:19:55 DEBUG "data": { 22:19:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:19:55 DEBUG { 22:19:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:19:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:19:55 DEBUG "resultType": "vector", 22:19:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:19:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:19:55 DEBUG "data": { 22:19:55 DEBUG "resultType": "vector", 22:19:55 DEBUG "resultType": "vector", 22:19:55 DEBUG "resultType": "vector", 22:19:55 DEBUG "resultType": "vector", 22:19:55 DEBUG "resultType": "vector", 22:19:55 DEBUG "resultType": "vector", 22:19:55 DEBUG "resultType": "vector", 22:19:55 DEBUG "resultType": "vector", 22:19:55 DEBUG "resultType": "vector", 22:19:55 DEBUG "resultType": "vector", 22:19:55 DEBUG "resultType": "vector", 22:19:55 DEBUG "status": "success", 22:19:55 DEBUG "resultType": "vector", 22:19:55 DEBUG "resultType": "vector", 22:19:55 DEBUG "result": [ 22:19:55 DEBUG --- http response --- 22:19:55 DEBUG "resultType": "vector", 22:19:55 DEBUG --- http response --- 22:19:55 DEBUG "status": "success", 22:19:55 DEBUG --- http response --- 22:19:55 DEBUG --- http response --- 22:19:55 DEBUG "result": [ 22:19:55 DEBUG --- http response --- 22:19:55 DEBUG --- http response --- 22:19:55 DEBUG "resultType": "vector", 22:19:55 DEBUG "result": [ 22:19:55 DEBUG "result": [ 22:19:55 DEBUG "result": [ 22:19:55 DEBUG "result": [ 22:19:55 DEBUG "result": [ 22:19:55 DEBUG "result": [ 22:19:55 DEBUG "result": [ 22:19:55 DEBUG "result": [ 22:19:55 DEBUG "result": [ 22:19:55 DEBUG "result": [ 22:19:55 DEBUG "result": [ 22:19:55 DEBUG "data": { 22:19:55 DEBUG "result": [ 22:19:55 DEBUG "result": [ 22:19:55 DEBUG { 22:19:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:19:56 DEBUG "result": [ 22:19:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:19:56 DEBUG "data": { 22:19:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:19:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:19:56 DEBUG { 22:19:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:19:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:19:56 DEBUG "result": [ 22:19:56 DEBUG { 22:19:56 DEBUG { 22:19:56 DEBUG { 22:19:56 DEBUG { 22:19:56 DEBUG { 22:19:56 DEBUG { 22:19:56 DEBUG { 22:19:56 DEBUG { 22:19:56 DEBUG { 22:19:56 DEBUG { 22:19:56 DEBUG { 22:19:56 DEBUG "resultType": "vector", 22:19:56 DEBUG { 22:19:56 DEBUG { 22:19:56 DEBUG "metric": { 22:19:56 DEBUG ----- output ----- 22:19:56 DEBUG { 22:19:56 DEBUG ----- output ----- 22:19:56 DEBUG "resultType": "vector", 22:19:56 DEBUG ----- output ----- 22:19:56 DEBUG ----- output ----- 22:19:56 DEBUG "metric": { 22:19:56 DEBUG ----- output ----- 22:19:56 DEBUG ----- output ----- 22:19:56 DEBUG { 22:19:56 DEBUG "metric": { 22:19:56 DEBUG "metric": { 22:19:56 DEBUG "metric": { 22:19:56 DEBUG "metric": { 22:19:56 DEBUG "metric": { 22:19:56 DEBUG "metric": { 22:19:56 DEBUG "metric": { 22:19:56 DEBUG "metric": { 22:19:56 DEBUG "metric": { 22:19:56 DEBUG "metric": { 22:19:56 DEBUG "metric": { 22:19:56 DEBUG "result": [ 22:19:56 DEBUG "metric": { 22:19:56 DEBUG "metric": { 22:19:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:19:56 DEBUG { 22:19:56 DEBUG "metric": { 22:19:56 DEBUG { 22:19:56 DEBUG "result": [ 22:19:56 DEBUG { 22:19:56 DEBUG { 22:19:56 DEBUG "pod": "ds-idrepo-2" 22:19:56 DEBUG { 22:19:56 DEBUG { 22:19:56 DEBUG "metric": { 22:19:56 DEBUG "pod": "ds-idrepo-2" 22:19:56 DEBUG "pod": "am-555f4db595-fz8sn" 22:19:56 DEBUG "pod": "am-555f4db595-fz8sn" 22:19:56 DEBUG "pod": "ds-idrepo-2" 22:19:56 DEBUG "pod": "ds-idrepo-2" 22:19:56 DEBUG "pod": "ds-idrepo-2" 22:19:56 DEBUG "instance": "10.142.0.107:9100" 22:19:56 DEBUG "pod": "am-555f4db595-fz8sn" 22:19:56 DEBUG "instance": "10.142.0.107:9100" 22:19:56 DEBUG "instance": "10.142.0.107:9100" 22:19:56 DEBUG "pod": "ds-idrepo-2" 22:19:56 DEBUG { 22:19:56 DEBUG "pod": "am-555f4db595-fz8sn" 22:19:56 DEBUG "pod": "am-555f4db595-fz8sn" 22:19:56 DEBUG }, 22:19:56 DEBUG "status": "success", 22:19:56 DEBUG "pod": "am-555f4db595-fz8sn" 22:19:56 DEBUG "status": "success", 22:19:56 DEBUG { 22:19:56 DEBUG "status": "success", 22:19:56 DEBUG "status": "success", 22:19:56 DEBUG }, 22:19:56 DEBUG "status": "success", 22:19:56 DEBUG "status": "success", 22:19:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:19:56 DEBUG }, 22:19:56 DEBUG }, 22:19:56 DEBUG }, 22:19:56 DEBUG }, 22:19:56 DEBUG }, 22:19:56 DEBUG }, 22:19:56 DEBUG }, 22:19:56 DEBUG }, 22:19:56 DEBUG }, 22:19:56 DEBUG }, 22:19:56 DEBUG }, 22:19:56 DEBUG "metric": { 22:19:56 DEBUG }, 22:19:56 DEBUG }, 22:19:56 DEBUG "value": [ 22:19:56 DEBUG "data": { 22:19:56 DEBUG }, 22:19:56 DEBUG "data": { 22:19:56 DEBUG "metric": { 22:19:56 DEBUG "data": { 22:19:56 DEBUG "data": { 22:19:56 DEBUG "value": [ 22:19:56 DEBUG "data": { 22:19:56 DEBUG "data": { 22:19:56 DEBUG }, 22:19:56 DEBUG "value": [ 22:19:56 DEBUG "value": [ 22:19:56 DEBUG "value": [ 22:19:56 DEBUG "value": [ 22:19:56 DEBUG "value": [ 22:19:56 DEBUG "value": [ 22:19:56 DEBUG "value": [ 22:19:56 DEBUG "value": [ 22:19:56 DEBUG "value": [ 22:19:56 DEBUG "value": [ 22:19:56 DEBUG "value": [ 22:19:56 DEBUG "pod": "am-555f4db595-fz8sn" 22:19:56 DEBUG "value": [ 22:19:56 DEBUG "value": [ 22:19:56 DEBUG 22:19:56 DEBUG "resultType": "vector", 22:19:56 DEBUG "value": [ 22:19:56 DEBUG "resultType": "vector", 22:19:56 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:19:56 DEBUG "resultType": "vector", 22:19:56 DEBUG "resultType": "vector", 22:19:56 DEBUG 22:19:56 DEBUG "resultType": "vector", 22:19:56 DEBUG "resultType": "vector", 22:19:56 DEBUG "value": [ 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG }, 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG "result": [ 22:19:56 DEBUG 22:19:56 DEBUG "result": [ 22:19:56 DEBUG }, 22:19:56 DEBUG "result": [ 22:19:56 DEBUG "result": [ 22:19:56 DEBUG 22:19:56 DEBUG "result": [ 22:19:56 DEBUG "result": [ 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG "value": [ 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG { 22:19:56 DEBUG 22:19:56 DEBUG { 22:19:56 DEBUG "value": [ 22:19:56 DEBUG { 22:19:56 DEBUG { 22:19:56 DEBUG 22:19:56 DEBUG { 22:19:56 DEBUG { 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG [...] 22:19:56 DEBUG "metric": { 22:19:56 DEBUG 22:19:56 DEBUG "metric": { 22:19:56 DEBUG 22:19:56 DEBUG "metric": { 22:19:56 DEBUG "metric": { 22:19:56 DEBUG [...] 22:19:56 DEBUG "metric": { 22:19:56 DEBUG "metric": { 22:19:56 DEBUG 22:19:56 DEBUG [...] 22:19:56 DEBUG [...] 22:19:56 DEBUG [...] 22:19:56 DEBUG [...] 22:19:56 DEBUG [...] 22:19:56 DEBUG [...] 22:19:56 DEBUG 22:19:56 DEBUG [...] 22:19:56 DEBUG [...] 22:19:56 DEBUG [...] 22:19:56 DEBUG [...] 22:19:56 DEBUG 22:19:56 DEBUG [...] 22:19:56 DEBUG [...] 22:19:56 DEBUG 22:19:56 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:19:56 DEBUG [...] 22:19:56 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:19:56 DEBUG 22:19:56 DEBUG "instance": "10.142.0.107:9100" 22:19:56 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:19:56 DEBUG 22:19:56 DEBUG "instance": "10.142.0.107:9100" 22:19:56 DEBUG "pod": "ds-idrepo-2" 22:19:56 DEBUG [...] 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG [...] 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG }, 22:19:56 DEBUG 22:19:56 DEBUG }, 22:19:56 DEBUG 22:19:56 DEBUG }, 22:19:56 DEBUG }, 22:19:56 DEBUG 22:19:56 DEBUG }, 22:19:56 DEBUG }, 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG [...] 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG "value": [ 22:19:56 DEBUG 22:19:56 DEBUG "value": [ 22:19:56 DEBUG [...] 22:19:56 DEBUG "value": [ 22:19:56 DEBUG "value": [ 22:19:56 DEBUG 22:19:56 DEBUG "value": [ 22:19:56 DEBUG "value": [ 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG "pod": "ds-idrepo-0" 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG "pod": "ds-idrepo-0" 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG "pod": "ds-idrepo-0" 22:19:56 DEBUG "pod": "am-555f4db595-kzxsp" 22:19:56 DEBUG "pod": "am-555f4db595-kzxsp" 22:19:56 DEBUG "pod": "ds-idrepo-0" 22:19:56 DEBUG "pod": "ds-idrepo-0" 22:19:56 DEBUG 22:19:56 DEBUG }, 22:19:56 DEBUG "pod": "am-555f4db595-kzxsp" 22:19:56 DEBUG "instance": "10.142.0.43:9100" 22:19:56 DEBUG "instance": "10.142.0.43:9100" 22:19:56 DEBUG "pod": "ds-cts-2" 22:19:56 DEBUG 22:19:56 DEBUG "pod": "am-555f4db595-kzxsp" 22:19:56 DEBUG "pod": "am-555f4db595-kzxsp" 22:19:56 DEBUG }, 22:19:56 DEBUG 22:19:56 DEBUG "pod": "am-555f4db595-kzxsp" 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG }, 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:19:56 DEBUG }, 22:19:56 DEBUG }, 22:19:56 DEBUG }, 22:19:56 DEBUG }, 22:19:56 DEBUG "value": [ 22:19:56 DEBUG "instance": "10.142.0.43:9100" 22:19:56 DEBUG "value": [ 22:19:56 DEBUG }, 22:19:56 DEBUG }, 22:19:56 DEBUG }, 22:19:56 DEBUG }, 22:19:56 DEBUG 22:19:56 DEBUG }, 22:19:56 DEBUG }, 22:19:56 DEBUG "value": [ 22:19:56 DEBUG 22:19:56 DEBUG }, 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG "value": [ 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG }, 22:19:56 DEBUG "value": [ 22:19:56 DEBUG "value": [ 22:19:56 DEBUG "value": [ 22:19:56 DEBUG }, 22:19:56 DEBUG 1713302395, 22:19:56 DEBUG }, 22:19:56 DEBUG 1713302395, 22:19:56 DEBUG "value": [ 22:19:56 DEBUG "value": [ 22:19:56 DEBUG "value": [ 22:19:56 DEBUG "value": [ 22:19:56 DEBUG "pod": "am-555f4db595-kzxsp" 22:19:56 DEBUG "value": [ 22:19:56 DEBUG "value": [ 22:19:56 DEBUG 1713302395, 22:19:56 DEBUG [...] 22:19:56 DEBUG "value": [ 22:19:56 DEBUG [...] 22:19:56 DEBUG "pod": "overseer-0-7c9b667974-9mr2s" 22:19:56 DEBUG [...] 22:19:56 DEBUG [...] 22:19:56 DEBUG 1713302395, 22:19:56 DEBUG [...] 22:19:56 DEBUG [...] 22:19:56 DEBUG "value": [ 22:19:56 DEBUG 1713302395, 22:19:56 DEBUG 1713302395, 22:19:56 DEBUG 1713302395, 22:19:56 DEBUG "value": [ 22:19:56 DEBUG "0" 22:19:56 DEBUG "value": [ 22:19:56 DEBUG "0" 22:19:56 DEBUG 1713302395, 22:19:56 DEBUG 1713302395, 22:19:56 DEBUG 1713302395, 22:19:56 DEBUG 1713302395, 22:19:56 DEBUG }, 22:19:56 DEBUG 1713302395, 22:19:56 DEBUG 1713302395, 22:19:56 DEBUG "638984192" 22:19:56 DEBUG 22:19:56 DEBUG 1713302395, 22:19:56 DEBUG 22:19:56 DEBUG }, 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG "0" 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG 1713302395, 22:19:56 DEBUG "0" 22:19:56 DEBUG "0" 22:19:56 DEBUG "2008.8666666666666" 22:19:56 DEBUG 1713302395, 22:19:56 DEBUG ] 22:19:56 DEBUG 1713302395, 22:19:56 DEBUG ] 22:19:56 DEBUG "2008.5" 22:19:56 DEBUG "45579659.66666666" 22:19:56 DEBUG "0.00036666666666652263" 22:19:56 DEBUG "0" 22:19:56 DEBUG "value": [ 22:19:56 DEBUG "0.023" 22:19:56 DEBUG "0" 22:19:56 DEBUG ] 22:19:56 DEBUG 22:19:56 DEBUG "920597" 22:19:56 DEBUG 22:19:56 DEBUG "value": [ 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG ] 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG "6.33192913671859" 22:19:56 DEBUG ] 22:19:56 DEBUG ] 22:19:56 DEBUG ] 22:19:56 DEBUG "0" 22:19:56 DEBUG } 22:19:56 DEBUG "19951082496" 22:19:56 DEBUG } 22:19:56 DEBUG ] 22:19:56 DEBUG ] 22:19:56 DEBUG ] 22:19:56 DEBUG ] 22:19:56 DEBUG 1713302395, 22:19:56 DEBUG ] 22:19:56 DEBUG ] 22:19:56 DEBUG } 22:19:56 DEBUG 22:19:56 DEBUG ] 22:19:56 DEBUG 22:19:56 DEBUG 1713302395, 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG } 22:19:56 DEBUG 22:19:56 DEBUG 22:19:56 DEBUG ] 22:19:56 DEBUG } 22:19:56 DEBUG } 22:19:56 DEBUG } 22:19:56 DEBUG ] 22:19:56 DEBUG ] 22:19:56 DEBUG ] 22:19:56 DEBUG ] 22:19:56 DEBUG } 22:19:56 DEBUG } 22:19:56 DEBUG } 22:19:56 DEBUG } 22:19:56 DEBUG "2008.8" 22:19:56 DEBUG } 22:19:56 DEBUG } 22:19:56 DEBUG ] 22:19:56 DEBUG "pod": "overseer-0-7c9b667974-9mr2s" 22:19:56 DEBUG } 22:19:56 DEBUG "pod": "overseer-0-7c9b667974-9mr2s" 22:19:56 DEBUG "20917.168828515983" 22:19:56 DEBUG "instance": "10.142.0.43:9100" 22:19:56 DEBUG "pod": "overseer-0-7c9b667974-9mr2s" 22:19:56 DEBUG ] 22:19:56 DEBUG "instance": "10.142.0.43:9100" 22:19:56 DEBUG "pod": "ds-idrepo-0" 22:19:56 DEBUG } 22:19:56 DEBUG ] 22:19:56 DEBUG ] 22:19:56 DEBUG ] 22:19:56 DEBUG } 22:19:56 DEBUG } 22:19:56 DEBUG } 22:19:56 DEBUG } 22:19:56 DEBUG ] 22:19:56 DEBUG ] 22:19:56 DEBUG ] 22:19:56 DEBUG ] 22:19:56 DEBUG ] 22:19:56 DEBUG ] 22:19:56 DEBUG ] 22:19:56 DEBUG } 22:19:56 DEBUG }, 22:19:56 DEBUG ] 22:19:56 DEBUG }, 22:19:56 DEBUG ] 22:19:56 DEBUG }, 22:19:56 DEBUG }, 22:19:56 DEBUG } 22:19:56 DEBUG }, 22:19:56 DEBUG }, 22:19:56 DEBUG ] 22:19:56 DEBUG } 22:19:56 DEBUG } 22:19:56 DEBUG } 22:19:56 DEBUG ] 22:19:56 DEBUG } 22:19:56 DEBUG ] 22:19:56 DEBUG } 22:19:56 DEBUG } 22:19:56 DEBUG } 22:19:56 DEBUG } 22:19:56 DEBUG } 22:19:56 DEBUG } 22:19:56 DEBUG } 22:19:56 DEBUG } 22:19:56 DEBUG } 22:19:56 DEBUG "value": [ 22:19:56 DEBUG } 22:19:56 DEBUG "value": [ 22:19:56 DEBUG } 22:19:56 DEBUG "value": [ 22:19:56 DEBUG "value": [ 22:19:56 DEBUG } 22:19:56 DEBUG "value": [ 22:19:56 DEBUG "value": [ 22:19:56 DEBUG } 22:19:56 DEBUG } 22:19:56 DEBUG } 22:19:56 DEBUG } 22:19:56 DEBUG } 22:19:56 DEBUG } 22:19:56 DEBUG } 22:19:56 DEBUG } 22:19:56 DEBUG } 22:19:56 DEBUG } 22:19:56 DEBUG ] 22:19:56 DEBUG } 22:19:56 DEBUG } 22:19:56 DEBUG 1713302395, 22:19:56 DEBUG } 22:19:56 DEBUG 1713302395, 22:19:56 DEBUG ] 22:19:56 DEBUG 1713302395, 22:19:56 DEBUG 1713302395, 22:19:56 DEBUG 1713302395, 22:19:56 DEBUG 1713302395, 22:19:56 DEBUG } 22:19:56 DEBUG } 22:19:56 DEBUG } 22:19:56 DEBUG } 22:19:56 DEBUG "0" 22:19:56 DEBUG "0" 22:19:56 DEBUG } 22:19:56 DEBUG "45580909.099999994" 22:19:56 DEBUG "6.294180012732779" 22:19:56 DEBUG "0" 22:19:56 DEBUG "0" 22:19:56 DEBUG } 22:19:56 DEBUG ] 22:19:56 DEBUG ] 22:19:56 DEBUG } 22:19:56 DEBUG ] 22:19:56 DEBUG ] 22:19:56 DEBUG ] 22:19:56 DEBUG ] 22:19:56 DEBUG } 22:19:56 DEBUG } 22:19:56 DEBUG } 22:19:56 DEBUG } 22:19:56 DEBUG } 22:19:56 DEBUG } 22:19:56 DEBUG ] 22:19:56 DEBUG ] 22:19:56 DEBUG ] 22:19:56 DEBUG ] 22:19:56 DEBUG ] 22:19:56 DEBUG ] 22:19:56 DEBUG } 22:19:56 DEBUG } 22:19:56 DEBUG } 22:19:56 DEBUG } 22:19:56 DEBUG } 22:19:56 DEBUG } 22:19:56 DEBUG } 22:19:56 DEBUG } 22:19:56 DEBUG } 22:19:56 DEBUG } 22:19:56 DEBUG } 22:19:56 DEBUG } 22:19:56 INFO [loop_until]: OK (rc = 0) 22:19:56 DEBUG --- stdout --- 22:19:56 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 62m 0% 5083Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5785m 19% 14719Mi 13% gke-xlou-cdm-default-pool-1b0769c8-nhlf 9445m 31% 10817Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 81m 0% 2639Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 4743m 15% 6405Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 6795m 22% 16617Mi 14% gke-xlou-cdm-default-pool-1b0769c8-xnmp 65m 0% 3136Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 9426m 31% 10372Mi 9% gke-xlou-cdm-ds-0f601751-hmm7 4752m 15% 5571Mi 4% gke-xlou-cdm-ds-0f601751-j488 5705m 19% 14061Mi 12% gke-xlou-cdm-ds-0f601751-mc5d 10708m 35% 3339Mi 2% gke-xlou-cdm-ds-0f601751-nj6q 9556m 32% 10415Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 4784m 16% 7363Mi 6% gke-xlou-cdm-frontend-a8771548-i82n 6474m 40% 5950Mi 10% 22:19:56 DEBUG --- stderr --- 22:19:56 DEBUG 22:20:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302455" 22:20:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713302455" 22:20:55 ERROR [http_cmd]: ERROR 22:20:55 INFO [http_cmd]: http status code OK 22:20:55 ERROR --- status code --- 22:20:55 DEBUG --- status code --- 22:20:55 ERROR http status code is 400 (expected [200, 401, 403, 404]) 22:20:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:20:55 ERROR --- http response --- 22:20:55 DEBUG --- http response --- 22:20:55 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 22:20:55 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 22:20:55 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 22:20:55 INFO 22:20:55 INFO [loop_until]: kubectl --namespace=xlou top pods 22:20:55 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:20:55 INFO [loop_until]: OK (rc = 0) 22:20:55 DEBUG --- stdout --- 22:20:55 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 9Mi am-555f4db595-4lbss 9367m 8584Mi am-555f4db595-fz8sn 9365m 8594Mi am-555f4db595-kzxsp 8820m 8552Mi ds-cts-0 6580m 12079Mi ds-cts-1 5682m 12150Mi ds-cts-2 5964m 12112Mi ds-idrepo-0 4733m 5032Mi ds-idrepo-1 4527m 2834Mi ds-idrepo-2 4531m 3093Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 4m 1208Mi idm-b85f9d978-rl76f 3m 1226Mi lodemon-bc756c647-cr7x6 22m 63Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-7c9b667974-9mr2s 6116m 519Mi 22:20:55 DEBUG --- stderr --- 22:20:55 DEBUG 22:20:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302456" 22:20:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302456" 22:20:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302456" 22:20:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302456" 22:20:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713302456" 22:20:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302456" 22:20:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302456" 22:20:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302456" 22:20:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713302456" 22:20:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713302456" 22:20:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302456" 22:20:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302456" 22:20:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713302456" 22:20:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302456" 22:20:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302456" 22:20:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302456" 22:20:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302456" 22:20:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302456" 22:20:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302456" 22:20:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302456" 22:20:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302456" 22:20:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302456" 22:20:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302456" 22:20:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302456" 22:20:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302456" 22:20:56 INFO [http_cmd]: http status code OK 22:20:56 INFO [http_cmd]: http status code OK 22:20:56 INFO [http_cmd]: http status code OK 22:20:56 INFO [http_cmd]: http status code OK 22:20:56 INFO [http_cmd]: http status code OK 22:20:56 INFO [http_cmd]: http status code OK 22:20:56 INFO [http_cmd]: http status code OK 22:20:56 INFO [http_cmd]: http status code OK 22:20:56 INFO [http_cmd]: http status code OK 22:20:56 INFO [http_cmd]: http status code OK 22:20:56 INFO [http_cmd]: http status code OK 22:20:56 INFO [http_cmd]: http status code OK 22:20:56 INFO [http_cmd]: http status code OK 22:20:56 INFO [http_cmd]: http status code OK 22:20:56 INFO [http_cmd]: http status code OK 22:20:56 INFO [http_cmd]: http status code OK 22:20:56 INFO [http_cmd]: http status code OK 22:20:56 INFO [http_cmd]: http status code OK 22:20:56 INFO [http_cmd]: http status code OK 22:20:56 DEBUG --- status code --- 22:20:56 INFO [http_cmd]: http status code OK 22:20:56 INFO [http_cmd]: http status code OK 22:20:56 DEBUG --- status code --- 22:20:56 DEBUG --- status code --- 22:20:56 DEBUG --- status code --- 22:20:56 DEBUG --- status code --- 22:20:56 DEBUG --- status code --- 22:20:56 DEBUG --- status code --- 22:20:56 DEBUG --- status code --- 22:20:56 DEBUG --- status code --- 22:20:56 DEBUG --- status code --- 22:20:56 DEBUG --- status code --- 22:20:56 DEBUG --- status code --- 22:20:56 DEBUG --- status code --- 22:20:56 DEBUG --- status code --- 22:20:56 DEBUG --- status code --- 22:20:56 INFO [http_cmd]: http status code OK 22:20:56 DEBUG --- status code --- 22:20:56 DEBUG --- status code --- 22:20:56 DEBUG --- status code --- 22:20:56 DEBUG --- status code --- 22:20:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:20:56 DEBUG --- status code --- 22:20:56 DEBUG --- status code --- 22:20:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:20:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:20:56 INFO [http_cmd]: http status code OK 22:20:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:20:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:20:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:20:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:20:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:20:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:20:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:20:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:20:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:20:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:20:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:20:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:20:56 DEBUG --- status code --- 22:20:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:20:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:20:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:20:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:20:56 DEBUG --- http response --- 22:20:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:20:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:20:56 DEBUG --- http response --- 22:20:56 DEBUG --- http response --- 22:20:56 DEBUG --- status code --- 22:20:56 DEBUG --- http response --- 22:20:56 DEBUG --- http response --- 22:20:56 INFO [http_cmd]: http status code OK 22:20:56 DEBUG --- http response --- 22:20:56 DEBUG --- http response --- 22:20:56 DEBUG --- http response --- 22:20:56 DEBUG --- http response --- 22:20:56 DEBUG --- http response --- 22:20:56 DEBUG --- http response --- 22:20:56 DEBUG --- http response --- 22:20:56 DEBUG --- http response --- 22:20:56 DEBUG --- http response --- 22:20:56 DEBUG --- http response --- 22:20:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:20:56 DEBUG --- http response --- 22:20:56 DEBUG --- http response --- 22:20:56 DEBUG --- http response --- 22:20:56 DEBUG --- http response --- 22:20:56 DEBUG --- http response --- 22:20:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:20:56 INFO [http_cmd]: http status code OK 22:20:56 DEBUG --- http response --- 22:20:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:20:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:20:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:20:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:20:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:20:56 DEBUG --- status code --- 22:20:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:20:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:20:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:20:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:20:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:20:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:20:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:20:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:20:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:20:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:20:56 DEBUG --- http response --- 22:20:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:20:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:20:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:20:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:20:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:20:56 DEBUG ----- output ----- 22:20:56 DEBUG --- status code --- 22:20:56 DEBUG ----- output ----- 22:20:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:20:56 DEBUG ----- output ----- 22:20:56 DEBUG --- http response --- 22:20:56 DEBUG ----- output ----- 22:20:56 DEBUG ----- output ----- 22:20:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:20:56 DEBUG ----- output ----- 22:20:56 DEBUG ----- output ----- 22:20:56 DEBUG ----- output ----- 22:20:56 DEBUG ----- output ----- 22:20:56 DEBUG ----- output ----- 22:20:56 DEBUG ----- output ----- 22:20:56 DEBUG ----- output ----- 22:20:56 DEBUG ----- output ----- 22:20:56 DEBUG ----- output ----- 22:20:56 DEBUG ----- output ----- 22:20:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:20:56 DEBUG ----- output ----- 22:20:56 DEBUG ----- output ----- 22:20:56 DEBUG ----- output ----- 22:20:56 DEBUG ----- output ----- 22:20:56 DEBUG ----- output ----- 22:20:56 DEBUG { 22:20:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:20:56 DEBUG { 22:20:56 DEBUG ----- output ----- 22:20:56 DEBUG { 22:20:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:20:56 DEBUG { 22:20:56 DEBUG { 22:20:56 DEBUG --- http response --- 22:20:56 DEBUG { 22:20:56 DEBUG { 22:20:56 DEBUG { 22:20:56 DEBUG { 22:20:56 DEBUG { 22:20:56 DEBUG { 22:20:56 DEBUG { 22:20:56 DEBUG { 22:20:56 DEBUG { 22:20:56 DEBUG { 22:20:56 DEBUG "status": "success", 22:20:56 DEBUG { 22:20:56 DEBUG { 22:20:56 DEBUG { 22:20:56 DEBUG { 22:20:56 DEBUG { 22:20:56 DEBUG "status": "success", 22:20:56 DEBUG --- http response --- 22:20:56 DEBUG "status": "success", 22:20:56 DEBUG { 22:20:56 DEBUG "status": "success", 22:20:56 DEBUG ----- output ----- 22:20:56 DEBUG "status": "success", 22:20:56 DEBUG "status": "success", 22:20:56 DEBUG "status": "success", 22:20:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:20:56 DEBUG "status": "success", 22:20:56 DEBUG "status": "success", 22:20:56 DEBUG "status": "success", 22:20:56 DEBUG "status": "success", 22:20:56 DEBUG "status": "success", 22:20:56 DEBUG "status": "success", 22:20:56 DEBUG "status": "success", 22:20:56 DEBUG "status": "success", 22:20:56 DEBUG "data": { 22:20:56 DEBUG ----- output ----- 22:20:56 DEBUG "status": "success", 22:20:56 DEBUG "status": "success", 22:20:56 DEBUG "status": "success", 22:20:56 DEBUG "status": "success", 22:20:56 DEBUG "status": "success", 22:20:56 DEBUG "data": { 22:20:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:20:56 DEBUG "data": { 22:20:56 DEBUG "status": "success", 22:20:56 DEBUG "data": { 22:20:56 DEBUG { 22:20:56 DEBUG "data": { 22:20:56 DEBUG "data": { 22:20:56 DEBUG "data": { 22:20:56 DEBUG ----- output ----- 22:20:56 DEBUG "data": { 22:20:56 DEBUG "data": { 22:20:56 DEBUG "data": { 22:20:56 DEBUG "data": { 22:20:56 DEBUG "data": { 22:20:56 DEBUG "data": { 22:20:56 DEBUG "data": { 22:20:56 DEBUG "data": { 22:20:56 DEBUG "resultType": "vector", 22:20:56 DEBUG { 22:20:56 DEBUG "data": { 22:20:56 DEBUG "data": { 22:20:56 DEBUG "data": { 22:20:56 DEBUG "data": { 22:20:56 DEBUG "data": { 22:20:56 DEBUG "resultType": "vector", 22:20:56 DEBUG ----- output ----- 22:20:56 DEBUG "resultType": "vector", 22:20:56 DEBUG "data": { 22:20:56 DEBUG "resultType": "vector", 22:20:56 DEBUG "status": "success", 22:20:56 DEBUG "resultType": "vector", 22:20:56 DEBUG "resultType": "vector", 22:20:56 DEBUG "resultType": "vector", 22:20:56 DEBUG { 22:20:56 DEBUG "resultType": "vector", 22:20:56 DEBUG "resultType": "vector", 22:20:56 DEBUG "resultType": "vector", 22:20:56 DEBUG "resultType": "vector", 22:20:56 DEBUG "resultType": "vector", 22:20:56 DEBUG "resultType": "vector", 22:20:56 DEBUG "resultType": "vector", 22:20:56 DEBUG "resultType": "vector", 22:20:56 DEBUG "result": [ 22:20:56 DEBUG "status": "success", 22:20:56 DEBUG "resultType": "vector", 22:20:56 DEBUG "resultType": "vector", 22:20:56 DEBUG "resultType": "vector", 22:20:56 DEBUG "resultType": "vector", 22:20:56 DEBUG "resultType": "vector", 22:20:56 DEBUG "result": [ 22:20:56 DEBUG { 22:20:56 DEBUG "result": [ 22:20:56 DEBUG "resultType": "vector", 22:20:56 DEBUG "result": [ 22:20:56 DEBUG "data": { 22:20:56 DEBUG "result": [ 22:20:56 DEBUG "result": [ 22:20:56 DEBUG "result": [ 22:20:56 DEBUG "status": "success", 22:20:56 DEBUG "result": [ 22:20:56 DEBUG "result": [ 22:20:56 DEBUG "result": [ 22:20:56 DEBUG "result": [ 22:20:56 DEBUG "result": [ 22:20:56 DEBUG "result": [ 22:20:56 DEBUG "result": [ 22:20:56 DEBUG "result": [ 22:20:56 DEBUG { 22:20:56 DEBUG "data": { 22:20:56 DEBUG "result": [ 22:20:56 DEBUG "result": [ 22:20:56 DEBUG "result": [ 22:20:56 DEBUG "result": [ 22:20:56 DEBUG "result": [ 22:20:56 DEBUG { 22:20:56 DEBUG "status": "success", 22:20:56 DEBUG { 22:20:56 DEBUG "result": [ 22:20:56 DEBUG { 22:20:56 DEBUG "resultType": "vector", 22:20:56 DEBUG { 22:20:56 DEBUG { 22:20:56 DEBUG { 22:20:56 DEBUG "data": { 22:20:56 DEBUG { 22:20:56 DEBUG { 22:20:56 DEBUG { 22:20:56 DEBUG { 22:20:56 DEBUG { 22:20:56 DEBUG { 22:20:56 DEBUG { 22:20:56 DEBUG { 22:20:56 DEBUG "metric": { 22:20:56 DEBUG "resultType": "vector", 22:20:56 DEBUG { 22:20:56 DEBUG { 22:20:56 DEBUG { 22:20:56 DEBUG { 22:20:56 DEBUG { 22:20:56 DEBUG "metric": { 22:20:56 DEBUG "data": { 22:20:56 DEBUG "metric": { 22:20:56 DEBUG { 22:20:56 DEBUG "metric": { 22:20:56 DEBUG "result": [ 22:20:56 DEBUG "metric": { 22:20:56 DEBUG "metric": { 22:20:56 DEBUG "metric": { 22:20:56 DEBUG "resultType": "vector", 22:20:56 DEBUG "metric": { 22:20:56 DEBUG "metric": { 22:20:56 DEBUG "metric": { 22:20:56 DEBUG "metric": { 22:20:56 DEBUG "metric": { 22:20:56 DEBUG "metric": { 22:20:56 DEBUG "metric": { 22:20:56 DEBUG "metric": { 22:20:56 DEBUG "pod": "am-555f4db595-fz8sn" 22:20:56 DEBUG "result": [ 22:20:56 DEBUG "metric": { 22:20:56 DEBUG "metric": { 22:20:56 DEBUG "metric": { 22:20:56 DEBUG "metric": { 22:20:56 DEBUG "metric": { 22:20:56 DEBUG "pod": "am-555f4db595-fz8sn" 22:20:56 DEBUG "resultType": "vector", 22:20:56 DEBUG "pod": "ds-idrepo-2" 22:20:56 DEBUG "metric": { 22:20:56 DEBUG "pod": "ds-idrepo-2" 22:20:56 DEBUG { 22:20:56 DEBUG "pod": "am-555f4db595-fz8sn" 22:20:56 DEBUG "instance": "10.142.0.107:9100" 22:20:56 DEBUG "pod": "ds-idrepo-2" 22:20:56 DEBUG "result": [ 22:20:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:20:56 DEBUG "pod": "am-555f4db595-fz8sn" 22:20:56 DEBUG "instance": "10.142.0.107:9100" 22:20:56 DEBUG "pod": "ds-idrepo-2" 22:20:56 DEBUG "pod": "am-555f4db595-fz8sn" 22:20:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:20:56 DEBUG "pod": "am-555f4db595-fz8sn" 22:20:56 DEBUG "pod": "am-555f4db595-fz8sn" 22:20:56 DEBUG }, 22:20:56 DEBUG { 22:20:56 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:20:56 DEBUG "pod": "ds-idrepo-2" 22:20:56 DEBUG "instance": "10.142.0.107:9100" 22:20:56 DEBUG "pod": "ds-idrepo-2" 22:20:56 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:20:56 DEBUG }, 22:20:56 DEBUG "result": [ 22:20:56 DEBUG }, 22:20:56 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:20:56 DEBUG }, 22:20:56 DEBUG "metric": { 22:20:56 DEBUG }, 22:20:56 DEBUG }, 22:20:56 DEBUG }, 22:20:56 DEBUG { 22:20:56 DEBUG }, 22:20:56 DEBUG }, 22:20:56 DEBUG }, 22:20:56 DEBUG }, 22:20:56 DEBUG }, 22:20:56 DEBUG }, 22:20:56 DEBUG }, 22:20:56 DEBUG }, 22:20:56 DEBUG "value": [ 22:20:56 DEBUG "metric": { 22:20:56 DEBUG }, 22:20:56 DEBUG }, 22:20:56 DEBUG }, 22:20:56 DEBUG }, 22:20:56 DEBUG }, 22:20:56 DEBUG "value": [ 22:20:56 DEBUG { 22:20:56 DEBUG "value": [ 22:20:56 DEBUG }, 22:20:56 DEBUG "value": [ 22:20:56 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:20:56 DEBUG "value": [ 22:20:56 DEBUG "value": [ 22:20:56 DEBUG "value": [ 22:20:56 DEBUG 22:20:56 DEBUG "value": [ 22:20:56 DEBUG "value": [ 22:20:56 DEBUG "value": [ 22:20:56 DEBUG "value": [ 22:20:56 DEBUG "value": [ 22:20:56 DEBUG "value": [ 22:20:56 DEBUG "value": [ 22:20:56 DEBUG "value": [ 22:20:56 DEBUG 22:20:56 DEBUG "instance": "10.142.0.107:9100" 22:20:56 DEBUG "value": [ 22:20:56 DEBUG "value": [ 22:20:56 DEBUG "value": [ 22:20:56 DEBUG "value": [ 22:20:56 DEBUG "value": [ 22:20:56 DEBUG 22:20:56 DEBUG "metric": { 22:20:56 DEBUG 22:20:56 DEBUG "value": [ 22:20:56 DEBUG 22:20:56 DEBUG }, 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG "metric": { 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG }, 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG "pod": "ds-idrepo-2" 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG "value": [ 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG "instance": "10.142.0.107:9100" 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG "value": [ 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG }, 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG }, 22:20:56 DEBUG [...] 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG [...] 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG [...] 22:20:56 DEBUG "value": [ 22:20:56 DEBUG [...] 22:20:56 DEBUG 22:20:56 DEBUG [...] 22:20:56 DEBUG 22:20:56 DEBUG [...] 22:20:56 DEBUG [...] 22:20:56 DEBUG "value": [ 22:20:56 DEBUG 22:20:56 DEBUG [...] 22:20:56 DEBUG [...] 22:20:56 DEBUG [...] 22:20:56 DEBUG [...] 22:20:56 DEBUG [...] 22:20:56 DEBUG [...] 22:20:56 DEBUG [...] 22:20:56 DEBUG [...] 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG [...] 22:20:56 DEBUG [...] 22:20:56 DEBUG [...] 22:20:56 DEBUG [...] 22:20:56 DEBUG [...] 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG [...] 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG [...] 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG [...] 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG "pod": "ds-idrepo-0" 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG "pod": "am-555f4db595-kzxsp" 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG 22:20:56 DEBUG "pod": "am-555f4db595-kzxsp" 22:20:56 DEBUG [...] 22:20:56 DEBUG "pod": "ds-idrepo-0" 22:20:56 DEBUG 22:20:56 DEBUG "pod": "ds-idrepo-0" 22:20:56 DEBUG 22:20:56 DEBUG "pod": "am-555f4db595-kzxsp" 22:20:56 DEBUG "instance": "10.142.0.43:9100" 22:20:56 DEBUG [...] 22:20:56 DEBUG }, 22:20:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:20:56 DEBUG "pod": "am-555f4db595-kzxsp" 22:20:56 DEBUG "instance": "10.142.0.43:9100" 22:20:56 DEBUG "pod": "ds-cts-2" 22:20:56 DEBUG "pod": "am-555f4db595-kzxsp" 22:20:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:20:56 DEBUG "pod": "am-555f4db595-kzxsp" 22:20:56 DEBUG "pod": "am-555f4db595-kzxsp" 22:20:56 DEBUG }, 22:20:56 DEBUG 22:20:56 DEBUG "pod": "overseer-0-7c9b667974-9mr2s" 22:20:56 DEBUG "pod": "ds-idrepo-0" 22:20:56 DEBUG "instance": "10.142.0.43:9100" 22:20:56 DEBUG "pod": "ds-idrepo-0" 22:20:56 DEBUG "pod": "overseer-0-7c9b667974-9mr2s" 22:20:56 DEBUG }, 22:20:56 DEBUG 22:20:56 DEBUG }, 22:20:56 DEBUG "pod": "overseer-0-7c9b667974-9mr2s" 22:20:56 DEBUG }, 22:20:56 DEBUG 22:20:56 DEBUG }, 22:20:56 DEBUG }, 22:20:56 DEBUG 22:20:56 DEBUG "value": [ 22:20:56 DEBUG }, 22:20:56 DEBUG }, 22:20:56 DEBUG }, 22:20:56 DEBUG }, 22:20:56 DEBUG }, 22:20:56 DEBUG }, 22:20:56 DEBUG }, 22:20:56 DEBUG }, 22:20:56 DEBUG "value": [ 22:20:56 DEBUG 22:20:56 DEBUG }, 22:20:56 DEBUG }, 22:20:56 DEBUG }, 22:20:56 DEBUG }, 22:20:56 DEBUG }, 22:20:56 DEBUG "value": [ 22:20:56 DEBUG 22:20:56 DEBUG "value": [ 22:20:56 DEBUG }, 22:20:56 DEBUG "value": [ 22:20:56 DEBUG "pod": "overseer-0-7c9b667974-9mr2s" 22:20:56 DEBUG "value": [ 22:20:56 DEBUG "value": [ 22:20:56 DEBUG 22:20:56 DEBUG 1713302456, 22:20:56 DEBUG "value": [ 22:20:56 DEBUG "value": [ 22:20:56 DEBUG "value": [ 22:20:56 DEBUG "value": [ 22:20:56 DEBUG "value": [ 22:20:56 DEBUG "value": [ 22:20:56 DEBUG "value": [ 22:20:56 DEBUG "value": [ 22:20:56 DEBUG 1713302456, 22:20:56 DEBUG "instance": "10.142.0.43:9100" 22:20:56 DEBUG "value": [ 22:20:56 DEBUG "value": [ 22:20:56 DEBUG "value": [ 22:20:56 DEBUG "value": [ 22:20:56 DEBUG "value": [ 22:20:56 DEBUG 1713302456, 22:20:56 DEBUG 22:20:56 DEBUG 1713302456, 22:20:56 DEBUG "value": [ 22:20:56 DEBUG 1713302456, 22:20:56 DEBUG }, 22:20:56 DEBUG 1713302456, 22:20:56 DEBUG 1713302456, 22:20:56 DEBUG 22:20:56 DEBUG "instance": "10.142.0.43:9100" 22:20:56 DEBUG 1713302456, 22:20:56 DEBUG 1713302456, 22:20:56 DEBUG 1713302456, 22:20:56 DEBUG 1713302456, 22:20:56 DEBUG 1713302456, 22:20:56 DEBUG 1713302456, 22:20:56 DEBUG 1713302456, 22:20:56 DEBUG 1713302456, 22:20:56 DEBUG "2000.7" 22:20:56 DEBUG }, 22:20:56 DEBUG 1713302456, 22:20:56 DEBUG 1713302456, 22:20:56 DEBUG 1713302456, 22:20:56 DEBUG 1713302456, 22:20:56 DEBUG 1713302456, 22:20:56 DEBUG "0" 22:20:56 DEBUG "pod": "ds-idrepo-0" 22:20:56 DEBUG "0" 22:20:56 DEBUG 1713302456, 22:20:56 DEBUG "0" 22:20:56 DEBUG "value": [ 22:20:56 DEBUG "2000.5333333333333" 22:20:56 DEBUG "45361742.93333333" 22:20:56 DEBUG "0" 22:20:56 DEBUG }, 22:20:56 DEBUG "644595712" 22:20:56 DEBUG "2000.8" 22:20:56 DEBUG "0.0005333333333339852" 22:20:56 DEBUG "0" 22:20:56 DEBUG "0" 22:20:56 DEBUG "6.2785921010582735" 22:20:56 DEBUG "1100318" 22:20:56 DEBUG "0.023" 22:20:56 DEBUG ] 22:20:56 DEBUG "value": [ 22:20:56 DEBUG "0" 22:20:56 DEBUG "0" 22:20:56 DEBUG "19951680512" 22:20:56 DEBUG "0" 22:20:56 DEBUG "27954.452554744523" 22:20:56 DEBUG ] 22:20:56 DEBUG }, 22:20:56 DEBUG ] 22:20:56 DEBUG "6.249636469598529" 22:20:56 DEBUG ] 22:20:56 DEBUG 1713302456, 22:20:56 DEBUG ] 22:20:56 DEBUG ] 22:20:56 DEBUG ] 22:20:56 DEBUG "value": [ 22:20:56 DEBUG ] 22:20:56 DEBUG ] 22:20:56 DEBUG ] 22:20:56 DEBUG ] 22:20:56 DEBUG ] 22:20:56 DEBUG ] 22:20:56 DEBUG ] 22:20:56 DEBUG ] 22:20:56 DEBUG } 22:20:56 DEBUG 1713302456, 22:20:56 DEBUG ] 22:20:56 DEBUG ] 22:20:56 DEBUG ] 22:20:56 DEBUG ] 22:20:56 DEBUG ] 22:20:56 DEBUG } 22:20:56 DEBUG "value": [ 22:20:56 DEBUG } 22:20:56 DEBUG ] 22:20:56 DEBUG } 22:20:56 DEBUG "0" 22:20:56 DEBUG } 22:20:56 DEBUG } 22:20:56 DEBUG ] 22:20:56 DEBUG 1713302456, 22:20:56 DEBUG } 22:20:56 DEBUG } 22:20:56 DEBUG } 22:20:56 DEBUG } 22:20:56 DEBUG } 22:20:56 DEBUG } 22:20:56 DEBUG } 22:20:56 DEBUG ] 22:20:56 DEBUG } 22:20:56 DEBUG "45371121.76666666" 22:20:56 DEBUG } 22:20:56 DEBUG } 22:20:56 DEBUG } 22:20:56 DEBUG } 22:20:56 DEBUG } 22:20:56 DEBUG ] 22:20:56 DEBUG 1713302456, 22:20:56 DEBUG ] 22:20:56 DEBUG } 22:20:56 DEBUG ] 22:20:56 DEBUG ] 22:20:56 DEBUG ] 22:20:56 DEBUG } 22:20:56 DEBUG } 22:20:56 DEBUG "0.00033333333333333365" 22:20:56 DEBUG ] 22:20:56 DEBUG ] 22:20:56 DEBUG ] 22:20:56 DEBUG ] 22:20:56 DEBUG ] 22:20:56 DEBUG ] 22:20:56 DEBUG ] 22:20:56 DEBUG } 22:20:56 DEBUG ] 22:20:56 DEBUG ] 22:20:56 DEBUG ] 22:20:56 DEBUG ] 22:20:56 DEBUG ] 22:20:56 DEBUG ] 22:20:56 DEBUG ] 22:20:56 DEBUG } 22:20:56 DEBUG "0" 22:20:56 DEBUG } 22:20:56 DEBUG ] 22:20:56 DEBUG } 22:20:56 DEBUG } 22:20:56 DEBUG } 22:20:56 DEBUG ] 22:20:56 DEBUG } 22:20:56 DEBUG ] 22:20:56 DEBUG } 22:20:56 DEBUG } 22:20:56 DEBUG } 22:20:56 DEBUG } 22:20:56 DEBUG } 22:20:56 DEBUG } 22:20:56 DEBUG } 22:20:56 DEBUG } 22:20:56 DEBUG } 22:20:56 DEBUG } 22:20:56 DEBUG } 22:20:56 DEBUG } 22:20:56 DEBUG } 22:20:56 DEBUG } 22:20:56 DEBUG } 22:20:56 DEBUG } 22:20:56 DEBUG ] 22:20:56 DEBUG } 22:20:56 DEBUG } 22:20:56 DEBUG } 22:20:56 DEBUG ] 22:20:56 DEBUG } 22:20:56 DEBUG } 22:20:56 DEBUG } 22:20:56 DEBUG } 22:20:56 DEBUG } 22:20:56 DEBUG } 22:20:56 DEBUG } 22:20:56 DEBUG } 22:20:56 DEBUG } 22:20:56 DEBUG } 22:20:56 DEBUG } 22:20:56 DEBUG ] 22:20:56 DEBUG } 22:20:56 DEBUG } 22:20:56 DEBUG } 22:20:56 DEBUG } 22:20:56 DEBUG } 22:20:56 DEBUG } 22:20:56 DEBUG } 22:20:56 DEBUG } 22:20:56 DEBUG } 22:20:56 DEBUG ] 22:20:56 DEBUG } 22:20:56 DEBUG ] 22:20:56 DEBUG } 22:20:56 DEBUG } 22:20:56 DEBUG } 22:20:56 DEBUG } 22:20:56 DEBUG } 22:20:56 DEBUG } 22:20:56 INFO 22:20:56 INFO [loop_until]: kubectl --namespace=xlou top node 22:20:56 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:20:56 INFO [loop_until]: OK (rc = 0) 22:20:56 DEBUG --- stdout --- 22:20:56 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 63m 0% 5093Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5912m 19% 14717Mi 13% gke-xlou-cdm-default-pool-1b0769c8-nhlf 9363m 31% 10811Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 84m 0% 2635Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 4666m 15% 6439Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 7069m 23% 16600Mi 14% gke-xlou-cdm-default-pool-1b0769c8-xnmp 69m 0% 3149Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 9024m 30% 10370Mi 9% gke-xlou-cdm-ds-0f601751-hmm7 4714m 15% 5579Mi 4% gke-xlou-cdm-ds-0f601751-j488 5808m 19% 14058Mi 12% gke-xlou-cdm-ds-0f601751-mc5d 10606m 35% 3343Mi 2% gke-xlou-cdm-ds-0f601751-nj6q 9352m 31% 10411Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 4700m 15% 7359Mi 6% gke-xlou-cdm-frontend-a8771548-i82n 6420m 40% 5949Mi 10% 22:20:56 DEBUG --- stderr --- 22:20:56 DEBUG 22:21:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302515" 22:21:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713302515" 22:21:55 ERROR [http_cmd]: ERROR 22:21:55 INFO [http_cmd]: http status code OK 22:21:55 ERROR --- status code --- 22:21:55 DEBUG --- status code --- 22:21:55 ERROR http status code is 400 (expected [200, 401, 403, 404]) 22:21:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:21:55 ERROR --- http response --- 22:21:55 DEBUG --- http response --- 22:21:55 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 22:21:55 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 22:21:55 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 22:21:55 INFO 22:21:55 INFO [loop_until]: kubectl --namespace=xlou top pods 22:21:55 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:21:55 INFO [loop_until]: OK (rc = 0) 22:21:55 DEBUG --- stdout --- 22:21:55 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 9Mi am-555f4db595-4lbss 9131m 8587Mi am-555f4db595-fz8sn 9406m 8599Mi am-555f4db595-kzxsp 9224m 8552Mi ds-cts-0 6935m 12069Mi ds-cts-1 6478m 12159Mi ds-cts-2 5970m 12120Mi ds-idrepo-0 4755m 5032Mi ds-idrepo-1 4422m 2833Mi ds-idrepo-2 4370m 3101Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 4m 1207Mi idm-b85f9d978-rl76f 4m 1226Mi lodemon-bc756c647-cr7x6 14m 63Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-7c9b667974-9mr2s 6244m 520Mi 22:21:55 DEBUG --- stderr --- 22:21:55 DEBUG 22:21:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302516" 22:21:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302516" 22:21:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302516" 22:21:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302516" 22:21:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713302516" 22:21:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302516" 22:21:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302516" 22:21:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302516" 22:21:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302516" 22:21:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302516" 22:21:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713302516" 22:21:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302516" 22:21:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713302516" 22:21:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713302516" 22:21:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302516" 22:21:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302516" 22:21:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302516" 22:21:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302516" 22:21:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302516" 22:21:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302516" 22:21:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302516" 22:21:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302516" 22:21:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302516" 22:21:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302516" 22:21:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302516" 22:21:56 INFO [http_cmd]: http status code OK 22:21:56 INFO [http_cmd]: http status code OK 22:21:56 INFO [http_cmd]: http status code OK 22:21:56 INFO [http_cmd]: http status code OK 22:21:56 INFO [http_cmd]: http status code OK 22:21:56 INFO [http_cmd]: http status code OK 22:21:56 INFO [http_cmd]: http status code OK 22:21:56 INFO [http_cmd]: http status code OK 22:21:56 INFO [http_cmd]: http status code OK 22:21:56 INFO [http_cmd]: http status code OK 22:21:56 INFO [http_cmd]: http status code OK 22:21:56 INFO [http_cmd]: http status code OK 22:21:56 INFO [http_cmd]: http status code OK 22:21:56 INFO [http_cmd]: http status code OK 22:21:56 INFO [http_cmd]: http status code OK 22:21:56 INFO [http_cmd]: http status code OK 22:21:56 DEBUG --- status code --- 22:21:56 INFO [http_cmd]: http status code OK 22:21:56 DEBUG --- status code --- 22:21:56 INFO [http_cmd]: http status code OK 22:21:56 INFO [http_cmd]: http status code OK 22:21:56 INFO [http_cmd]: http status code OK 22:21:56 INFO [http_cmd]: http status code OK 22:21:56 DEBUG --- status code --- 22:21:56 DEBUG --- status code --- 22:21:56 DEBUG --- status code --- 22:21:56 DEBUG --- status code --- 22:21:56 DEBUG --- status code --- 22:21:56 DEBUG --- status code --- 22:21:56 DEBUG --- status code --- 22:21:56 DEBUG --- status code --- 22:21:56 DEBUG --- status code --- 22:21:56 DEBUG --- status code --- 22:21:56 DEBUG --- status code --- 22:21:56 DEBUG --- status code --- 22:21:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:21:56 DEBUG --- status code --- 22:21:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:21:56 DEBUG --- status code --- 22:21:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:21:56 DEBUG --- status code --- 22:21:56 DEBUG --- status code --- 22:21:56 DEBUG --- status code --- 22:21:56 DEBUG --- status code --- 22:21:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:21:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:21:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:21:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:21:56 INFO [http_cmd]: http status code OK 22:21:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:21:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:21:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:21:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:21:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:21:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:21:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:21:56 DEBUG --- status code --- 22:21:56 DEBUG --- http response --- 22:21:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:21:56 DEBUG --- http response --- 22:21:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:21:56 DEBUG --- http response --- 22:21:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:21:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:21:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:21:56 INFO [http_cmd]: http status code OK 22:21:56 INFO [http_cmd]: http status code OK 22:21:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:21:56 DEBUG --- http response --- 22:21:56 DEBUG --- http response --- 22:21:56 DEBUG --- http response --- 22:21:56 DEBUG --- http response --- 22:21:56 DEBUG --- status code --- 22:21:56 DEBUG --- http response --- 22:21:56 DEBUG --- http response --- 22:21:56 DEBUG --- http response --- 22:21:56 DEBUG --- http response --- 22:21:56 DEBUG --- http response --- 22:21:56 DEBUG --- http response --- 22:21:56 DEBUG --- http response --- 22:21:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:21:56 DEBUG --- http response --- 22:21:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:21:56 DEBUG --- http response --- 22:21:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:21:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:21:56 INFO [http_cmd]: http status code OK 22:21:56 DEBUG --- http response --- 22:21:56 DEBUG --- http response --- 22:21:56 DEBUG --- http response --- 22:21:56 DEBUG --- status code --- 22:21:56 DEBUG --- status code --- 22:21:56 DEBUG --- http response --- 22:21:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:21:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:21:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:21:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:21:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:21:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:21:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:21:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:21:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:21:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:21:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:21:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:21:56 DEBUG --- http response --- 22:21:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:21:56 DEBUG ----- output ----- 22:21:56 DEBUG ----- output ----- 22:21:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:21:56 DEBUG ----- output ----- 22:21:56 DEBUG { 22:21:56 DEBUG --- status code --- 22:21:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:21:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:21:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:21:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:21:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:21:56 DEBUG ----- output ----- 22:21:56 DEBUG ----- output ----- 22:21:56 DEBUG ----- output ----- 22:21:56 DEBUG ----- output ----- 22:21:56 DEBUG --- http response --- 22:21:56 DEBUG ----- output ----- 22:21:56 DEBUG ----- output ----- 22:21:56 DEBUG ----- output ----- 22:21:56 DEBUG ----- output ----- 22:21:56 DEBUG ----- output ----- 22:21:56 DEBUG ----- output ----- 22:21:56 DEBUG ----- output ----- 22:21:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:21:56 DEBUG ----- output ----- 22:21:56 DEBUG { 22:21:56 DEBUG { 22:21:56 DEBUG ----- output ----- 22:21:56 DEBUG "status": "success", 22:21:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:21:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:21:56 DEBUG ----- output ----- 22:21:56 DEBUG ----- output ----- 22:21:56 DEBUG --- http response --- 22:21:56 DEBUG --- http response --- 22:21:56 DEBUG ----- output ----- 22:21:56 DEBUG { 22:21:56 DEBUG { 22:21:56 DEBUG "status": "success", 22:21:56 DEBUG { 22:21:56 DEBUG { 22:21:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:21:56 DEBUG { 22:21:56 DEBUG { 22:21:56 DEBUG { 22:21:56 DEBUG { 22:21:56 DEBUG { 22:21:56 DEBUG { 22:21:56 DEBUG ----- output ----- 22:21:56 DEBUG { 22:21:56 DEBUG "status": "success", 22:21:56 DEBUG "status": "success", 22:21:56 DEBUG { 22:21:56 DEBUG "data": { 22:21:56 DEBUG ----- output ----- 22:21:56 DEBUG --- http response --- 22:21:56 DEBUG { 22:21:56 DEBUG { 22:21:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:21:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:21:56 DEBUG { 22:21:56 DEBUG "status": "success", 22:21:56 DEBUG { 22:21:56 DEBUG "data": { 22:21:56 DEBUG "status": "success", 22:21:56 DEBUG "status": "success", 22:21:56 DEBUG ----- output ----- 22:21:56 DEBUG "status": "success", 22:21:56 DEBUG "status": "success", 22:21:56 DEBUG "status": "success", 22:21:56 DEBUG "status": "success", 22:21:56 DEBUG "status": "success", 22:21:56 DEBUG "status": "success", 22:21:56 DEBUG { 22:21:56 DEBUG "status": "success", 22:21:56 DEBUG "data": { 22:21:56 DEBUG "data": { 22:21:56 DEBUG "status": "success", 22:21:56 DEBUG "resultType": "vector", 22:21:56 DEBUG { 22:21:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:21:56 DEBUG "status": "success", 22:21:56 DEBUG "status": "success", 22:21:56 DEBUG ----- output ----- 22:21:56 DEBUG ----- output ----- 22:21:56 DEBUG "status": "success", 22:21:56 DEBUG "data": { 22:21:56 DEBUG "status": "success", 22:21:56 DEBUG "resultType": "vector", 22:21:56 DEBUG "data": { 22:21:56 DEBUG "data": { 22:21:56 DEBUG { 22:21:56 DEBUG "data": { 22:21:56 DEBUG "data": { 22:21:56 DEBUG "data": { 22:21:56 DEBUG "data": { 22:21:56 DEBUG "data": { 22:21:56 DEBUG "data": { 22:21:56 DEBUG "status": "success", 22:21:56 DEBUG "data": { 22:21:56 DEBUG "resultType": "vector", 22:21:56 DEBUG "resultType": "vector", 22:21:56 DEBUG "data": { 22:21:56 DEBUG "result": [ 22:21:56 DEBUG "status": "success", 22:21:56 DEBUG ----- output ----- 22:21:56 DEBUG "data": { 22:21:56 DEBUG "data": { 22:21:56 DEBUG { 22:21:56 DEBUG { 22:21:56 DEBUG "data": { 22:21:56 DEBUG "resultType": "vector", 22:21:56 DEBUG "data": { 22:21:56 DEBUG "result": [ 22:21:56 DEBUG "resultType": "vector", 22:21:56 DEBUG "resultType": "vector", 22:21:56 DEBUG "result": [ 22:21:56 DEBUG "resultType": "vector", 22:21:56 DEBUG "resultType": "vector", 22:21:56 DEBUG "resultType": "vector", 22:21:56 DEBUG "resultType": "vector", 22:21:56 DEBUG "resultType": "vector", 22:21:56 DEBUG "resultType": "vector", 22:21:56 DEBUG "data": { 22:21:56 DEBUG "resultType": "vector", 22:21:56 DEBUG "result": [ 22:21:56 DEBUG "result": [ 22:21:56 DEBUG "resultType": "vector", 22:21:56 DEBUG { 22:21:56 DEBUG "data": { 22:21:56 DEBUG { 22:21:56 DEBUG "resultType": "vector", 22:21:56 DEBUG "resultType": "vector", 22:21:56 DEBUG "status": "success", 22:21:56 DEBUG "status": "success", 22:21:56 DEBUG "resultType": "vector", 22:21:56 DEBUG "result": [ 22:21:56 DEBUG "resultType": "vector", 22:21:56 DEBUG { 22:21:56 DEBUG "result": [ 22:21:56 DEBUG "status": "success", 22:21:56 DEBUG { 22:21:56 DEBUG "result": [ 22:21:56 DEBUG "result": [ 22:21:56 DEBUG "result": [ 22:21:56 DEBUG "result": [ 22:21:56 DEBUG "result": [ 22:21:56 DEBUG "result": [ 22:21:56 DEBUG "resultType": "vector", 22:21:56 DEBUG "result": [ 22:21:56 DEBUG { 22:21:56 DEBUG { 22:21:56 DEBUG "result": [ 22:21:56 DEBUG "metric": { 22:21:56 DEBUG "resultType": "vector", 22:21:56 DEBUG "status": "success", 22:21:56 DEBUG "result": [ 22:21:56 DEBUG "result": [ 22:21:56 DEBUG "data": { 22:21:56 DEBUG "data": { 22:21:56 DEBUG "result": [ 22:21:56 DEBUG { 22:21:56 DEBUG "result": [ 22:21:56 DEBUG "metric": { 22:21:56 DEBUG { 22:21:56 DEBUG "data": { 22:21:56 DEBUG "metric": { 22:21:56 DEBUG { 22:21:56 DEBUG { 22:21:56 DEBUG { 22:21:56 DEBUG { 22:21:56 DEBUG { 22:21:56 DEBUG { 22:21:56 DEBUG "result": [ 22:21:56 DEBUG { 22:21:56 DEBUG "metric": { 22:21:56 DEBUG "metric": { 22:21:56 DEBUG { 22:21:56 DEBUG "metric": { 22:21:56 DEBUG "instance": "10.142.0.107:9100" 22:21:56 DEBUG "data": { 22:21:56 DEBUG { 22:21:56 DEBUG { 22:21:56 DEBUG "resultType": "vector", 22:21:56 DEBUG "resultType": "vector", 22:21:56 DEBUG { 22:21:56 DEBUG "metric": { 22:21:56 DEBUG { 22:21:56 DEBUG "pod": "ds-idrepo-2" 22:21:56 DEBUG }, 22:21:56 DEBUG "resultType": "vector", 22:21:56 DEBUG "pod": "am-555f4db595-fz8sn" 22:21:56 DEBUG "metric": { 22:21:56 DEBUG "metric": { 22:21:56 DEBUG "metric": { 22:21:56 DEBUG "metric": { 22:21:56 DEBUG "metric": { 22:21:56 DEBUG "metric": { 22:21:56 DEBUG { 22:21:56 DEBUG "metric": { 22:21:56 DEBUG "pod": "ds-idrepo-2" 22:21:56 DEBUG "pod": "am-555f4db595-fz8sn" 22:21:56 DEBUG "result": [ 22:21:56 DEBUG "pod": "ds-idrepo-2" 22:21:56 DEBUG }, 22:21:56 DEBUG "resultType": "vector", 22:21:56 DEBUG "metric": { 22:21:56 DEBUG "metric": { 22:21:56 DEBUG "result": [ 22:21:56 DEBUG "result": [ 22:21:56 DEBUG "metric": { 22:21:56 DEBUG "pod": "am-555f4db595-fz8sn" 22:21:56 DEBUG "metric": { 22:21:56 DEBUG "metric": { 22:21:56 DEBUG "value": [ 22:21:56 DEBUG "result": [ 22:21:56 DEBUG { 22:21:56 DEBUG }, 22:21:56 DEBUG "pod": "am-555f4db595-fz8sn" 22:21:56 DEBUG "pod": "ds-idrepo-2" 22:21:56 DEBUG "pod": "am-555f4db595-fz8sn" 22:21:56 DEBUG "pod": "am-555f4db595-fz8sn" 22:21:56 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:21:56 DEBUG "metric": { 22:21:56 DEBUG "instance": "10.142.0.107:9100" 22:21:56 DEBUG }, 22:21:56 DEBUG "value": [ 22:21:56 DEBUG { 22:21:56 DEBUG }, 22:21:56 DEBUG "value": [ 22:21:56 DEBUG "result": [ 22:21:56 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:21:56 DEBUG "pod": "ds-idrepo-2" 22:21:56 DEBUG { 22:21:56 DEBUG { 22:21:56 DEBUG "pod": "ds-idrepo-2" 22:21:56 DEBUG }, 22:21:56 DEBUG "pod": "am-555f4db595-fz8sn" 22:21:56 DEBUG "instance": "10.142.0.107:9100" 22:21:56 DEBUG 22:21:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:21:56 DEBUG "metric": { 22:21:56 DEBUG "value": [ 22:21:56 DEBUG }, 22:21:56 DEBUG }, 22:21:56 DEBUG }, 22:21:56 DEBUG }, 22:21:56 DEBUG }, 22:21:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:21:56 DEBUG }, 22:21:56 DEBUG }, 22:21:56 DEBUG 22:21:56 DEBUG "metric": { 22:21:56 DEBUG "value": [ 22:21:56 DEBUG 22:21:56 DEBUG { 22:21:56 DEBUG }, 22:21:56 DEBUG }, 22:21:56 DEBUG "metric": { 22:21:56 DEBUG "metric": { 22:21:56 DEBUG }, 22:21:56 DEBUG "value": [ 22:21:56 DEBUG }, 22:21:56 DEBUG }, 22:21:56 DEBUG 22:21:56 DEBUG }, 22:21:56 DEBUG "instance": "10.142.0.107:9100" 22:21:56 DEBUG 22:21:56 DEBUG "value": [ 22:21:56 DEBUG "value": [ 22:21:56 DEBUG "value": [ 22:21:56 DEBUG "value": [ 22:21:56 DEBUG "value": [ 22:21:56 DEBUG }, 22:21:56 DEBUG "value": [ 22:21:56 DEBUG "value": [ 22:21:56 DEBUG 22:21:56 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:21:56 INFO 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG "metric": { 22:21:56 DEBUG "value": [ 22:21:56 DEBUG "value": [ 22:21:56 DEBUG "pod": "ds-idrepo-2" 22:21:56 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:21:56 DEBUG "value": [ 22:21:56 DEBUG 22:21:56 DEBUG "value": [ 22:21:56 DEBUG "value": [ 22:21:56 DEBUG 22:21:56 DEBUG "value": [ 22:21:56 DEBUG }, 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG "value": [ 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG }, 22:21:56 INFO [loop_until]: kubectl --namespace=xlou top node 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG "instance": "10.142.0.107:9100" 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG }, 22:21:56 DEBUG }, 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG [...] 22:21:56 DEBUG 22:21:56 DEBUG "value": [ 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG [...] 22:21:56 DEBUG "value": [ 22:21:56 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:21:56 DEBUG 22:21:56 DEBUG [...] 22:21:56 DEBUG }, 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG "value": [ 22:21:56 DEBUG "value": [ 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG [...] 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG [...] 22:21:56 DEBUG 22:21:56 DEBUG "value": [ 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG [...] 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG [...] 22:21:56 DEBUG [...] 22:21:56 DEBUG [...] 22:21:56 DEBUG [...] 22:21:56 DEBUG [...] 22:21:56 DEBUG 22:21:56 DEBUG [...] 22:21:56 DEBUG [...] 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG [...] 22:21:56 DEBUG [...] 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG [...] 22:21:56 DEBUG 22:21:56 DEBUG [...] 22:21:56 DEBUG [...] 22:21:56 DEBUG 22:21:56 DEBUG [...] 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG [...] 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG "pod": "ds-idrepo-0" 22:21:56 DEBUG 22:21:56 DEBUG [...] 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG "pod": "ds-cts-2" 22:21:56 DEBUG [...] 22:21:56 DEBUG 22:21:56 DEBUG "instance": "10.142.0.43:9100" 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG [...] 22:21:56 DEBUG [...] 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG }, 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG }, 22:21:56 DEBUG 22:21:56 DEBUG "pod": "ds-idrepo-0" 22:21:56 DEBUG }, 22:21:56 DEBUG [...] 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG "pod": "am-555f4db595-kzxsp" 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG "value": [ 22:21:56 DEBUG 22:21:56 DEBUG "pod": "am-555f4db595-kzxsp" 22:21:56 DEBUG 22:21:56 DEBUG "pod": "am-555f4db595-kzxsp" 22:21:56 DEBUG "pod": "ds-idrepo-0" 22:21:56 DEBUG "pod": "am-555f4db595-kzxsp" 22:21:56 DEBUG "pod": "am-555f4db595-kzxsp" 22:21:56 DEBUG "pod": "overseer-0-7c9b667974-9mr2s" 22:21:56 DEBUG 22:21:56 DEBUG "instance": "10.142.0.43:9100" 22:21:56 DEBUG "pod": "am-555f4db595-kzxsp" 22:21:56 DEBUG "value": [ 22:21:56 DEBUG 22:21:56 DEBUG }, 22:21:56 DEBUG "value": [ 22:21:56 DEBUG 22:21:56 DEBUG "pod": "overseer-0-7c9b667974-9mr2s" 22:21:56 DEBUG "pod": "ds-idrepo-0" 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG "pod": "ds-idrepo-0" 22:21:56 DEBUG }, 22:21:56 DEBUG "pod": "am-555f4db595-kzxsp" 22:21:56 DEBUG "instance": "10.142.0.43:9100" 22:21:56 DEBUG 1713302516, 22:21:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:21:56 DEBUG }, 22:21:56 DEBUG "instance": "10.142.0.43:9100" 22:21:56 DEBUG }, 22:21:56 DEBUG }, 22:21:56 DEBUG }, 22:21:56 DEBUG }, 22:21:56 DEBUG }, 22:21:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:21:56 DEBUG }, 22:21:56 DEBUG }, 22:21:56 DEBUG 1713302516, 22:21:56 DEBUG 22:21:56 DEBUG "value": [ 22:21:56 DEBUG 1713302516, 22:21:56 DEBUG 22:21:56 DEBUG }, 22:21:56 DEBUG }, 22:21:56 DEBUG 22:21:56 DEBUG 22:21:56 DEBUG }, 22:21:56 DEBUG "value": [ 22:21:56 DEBUG }, 22:21:56 DEBUG }, 22:21:56 DEBUG "0" 22:21:56 DEBUG }, 22:21:56 DEBUG "value": [ 22:21:56 DEBUG }, 22:21:56 DEBUG "value": [ 22:21:56 DEBUG "value": [ 22:21:56 DEBUG "value": [ 22:21:56 DEBUG "value": [ 22:21:56 DEBUG "value": [ 22:21:56 DEBUG }, 22:21:56 DEBUG "value": [ 22:21:56 DEBUG "value": [ 22:21:56 DEBUG "0" 22:21:56 DEBUG "pod": "overseer-0-7c9b667974-9mr2s" 22:21:56 DEBUG 1713302516, 22:21:56 DEBUG "45265778.5" 22:21:56 DEBUG 22:21:56 DEBUG "value": [ 22:21:56 DEBUG "value": [ 22:21:56 DEBUG "pod": "ds-idrepo-0" 22:21:56 DEBUG "pod": "overseer-0-7c9b667974-9mr2s" 22:21:56 DEBUG "value": [ 22:21:56 DEBUG 1713302516, 22:21:56 DEBUG "value": [ 22:21:56 DEBUG "value": [ 22:21:56 DEBUG ] 22:21:56 DEBUG "value": [ 22:21:56 DEBUG 1713302516, 22:21:56 DEBUG "value": [ 22:21:56 DEBUG 1713302516, 22:21:56 DEBUG "1993.2" 22:21:56 DEBUG 1713302516, 22:21:56 DEBUG 1713302516, 22:21:56 DEBUG 1713302516, 22:21:56 DEBUG "value": [ 22:21:56 DEBUG 1713302516, 22:21:56 DEBUG 1713302516, 22:21:56 DEBUG ] 22:21:56 DEBUG }, 22:21:56 DEBUG "0" 22:21:56 DEBUG ] 22:21:56 DEBUG "instance": "10.142.0.43:9100" 22:21:56 DEBUG 1713302516, 22:21:56 DEBUG 1713302516, 22:21:56 DEBUG }, 22:21:56 DEBUG }, 22:21:56 DEBUG 1713302516, 22:21:56 DEBUG "0" 22:21:56 DEBUG 1713302516, 22:21:56 DEBUG 1713302516, 22:21:56 DEBUG } 22:21:56 DEBUG 1713302516, 22:21:56 DEBUG "0" 22:21:56 DEBUG 1713302516, 22:21:56 DEBUG 1713302516, 22:21:56 DEBUG ] 22:21:56 DEBUG "1993" 22:21:56 DEBUG "0.0235" 22:21:56 DEBUG "0" 22:21:56 DEBUG 1713302516, 22:21:56 DEBUG "19952405504" 22:21:56 DEBUG "1220073" 22:21:56 DEBUG } 22:21:56 DEBUG "value": [ 22:21:56 DEBUG ] 22:21:56 DEBUG } 22:21:56 DEBUG }, 22:21:56 DEBUG "3.319075028341138" 22:21:56 DEBUG "0" 22:21:56 DEBUG "value": [ 22:21:56 DEBUG "value": [ 22:21:56 DEBUG "1993.0666666666666" 22:21:56 DEBUG ] 22:21:56 DEBUG "0" 22:21:56 DEBUG "0.0008666666666670153" 22:21:56 DEBUG ] 22:21:56 DEBUG "643944448" 22:21:56 DEBUG ] 22:21:56 DEBUG "45275086.8" 22:21:56 DEBUG "0" 22:21:56 DEBUG } 22:21:56 DEBUG ] 22:21:56 DEBUG ] 22:21:56 DEBUG ] 22:21:56 DEBUG "6.267143605891801" 22:21:56 DEBUG ] 22:21:56 DEBUG ] 22:21:56 DEBUG ] 22:21:56 DEBUG 1713302516, 22:21:56 DEBUG } 22:21:56 DEBUG ] 22:21:56 DEBUG "value": [ 22:21:56 DEBUG 1713302516, 22:21:56 DEBUG ] 22:21:56 DEBUG 1713302516, 22:21:56 DEBUG 1713302516, 22:21:56 DEBUG "0" 22:21:56 DEBUG } 22:21:56 DEBUG ] 22:21:56 DEBUG } 22:21:56 DEBUG ] 22:21:56 DEBUG } 22:21:56 DEBUG } 22:21:56 DEBUG ] 22:21:56 DEBUG ] 22:21:56 DEBUG ] 22:21:56 DEBUG } 22:21:56 DEBUG } 22:21:56 DEBUG } 22:21:56 DEBUG ] 22:21:56 DEBUG } 22:21:56 DEBUG } 22:21:56 DEBUG } 22:21:56 DEBUG "16693.569334979267" 22:21:56 DEBUG ] 22:21:56 DEBUG } 22:21:56 DEBUG ] 22:21:56 DEBUG "0.0003333333333333262" 22:21:56 DEBUG } 22:21:56 DEBUG "0" 22:21:56 DEBUG ] 22:21:56 DEBUG ] 22:21:56 DEBUG ] 22:21:56 DEBUG } 22:21:56 DEBUG ] 22:21:56 DEBUG } 22:21:56 DEBUG ] 22:21:56 DEBUG ] 22:21:56 DEBUG } 22:21:56 DEBUG } 22:21:56 DEBUG } 22:21:56 DEBUG ] 22:21:56 DEBUG ] 22:21:56 DEBUG ] 22:21:56 DEBUG } 22:21:56 DEBUG ] 22:21:56 DEBUG ] 22:21:56 DEBUG } 22:21:56 DEBUG ] 22:21:56 DEBUG } 22:21:56 DEBUG } 22:21:56 DEBUG } 22:21:56 DEBUG ] 22:21:56 DEBUG ] 22:21:56 DEBUG ] 22:21:56 DEBUG } 22:21:56 DEBUG } 22:21:56 DEBUG } 22:21:56 DEBUG ] 22:21:56 DEBUG } 22:21:56 DEBUG } 22:21:56 DEBUG } 22:21:56 DEBUG ] 22:21:56 DEBUG ] 22:21:56 DEBUG } 22:21:56 DEBUG } 22:21:56 DEBUG } 22:21:56 DEBUG } 22:21:56 DEBUG ] 22:21:56 DEBUG } 22:21:56 DEBUG } 22:21:56 DEBUG } 22:21:56 DEBUG } 22:21:56 DEBUG ] 22:21:56 DEBUG } 22:21:56 DEBUG } 22:21:56 DEBUG } 22:21:56 DEBUG ] 22:21:56 DEBUG ] 22:21:56 DEBUG } 22:21:56 DEBUG } 22:21:56 DEBUG } 22:21:56 DEBUG ] 22:21:56 DEBUG } 22:21:56 DEBUG } 22:21:56 DEBUG } 22:21:56 DEBUG } 22:21:56 DEBUG } 22:21:56 DEBUG } 22:21:56 DEBUG } 22:21:56 DEBUG } 22:21:56 DEBUG } 22:21:56 DEBUG ] 22:21:56 DEBUG } 22:21:56 DEBUG ] 22:21:56 DEBUG } 22:21:56 DEBUG ] 22:21:56 DEBUG } 22:21:56 DEBUG } 22:21:56 DEBUG } 22:21:56 DEBUG } 22:21:56 DEBUG } 22:21:56 DEBUG } 22:21:56 DEBUG } 22:21:56 DEBUG } 22:21:56 DEBUG } 22:21:56 DEBUG } 22:21:56 DEBUG } 22:21:56 DEBUG } 22:21:56 DEBUG } 22:21:56 DEBUG } 22:21:56 DEBUG } 22:21:56 DEBUG } 22:21:56 DEBUG } 22:21:56 INFO [loop_until]: OK (rc = 0) 22:21:56 DEBUG --- stdout --- 22:21:56 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 65m 0% 5101Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 6472m 21% 14725Mi 13% gke-xlou-cdm-default-pool-1b0769c8-nhlf 9369m 31% 10817Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 87m 0% 2653Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 4730m 15% 6444Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 7670m 25% 16558Mi 14% gke-xlou-cdm-default-pool-1b0769c8-xnmp 64m 0% 3149Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 9421m 31% 10370Mi 9% gke-xlou-cdm-ds-0f601751-hmm7 4672m 15% 5589Mi 5% gke-xlou-cdm-ds-0f601751-j488 6304m 21% 14060Mi 12% gke-xlou-cdm-ds-0f601751-mc5d 10574m 35% 3356Mi 3% gke-xlou-cdm-ds-0f601751-nj6q 9410m 31% 10427Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 4697m 15% 7359Mi 6% gke-xlou-cdm-frontend-a8771548-i82n 6409m 40% 5948Mi 10% 22:21:56 DEBUG --- stderr --- 22:21:56 DEBUG 22:22:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713302575" 22:22:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302575" 22:22:55 ERROR [http_cmd]: ERROR 22:22:55 INFO [http_cmd]: http status code OK 22:22:55 ERROR --- status code --- 22:22:55 DEBUG --- status code --- 22:22:55 ERROR http status code is 400 (expected [200, 401, 403, 404]) 22:22:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:22:55 ERROR --- http response --- 22:22:55 DEBUG --- http response --- 22:22:55 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 22:22:55 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 22:22:55 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 22:22:55 INFO 22:22:55 INFO [loop_until]: kubectl --namespace=xlou top pods 22:22:55 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:22:56 INFO [loop_until]: OK (rc = 0) 22:22:56 DEBUG --- stdout --- 22:22:56 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 9Mi am-555f4db595-4lbss 8946m 8588Mi am-555f4db595-fz8sn 9338m 8600Mi am-555f4db595-kzxsp 9053m 8552Mi ds-cts-0 7497m 12075Mi ds-cts-1 6666m 12166Mi ds-cts-2 6768m 12127Mi ds-idrepo-0 4750m 5031Mi ds-idrepo-1 4475m 2834Mi ds-idrepo-2 4506m 3116Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 4m 1207Mi idm-b85f9d978-rl76f 4m 1226Mi lodemon-bc756c647-cr7x6 24m 63Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-7c9b667974-9mr2s 6154m 522Mi 22:22:56 DEBUG --- stderr --- 22:22:56 DEBUG 22:22:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302576" 22:22:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713302576" 22:22:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302576" 22:22:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302576" 22:22:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302576" 22:22:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302576" 22:22:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302576" 22:22:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302576" 22:22:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302576" 22:22:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713302576" 22:22:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302576" 22:22:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302576" 22:22:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302576" 22:22:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302576" 22:22:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302576" 22:22:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302576" 22:22:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713302576" 22:22:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302576" 22:22:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302576" 22:22:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302576" 22:22:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302576" 22:22:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302576" 22:22:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713302576" 22:22:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302576" 22:22:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302576" 22:22:56 INFO [http_cmd]: http status code OK 22:22:56 INFO [http_cmd]: http status code OK 22:22:56 INFO [http_cmd]: http status code OK 22:22:56 INFO [http_cmd]: http status code OK 22:22:56 INFO [http_cmd]: http status code OK 22:22:56 INFO [http_cmd]: http status code OK 22:22:56 INFO [http_cmd]: http status code OK 22:22:56 INFO [http_cmd]: http status code OK 22:22:56 INFO [http_cmd]: http status code OK 22:22:56 INFO [http_cmd]: http status code OK 22:22:56 INFO [http_cmd]: http status code OK 22:22:56 INFO [http_cmd]: http status code OK 22:22:56 INFO 22:22:56 INFO [http_cmd]: http status code OK 22:22:56 DEBUG --- status code --- 22:22:56 INFO [http_cmd]: http status code OK 22:22:56 DEBUG --- status code --- 22:22:56 INFO [http_cmd]: http status code OK 22:22:56 INFO [http_cmd]: http status code OK 22:22:56 DEBUG --- status code --- 22:22:56 INFO [http_cmd]: http status code OK 22:22:56 INFO [http_cmd]: http status code OK 22:22:56 DEBUG --- status code --- 22:22:56 INFO [http_cmd]: http status code OK 22:22:56 INFO [http_cmd]: http status code OK 22:22:56 INFO [http_cmd]: http status code OK 22:22:56 INFO [http_cmd]: http status code OK 22:22:56 DEBUG --- status code --- 22:22:56 DEBUG --- status code --- 22:22:56 INFO [http_cmd]: http status code OK 22:22:56 DEBUG --- status code --- 22:22:56 DEBUG --- status code --- 22:22:56 DEBUG --- status code --- 22:22:56 DEBUG --- status code --- 22:22:56 DEBUG --- status code --- 22:22:56 DEBUG --- status code --- 22:22:56 INFO [loop_until]: kubectl --namespace=xlou top node 22:22:56 DEBUG --- status code --- 22:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:22:56 DEBUG --- status code --- 22:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:22:56 INFO [http_cmd]: http status code OK 22:22:56 DEBUG --- status code --- 22:22:56 DEBUG --- status code --- 22:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:22:56 DEBUG --- status code --- 22:22:56 DEBUG --- status code --- 22:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:22:56 DEBUG --- status code --- 22:22:56 DEBUG --- status code --- 22:22:56 DEBUG --- status code --- 22:22:56 DEBUG --- status code --- 22:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:22:56 DEBUG --- status code --- 22:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:22:56 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:22:56 INFO [http_cmd]: http status code OK 22:22:56 DEBUG --- http response --- 22:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:22:56 DEBUG --- http response --- 22:22:56 DEBUG --- status code --- 22:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:22:56 DEBUG --- http response --- 22:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:22:56 DEBUG --- http response --- 22:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:22:56 DEBUG --- http response --- 22:22:56 DEBUG --- http response --- 22:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:22:56 DEBUG --- http response --- 22:22:56 DEBUG --- http response --- 22:22:56 DEBUG --- http response --- 22:22:56 DEBUG --- http response --- 22:22:56 DEBUG --- http response --- 22:22:56 DEBUG --- http response --- 22:22:56 DEBUG --- http response --- 22:22:56 DEBUG --- status code --- 22:22:56 DEBUG --- http response --- 22:22:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:22:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:22:56 DEBUG --- http response --- 22:22:56 DEBUG --- http response --- 22:22:56 DEBUG --- http response --- 22:22:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:22:56 DEBUG --- http response --- 22:22:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:22:56 DEBUG --- http response --- 22:22:56 DEBUG --- http response --- 22:22:56 DEBUG --- http response --- 22:22:56 DEBUG --- http response --- 22:22:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:22:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:22:56 DEBUG --- http response --- 22:22:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:22:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:22:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:22:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:22:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:22:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:22:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:22:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:22:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:22:56 DEBUG ----- output ----- 22:22:56 DEBUG ----- output ----- 22:22:56 DEBUG --- http response --- 22:22:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:22:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:22:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:22:56 DEBUG ----- output ----- 22:22:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:22:56 DEBUG ----- output ----- 22:22:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:22:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:22:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:22:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:22:56 DEBUG ----- output ----- 22:22:56 DEBUG ----- output ----- 22:22:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:22:56 DEBUG ----- output ----- 22:22:56 DEBUG ----- output ----- 22:22:56 DEBUG ----- output ----- 22:22:56 DEBUG ----- output ----- 22:22:56 DEBUG ----- output ----- 22:22:56 DEBUG ----- output ----- 22:22:56 DEBUG ----- output ----- 22:22:56 DEBUG --- http response --- 22:22:56 DEBUG ----- output ----- 22:22:56 DEBUG { 22:22:56 DEBUG { 22:22:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:22:56 DEBUG ----- output ----- 22:22:56 DEBUG ----- output ----- 22:22:56 DEBUG ----- output ----- 22:22:56 DEBUG { 22:22:56 DEBUG ----- output ----- 22:22:56 DEBUG { 22:22:56 DEBUG ----- output ----- 22:22:56 DEBUG ----- output ----- 22:22:56 DEBUG ----- output ----- 22:22:56 DEBUG ----- output ----- 22:22:56 DEBUG { 22:22:56 DEBUG { 22:22:56 DEBUG ----- output ----- 22:22:56 DEBUG { 22:22:56 DEBUG { 22:22:56 DEBUG { 22:22:56 DEBUG "status": "success", 22:22:56 DEBUG { 22:22:56 DEBUG { 22:22:56 DEBUG { 22:22:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:22:56 DEBUG { 22:22:56 DEBUG "status": "success", 22:22:56 DEBUG "status": "success", 22:22:56 DEBUG ----- output ----- 22:22:56 DEBUG { 22:22:56 DEBUG { 22:22:56 DEBUG { 22:22:56 DEBUG "status": "success", 22:22:56 DEBUG { 22:22:56 DEBUG "status": "success", 22:22:56 DEBUG { 22:22:56 DEBUG { 22:22:56 DEBUG { 22:22:56 DEBUG { 22:22:56 DEBUG "status": "success", 22:22:56 DEBUG "status": "success", 22:22:56 DEBUG { 22:22:56 DEBUG "status": "success", 22:22:56 DEBUG "status": "success", 22:22:56 DEBUG { 22:22:56 DEBUG "data": { 22:22:56 DEBUG "status": "success", 22:22:56 DEBUG "status": "success", 22:22:56 DEBUG "status": "success", 22:22:56 DEBUG ----- output ----- 22:22:56 DEBUG "status": "success", 22:22:56 DEBUG "data": { 22:22:56 DEBUG "data": { 22:22:56 DEBUG { 22:22:56 DEBUG "status": "success", 22:22:56 DEBUG "status": "success", 22:22:56 DEBUG "status": "success", 22:22:56 DEBUG "data": { 22:22:56 DEBUG "status": "success", 22:22:56 DEBUG "data": { 22:22:56 DEBUG "status": "success", 22:22:56 DEBUG "status": "success", 22:22:56 DEBUG "status": "success", 22:22:56 DEBUG "status": "success", 22:22:56 DEBUG "data": { 22:22:56 DEBUG "data": { 22:22:56 DEBUG "status": "success", 22:22:56 DEBUG "data": { 22:22:56 DEBUG "data": { 22:22:56 DEBUG "status": "success", 22:22:56 DEBUG "resultType": "vector", 22:22:56 DEBUG "data": { 22:22:56 DEBUG "data": { 22:22:56 DEBUG "data": { 22:22:56 DEBUG { 22:22:56 DEBUG "data": { 22:22:56 DEBUG "resultType": "vector", 22:22:56 DEBUG "resultType": "vector", 22:22:56 DEBUG "status": "success", 22:22:56 DEBUG "data": { 22:22:56 DEBUG "data": { 22:22:56 DEBUG "data": { 22:22:56 DEBUG "resultType": "vector", 22:22:56 DEBUG "data": { 22:22:56 DEBUG "resultType": "vector", 22:22:56 DEBUG "data": { 22:22:56 DEBUG "data": { 22:22:56 DEBUG "data": { 22:22:56 DEBUG "data": { 22:22:56 DEBUG "resultType": "vector", 22:22:56 DEBUG "resultType": "vector", 22:22:56 DEBUG "data": { 22:22:56 DEBUG "resultType": "vector", 22:22:56 DEBUG "resultType": "vector", 22:22:56 DEBUG "data": { 22:22:56 DEBUG "result": [ 22:22:56 DEBUG "resultType": "vector", 22:22:56 DEBUG "resultType": "vector", 22:22:56 DEBUG "resultType": "vector", 22:22:56 DEBUG "status": "success", 22:22:56 DEBUG "resultType": "vector", 22:22:56 DEBUG "result": [ 22:22:56 DEBUG "result": [ 22:22:56 DEBUG "data": { 22:22:56 DEBUG "resultType": "vector", 22:22:56 DEBUG "resultType": "vector", 22:22:56 DEBUG "resultType": "vector", 22:22:56 DEBUG "result": [ 22:22:56 DEBUG "resultType": "vector", 22:22:56 DEBUG "result": [ 22:22:56 DEBUG "resultType": "vector", 22:22:56 DEBUG "resultType": "vector", 22:22:56 DEBUG "resultType": "vector", 22:22:56 DEBUG "resultType": "vector", 22:22:56 DEBUG "result": [ 22:22:56 DEBUG "result": [ 22:22:56 DEBUG "resultType": "vector", 22:22:56 DEBUG "result": [ 22:22:56 DEBUG "result": [ 22:22:56 DEBUG "resultType": "vector", 22:22:56 DEBUG { 22:22:56 DEBUG "result": [ 22:22:56 DEBUG "result": [ 22:22:56 DEBUG "result": [ 22:22:56 DEBUG "data": { 22:22:56 DEBUG "result": [ 22:22:56 DEBUG { 22:22:56 DEBUG { 22:22:56 DEBUG "resultType": "vector", 22:22:56 DEBUG "result": [ 22:22:56 DEBUG "result": [ 22:22:56 DEBUG "result": [ 22:22:56 DEBUG { 22:22:56 DEBUG "metric": { 22:22:56 DEBUG { 22:22:56 DEBUG "result": [ 22:22:56 DEBUG "result": [ 22:22:56 DEBUG "result": [ 22:22:56 DEBUG "result": [ 22:22:56 DEBUG { 22:22:56 DEBUG { 22:22:56 DEBUG "result": [ 22:22:56 DEBUG { 22:22:56 DEBUG { 22:22:56 DEBUG "result": [ 22:22:56 DEBUG "metric": { 22:22:56 DEBUG { 22:22:56 DEBUG { 22:22:56 DEBUG { 22:22:56 DEBUG "resultType": "vector", 22:22:56 DEBUG { 22:22:56 DEBUG "metric": { 22:22:56 DEBUG "metric": { 22:22:56 DEBUG "result": [ 22:22:56 DEBUG { 22:22:56 DEBUG { 22:22:56 DEBUG { 22:22:56 DEBUG "result": [ 22:22:56 DEBUG "pod": "am-555f4db595-fz8sn" 22:22:56 DEBUG "metric": { 22:22:56 DEBUG { 22:22:56 DEBUG { 22:22:56 DEBUG { 22:22:56 DEBUG { 22:22:56 DEBUG "metric": { 22:22:56 DEBUG "metric": { 22:22:56 DEBUG { 22:22:56 DEBUG "metric": { 22:22:56 DEBUG "metric": { 22:22:56 DEBUG { 22:22:56 DEBUG "pod": "ds-idrepo-2" 22:22:56 DEBUG "metric": { 22:22:56 DEBUG "metric": { 22:22:56 DEBUG "metric": { 22:22:56 DEBUG "result": [ 22:22:56 DEBUG "metric": { 22:22:56 DEBUG "pod": "ds-idrepo-2" 22:22:56 DEBUG "pod": "ds-idrepo-2" 22:22:56 DEBUG { 22:22:56 DEBUG "metric": { 22:22:56 DEBUG "metric": { 22:22:56 DEBUG "metric": { 22:22:56 DEBUG { 22:22:56 DEBUG }, 22:22:56 DEBUG "pod": "ds-idrepo-2" 22:22:56 DEBUG "metric": { 22:22:56 DEBUG "metric": { 22:22:56 DEBUG "metric": { 22:22:56 DEBUG "metric": { 22:22:56 DEBUG "pod": "ds-idrepo-2" 22:22:56 DEBUG "pod": "am-555f4db595-fz8sn" 22:22:56 DEBUG "metric": { 22:22:56 DEBUG "instance": "10.142.0.107:9100" 22:22:56 DEBUG "pod": "am-555f4db595-fz8sn" 22:22:56 DEBUG "metric": { 22:22:56 DEBUG }, 22:22:56 DEBUG "pod": "am-555f4db595-fz8sn" 22:22:56 DEBUG "instance": "10.142.0.107:9100" 22:22:56 DEBUG "pod": "ds-idrepo-2" 22:22:56 DEBUG { 22:22:56 DEBUG "pod": "am-555f4db595-fz8sn" 22:22:56 DEBUG }, 22:22:56 DEBUG }, 22:22:56 DEBUG "metric": { 22:22:56 DEBUG "instance": "10.142.0.107:9100" 22:22:56 DEBUG "pod": "am-555f4db595-fz8sn" 22:22:56 DEBUG "pod": "am-555f4db595-fz8sn" 22:22:56 DEBUG "metric": { 22:22:56 DEBUG "value": [ 22:22:56 DEBUG }, 22:22:56 DEBUG "pod": "ds-idrepo-2" 22:22:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:22:56 DEBUG "instance": "10.142.0.107:9100" 22:22:56 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:22:56 DEBUG }, 22:22:56 DEBUG }, 22:22:56 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:22:56 DEBUG }, 22:22:56 DEBUG }, 22:22:56 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:22:56 DEBUG "value": [ 22:22:56 DEBUG }, 22:22:56 DEBUG }, 22:22:56 DEBUG }, 22:22:56 DEBUG "metric": { 22:22:56 DEBUG }, 22:22:56 DEBUG "value": [ 22:22:56 DEBUG "value": [ 22:22:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:22:56 DEBUG }, 22:22:56 DEBUG }, 22:22:56 DEBUG }, 22:22:56 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:22:56 DEBUG 22:22:56 DEBUG "value": [ 22:22:56 DEBUG }, 22:22:56 DEBUG }, 22:22:56 DEBUG }, 22:22:56 DEBUG }, 22:22:56 DEBUG "value": [ 22:22:56 DEBUG "value": [ 22:22:56 DEBUG }, 22:22:56 DEBUG "value": [ 22:22:56 DEBUG "value": [ 22:22:56 DEBUG }, 22:22:56 DEBUG 22:22:56 DEBUG "value": [ 22:22:56 DEBUG "value": [ 22:22:56 DEBUG "value": [ 22:22:56 DEBUG "instance": "10.142.0.107:9100" 22:22:56 DEBUG "value": [ 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG }, 22:22:56 DEBUG "value": [ 22:22:56 DEBUG "value": [ 22:22:56 DEBUG "value": [ 22:22:56 DEBUG }, 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG "value": [ 22:22:56 DEBUG "value": [ 22:22:56 DEBUG "value": [ 22:22:56 DEBUG "value": [ 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG "value": [ 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG "value": [ 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG }, 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG "value": [ 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG "value": [ 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG "value": [ 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG [...] 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG [...] 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG [...] 22:22:56 DEBUG [...] 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG [...] 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG [...] 22:22:56 DEBUG [...] 22:22:56 DEBUG 22:22:56 DEBUG [...] 22:22:56 DEBUG [...] 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG [...] 22:22:56 DEBUG [...] 22:22:56 DEBUG [...] 22:22:56 DEBUG 22:22:56 DEBUG [...] 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG [...] 22:22:56 DEBUG [...] 22:22:56 DEBUG [...] 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG [...] 22:22:56 DEBUG [...] 22:22:56 DEBUG [...] 22:22:56 DEBUG [...] 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG [...] 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG [...] 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG [...] 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG [...] 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG [...] 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG "pod": "am-555f4db595-kzxsp" 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG "pod": "ds-idrepo-0" 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG "pod": "ds-cts-2" 22:22:56 DEBUG "pod": "ds-idrepo-0" 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG }, 22:22:56 DEBUG "pod": "ds-idrepo-0" 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG 22:22:56 DEBUG "pod": "overseer-0-7c9b667974-9mr2s" 22:22:56 DEBUG "pod": "am-555f4db595-kzxsp" 22:22:56 DEBUG 22:22:56 DEBUG "pod": "overseer-0-7c9b667974-9mr2s" 22:22:56 DEBUG "pod": "am-555f4db595-kzxsp" 22:22:56 DEBUG 22:22:56 DEBUG }, 22:22:56 DEBUG "pod": "am-555f4db595-kzxsp" 22:22:56 DEBUG "instance": "10.142.0.43:9100" 22:22:56 DEBUG "pod": "ds-idrepo-0" 22:22:56 DEBUG 22:22:56 DEBUG "pod": "am-555f4db595-kzxsp" 22:22:56 DEBUG }, 22:22:56 DEBUG }, 22:22:56 DEBUG 22:22:56 DEBUG "instance": "10.142.0.43:9100" 22:22:56 DEBUG "pod": "am-555f4db595-kzxsp" 22:22:56 DEBUG "pod": "am-555f4db595-kzxsp" 22:22:56 DEBUG 22:22:56 DEBUG "value": [ 22:22:56 DEBUG }, 22:22:56 DEBUG "pod": "ds-idrepo-0" 22:22:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:22:56 DEBUG "instance": "10.142.0.43:9100" 22:22:56 DEBUG "pod": "ds-idrepo-0" 22:22:56 DEBUG }, 22:22:56 DEBUG }, 22:22:56 DEBUG "instance": "10.142.0.43:9100" 22:22:56 DEBUG }, 22:22:56 DEBUG }, 22:22:56 DEBUG "pod": "overseer-0-7c9b667974-9mr2s" 22:22:56 DEBUG "value": [ 22:22:56 DEBUG }, 22:22:56 DEBUG }, 22:22:56 DEBUG }, 22:22:56 DEBUG 22:22:56 DEBUG }, 22:22:56 DEBUG "value": [ 22:22:56 DEBUG "value": [ 22:22:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:22:56 DEBUG }, 22:22:56 DEBUG }, 22:22:56 DEBUG }, 22:22:56 DEBUG "pod": "overseer-0-7c9b667974-9mr2s" 22:22:56 DEBUG 1713302576, 22:22:56 DEBUG "value": [ 22:22:56 DEBUG }, 22:22:56 DEBUG }, 22:22:56 DEBUG }, 22:22:56 DEBUG }, 22:22:56 DEBUG "value": [ 22:22:56 DEBUG "value": [ 22:22:56 DEBUG }, 22:22:56 DEBUG "value": [ 22:22:56 DEBUG "value": [ 22:22:56 DEBUG }, 22:22:56 DEBUG "value": [ 22:22:56 DEBUG "value": [ 22:22:56 DEBUG "value": [ 22:22:56 DEBUG 1713302576, 22:22:56 DEBUG "instance": "10.142.0.43:9100" 22:22:56 DEBUG "value": [ 22:22:56 DEBUG 1713302576, 22:22:56 DEBUG 1713302576, 22:22:56 DEBUG }, 22:22:56 DEBUG "value": [ 22:22:56 DEBUG "value": [ 22:22:56 DEBUG "value": [ 22:22:56 DEBUG }, 22:22:56 DEBUG "1999.6333333333332" 22:22:56 DEBUG 1713302576, 22:22:56 DEBUG "value": [ 22:22:56 DEBUG "value": [ 22:22:56 DEBUG "value": [ 22:22:56 DEBUG "value": [ 22:22:56 DEBUG 1713302576, 22:22:56 DEBUG 1713302576, 22:22:56 DEBUG "value": [ 22:22:56 DEBUG 1713302576, 22:22:56 DEBUG 1713302576, 22:22:56 DEBUG 1713302576, 22:22:56 DEBUG 1713302576, 22:22:56 DEBUG 1713302576, 22:22:56 DEBUG "45434832.06666667" 22:22:56 DEBUG }, 22:22:56 DEBUG 1713302576, 22:22:56 DEBUG "0" 22:22:56 DEBUG "0" 22:22:56 DEBUG "value": [ 22:22:56 DEBUG 1713302576, 22:22:56 DEBUG "45444724.43333332" 22:22:56 DEBUG 1713302576, 22:22:56 DEBUG "value": [ 22:22:56 DEBUG ] 22:22:56 DEBUG "0" 22:22:56 DEBUG 1713302576, 22:22:56 DEBUG 1713302576, 22:22:56 DEBUG 1713302576, 22:22:56 DEBUG 1713302576, 22:22:56 DEBUG "0" 22:22:56 DEBUG "0" 22:22:56 DEBUG 1713302576, 22:22:56 DEBUG "6674.379378425769" 22:22:56 DEBUG "0.024" 22:22:56 DEBUG "0" 22:22:56 DEBUG "0" 22:22:56 DEBUG "1999.6666666666667" 22:22:56 DEBUG ] 22:22:56 DEBUG "value": [ 22:22:56 DEBUG "0" 22:22:56 DEBUG ] 22:22:56 DEBUG ] 22:22:56 DEBUG 1713302576, 22:22:56 DEBUG 1713302576, 22:22:56 DEBUG ] 22:22:56 DEBUG "1999.7" 22:22:56 DEBUG 1713302576, 22:22:56 DEBUG } 22:22:56 DEBUG ] 22:22:56 DEBUG "value": [ 22:22:56 DEBUG "0" 22:22:56 DEBUG "6.317871784183776" 22:22:56 DEBUG "19953259520" 22:22:56 DEBUG "0" 22:22:56 DEBUG ] 22:22:56 DEBUG ] 22:22:56 DEBUG "0.0006333333333335152" 22:22:56 DEBUG ] 22:22:56 DEBUG ] 22:22:56 DEBUG ] 22:22:56 DEBUG ] 22:22:56 DEBUG ] 22:22:56 DEBUG } 22:22:56 DEBUG 1713302576, 22:22:56 DEBUG "0.00033333333333333365" 22:22:56 DEBUG ] 22:22:56 DEBUG } 22:22:56 DEBUG "644644864" 22:22:56 DEBUG "1339939.5" 22:22:56 DEBUG } 22:22:56 DEBUG ] 22:22:56 DEBUG "4.179032326417742" 22:22:56 DEBUG ] 22:22:56 DEBUG } 22:22:56 DEBUG 1713302576, 22:22:56 DEBUG ] 22:22:56 DEBUG ] 22:22:56 DEBUG ] 22:22:56 DEBUG ] 22:22:56 DEBUG } 22:22:56 DEBUG } 22:22:56 DEBUG ] 22:22:56 DEBUG } 22:22:56 DEBUG } 22:22:56 DEBUG } 22:22:56 DEBUG } 22:22:56 DEBUG } 22:22:56 DEBUG ] 22:22:56 DEBUG } 22:22:56 DEBUG ] 22:22:56 DEBUG } 22:22:56 DEBUG ] 22:22:56 DEBUG ] 22:22:56 DEBUG ] 22:22:56 DEBUG ] 22:22:56 DEBUG } 22:22:56 DEBUG ] 22:22:56 DEBUG } 22:22:56 DEBUG ] 22:22:56 DEBUG "0" 22:22:56 DEBUG } 22:22:56 DEBUG } 22:22:56 DEBUG } 22:22:56 DEBUG } 22:22:56 DEBUG ] 22:22:56 DEBUG ] 22:22:56 DEBUG } 22:22:56 DEBUG ] 22:22:56 DEBUG ] 22:22:56 DEBUG ] 22:22:56 DEBUG ] 22:22:56 DEBUG ] 22:22:56 DEBUG } 22:22:56 DEBUG ] 22:22:56 DEBUG } 22:22:56 DEBUG ] 22:22:56 DEBUG } 22:22:56 DEBUG } 22:22:56 DEBUG } 22:22:56 DEBUG } 22:22:56 DEBUG ] 22:22:56 DEBUG } 22:22:56 DEBUG } 22:22:56 DEBUG } 22:22:56 DEBUG ] 22:22:56 DEBUG ] 22:22:56 DEBUG ] 22:22:56 DEBUG ] 22:22:56 DEBUG ] 22:22:56 DEBUG } 22:22:56 DEBUG } 22:22:56 DEBUG ] 22:22:56 DEBUG } 22:22:56 DEBUG } 22:22:56 DEBUG } 22:22:56 DEBUG } 22:22:56 DEBUG } 22:22:56 DEBUG } 22:22:56 DEBUG } 22:22:56 DEBUG ] 22:22:56 DEBUG } 22:22:56 DEBUG } 22:22:56 DEBUG ] 22:22:56 DEBUG ] 22:22:56 DEBUG } 22:22:56 DEBUG } 22:22:56 DEBUG ] 22:22:56 DEBUG } 22:22:56 DEBUG } 22:22:56 DEBUG } 22:22:56 DEBUG } 22:22:56 DEBUG } 22:22:56 DEBUG } 22:22:56 DEBUG } 22:22:56 DEBUG } 22:22:56 DEBUG } 22:22:56 DEBUG } 22:22:56 DEBUG } 22:22:56 DEBUG } 22:22:56 DEBUG } 22:22:56 DEBUG } 22:22:56 DEBUG } 22:22:56 DEBUG } 22:22:56 DEBUG } 22:22:56 DEBUG } 22:22:56 DEBUG } 22:22:56 DEBUG } 22:22:56 DEBUG } 22:22:56 DEBUG ] 22:22:56 DEBUG } 22:22:56 DEBUG } 22:22:56 DEBUG } 22:22:56 DEBUG } 22:22:56 INFO [loop_until]: OK (rc = 0) 22:22:56 DEBUG } 22:22:56 DEBUG } 22:22:56 DEBUG } 22:22:56 DEBUG } 22:22:56 DEBUG } 22:22:56 DEBUG } 22:22:56 DEBUG --- stdout --- 22:22:56 DEBUG } 22:22:56 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 63m 0% 5090Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 6774m 22% 14718Mi 13% gke-xlou-cdm-default-pool-1b0769c8-nhlf 9403m 31% 10815Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 84m 0% 2645Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 4712m 15% 6451Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 7950m 26% 16577Mi 14% gke-xlou-cdm-default-pool-1b0769c8-xnmp 62m 0% 3141Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 9239m 30% 10365Mi 9% gke-xlou-cdm-ds-0f601751-hmm7 4657m 15% 5581Mi 4% gke-xlou-cdm-ds-0f601751-j488 6812m 22% 14062Mi 12% gke-xlou-cdm-ds-0f601751-mc5d 10677m 35% 3341Mi 2% gke-xlou-cdm-ds-0f601751-nj6q 9493m 31% 10427Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 4691m 15% 7357Mi 6% gke-xlou-cdm-frontend-a8771548-i82n 6228m 39% 5951Mi 10% 22:22:56 DEBUG --- stderr --- 22:22:56 DEBUG 22:23:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713302635" 22:23:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302635" 22:23:55 ERROR [http_cmd]: ERROR 22:23:55 INFO [http_cmd]: http status code OK 22:23:55 DEBUG --- status code --- 22:23:55 ERROR --- status code --- 22:23:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:23:55 ERROR http status code is 400 (expected [200, 401, 403, 404]) 22:23:55 DEBUG --- http response --- 22:23:55 ERROR --- http response --- 22:23:55 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 22:23:55 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 22:23:55 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 22:23:56 INFO 22:23:56 INFO [loop_until]: kubectl --namespace=xlou top pods 22:23:56 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:23:56 INFO [loop_until]: OK (rc = 0) 22:23:56 DEBUG --- stdout --- 22:23:56 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 9Mi am-555f4db595-4lbss 9320m 8588Mi am-555f4db595-fz8sn 9288m 8600Mi am-555f4db595-kzxsp 8923m 8553Mi ds-cts-0 7732m 12086Mi ds-cts-1 7113m 12174Mi ds-cts-2 6850m 12136Mi ds-idrepo-0 4795m 5032Mi ds-idrepo-1 4396m 2834Mi ds-idrepo-2 4458m 3116Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 4m 1207Mi idm-b85f9d978-rl76f 3m 1225Mi lodemon-bc756c647-cr7x6 12m 63Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-7c9b667974-9mr2s 6376m 523Mi 22:23:56 DEBUG --- stderr --- 22:23:56 DEBUG 22:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302636" 22:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302636" 22:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302636" 22:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302636" 22:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302636" 22:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302636" 22:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713302636" 22:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302636" 22:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302636" 22:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302636" 22:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302636" 22:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713302636" 22:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302636" 22:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713302636" 22:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302636" 22:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302636" 22:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302636" 22:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302636" 22:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713302636" 22:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302636" 22:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302636" 22:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302636" 22:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302636" 22:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302636" 22:23:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302636" 22:23:56 INFO 22:23:56 INFO [http_cmd]: http status code OK 22:23:56 INFO [http_cmd]: http status code OK 22:23:56 INFO [http_cmd]: http status code OK 22:23:56 INFO [http_cmd]: http status code OK 22:23:56 INFO [http_cmd]: http status code OK 22:23:56 INFO [http_cmd]: http status code OK 22:23:56 INFO [http_cmd]: http status code OK 22:23:56 INFO [http_cmd]: http status code OK 22:23:56 INFO [http_cmd]: http status code OK 22:23:56 INFO [http_cmd]: http status code OK 22:23:56 INFO [http_cmd]: http status code OK 22:23:56 INFO [http_cmd]: http status code OK 22:23:56 INFO [http_cmd]: http status code OK 22:23:56 INFO [http_cmd]: http status code OK 22:23:56 INFO [http_cmd]: http status code OK 22:23:56 INFO [http_cmd]: http status code OK 22:23:56 INFO [http_cmd]: http status code OK 22:23:56 INFO [loop_until]: kubectl --namespace=xlou top node 22:23:56 INFO [http_cmd]: http status code OK 22:23:56 DEBUG --- status code --- 22:23:56 INFO [http_cmd]: http status code OK 22:23:56 DEBUG --- status code --- 22:23:56 DEBUG --- status code --- 22:23:56 DEBUG --- status code --- 22:23:56 DEBUG --- status code --- 22:23:56 DEBUG --- status code --- 22:23:56 DEBUG --- status code --- 22:23:56 DEBUG --- status code --- 22:23:56 DEBUG --- status code --- 22:23:56 DEBUG --- status code --- 22:23:56 DEBUG --- status code --- 22:23:56 DEBUG --- status code --- 22:23:56 INFO [http_cmd]: http status code OK 22:23:56 DEBUG --- status code --- 22:23:56 INFO [http_cmd]: http status code OK 22:23:56 INFO [http_cmd]: http status code OK 22:23:56 DEBUG --- status code --- 22:23:56 DEBUG --- status code --- 22:23:56 DEBUG --- status code --- 22:23:56 DEBUG --- status code --- 22:23:56 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:23:56 DEBUG --- status code --- 22:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:23:56 DEBUG --- status code --- 22:23:56 INFO [http_cmd]: http status code OK 22:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:23:56 INFO [http_cmd]: http status code OK 22:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:23:56 DEBUG --- status code --- 22:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:23:56 DEBUG --- status code --- 22:23:56 DEBUG --- status code --- 22:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:23:56 DEBUG --- http response --- 22:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:23:56 DEBUG --- status code --- 22:23:56 INFO [http_cmd]: http status code OK 22:23:56 DEBUG --- http response --- 22:23:56 DEBUG --- status code --- 22:23:56 DEBUG --- http response --- 22:23:56 DEBUG --- http response --- 22:23:56 DEBUG --- http response --- 22:23:56 DEBUG --- http response --- 22:23:56 DEBUG --- http response --- 22:23:56 DEBUG --- http response --- 22:23:56 DEBUG --- http response --- 22:23:56 DEBUG --- http response --- 22:23:56 DEBUG --- http response --- 22:23:56 DEBUG --- http response --- 22:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:23:56 DEBUG --- http response --- 22:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:23:56 DEBUG --- http response --- 22:23:56 DEBUG --- http response --- 22:23:56 DEBUG --- http response --- 22:23:56 DEBUG --- http response --- 22:23:56 DEBUG --- http response --- 22:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:23:56 DEBUG --- http response --- 22:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:23:56 DEBUG --- status code --- 22:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:23:56 DEBUG --- http response --- 22:23:56 DEBUG --- http response --- 22:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:23:56 DEBUG --- http response --- 22:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:23:56 DEBUG ----- output ----- 22:23:56 DEBUG { 22:23:56 DEBUG --- http response --- 22:23:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:23:56 DEBUG ----- output ----- 22:23:56 DEBUG --- http response --- 22:23:56 DEBUG ----- output ----- 22:23:56 DEBUG ----- output ----- 22:23:56 DEBUG ----- output ----- 22:23:56 DEBUG ----- output ----- 22:23:56 DEBUG ----- output ----- 22:23:56 DEBUG ----- output ----- 22:23:56 DEBUG ----- output ----- 22:23:56 DEBUG ----- output ----- 22:23:56 DEBUG ----- output ----- 22:23:56 DEBUG ----- output ----- 22:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:23:56 DEBUG ----- output ----- 22:23:56 DEBUG ----- output ----- 22:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:23:56 DEBUG ----- output ----- 22:23:56 DEBUG ----- output ----- 22:23:56 DEBUG ----- output ----- 22:23:56 DEBUG ----- output ----- 22:23:56 DEBUG "status": "success", 22:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:23:56 DEBUG --- http response --- 22:23:56 DEBUG { 22:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:23:56 DEBUG { 22:23:56 DEBUG { 22:23:56 DEBUG { 22:23:56 DEBUG { 22:23:56 DEBUG { 22:23:56 DEBUG { 22:23:56 DEBUG { 22:23:56 DEBUG { 22:23:56 DEBUG { 22:23:56 DEBUG { 22:23:56 DEBUG ----- output ----- 22:23:56 DEBUG ----- output ----- 22:23:56 DEBUG { 22:23:56 DEBUG { 22:23:56 DEBUG ----- output ----- 22:23:56 DEBUG { 22:23:56 DEBUG { 22:23:56 DEBUG { 22:23:56 DEBUG { 22:23:56 DEBUG "status": "success", 22:23:56 DEBUG ----- output ----- 22:23:56 DEBUG ----- output ----- 22:23:56 DEBUG "status": "success", 22:23:56 DEBUG ----- output ----- 22:23:56 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:23:56 DEBUG "status": "success", 22:23:56 DEBUG "status": "success", 22:23:56 DEBUG "data": { 22:23:56 DEBUG "status": "success", 22:23:56 DEBUG "status": "success", 22:23:56 DEBUG "status": "success", 22:23:56 DEBUG "status": "success", 22:23:56 DEBUG "status": "success", 22:23:56 DEBUG "status": "success", 22:23:56 DEBUG "status": "success", 22:23:56 DEBUG { 22:23:56 DEBUG { 22:23:56 DEBUG "status": "success", 22:23:56 DEBUG "status": "success", 22:23:56 DEBUG { 22:23:56 DEBUG "status": "success", 22:23:56 DEBUG "status": "success", 22:23:56 DEBUG "status": "success", 22:23:56 DEBUG "data": { 22:23:56 DEBUG "data": { 22:23:56 DEBUG { 22:23:56 DEBUG { 22:23:56 DEBUG "data": { 22:23:56 DEBUG { 22:23:56 DEBUG ----- output ----- 22:23:56 DEBUG "data": { 22:23:56 DEBUG "status": "success", 22:23:56 DEBUG "resultType": "vector", 22:23:56 DEBUG "data": { 22:23:56 DEBUG "data": { 22:23:56 DEBUG "data": { 22:23:56 DEBUG "data": { 22:23:56 DEBUG "data": { 22:23:56 DEBUG "data": { 22:23:56 DEBUG "resultType": "vector", 22:23:56 DEBUG "status": "success", 22:23:56 DEBUG "status": "success", 22:23:56 DEBUG "data": { 22:23:56 DEBUG "data": { 22:23:56 DEBUG "status": "success", 22:23:56 DEBUG "data": { 22:23:56 DEBUG "data": { 22:23:56 DEBUG "data": { 22:23:56 DEBUG "resultType": "vector", 22:23:56 DEBUG "resultType": "vector", 22:23:56 DEBUG "status": "success", 22:23:56 DEBUG "status": "success", 22:23:56 DEBUG "resultType": "vector", 22:23:56 DEBUG "status": "success", 22:23:56 DEBUG { 22:23:56 DEBUG "resultType": "vector", 22:23:56 DEBUG "data": { 22:23:56 DEBUG "result": [ 22:23:56 DEBUG "resultType": "vector", 22:23:56 DEBUG "resultType": "vector", 22:23:56 DEBUG "resultType": "vector", 22:23:56 DEBUG "resultType": "vector", 22:23:56 DEBUG "resultType": "vector", 22:23:56 DEBUG "result": [ 22:23:56 DEBUG { 22:23:56 DEBUG "data": { 22:23:56 DEBUG "data": { 22:23:56 DEBUG "resultType": "vector", 22:23:56 DEBUG "resultType": "vector", 22:23:56 DEBUG "data": { 22:23:56 DEBUG "resultType": "vector", 22:23:56 DEBUG "resultType": "vector", 22:23:56 DEBUG "resultType": "vector", 22:23:56 DEBUG "result": [ 22:23:56 DEBUG "result": [ 22:23:56 DEBUG "data": { 22:23:56 DEBUG "data": { 22:23:56 DEBUG "result": [ 22:23:56 DEBUG "data": { 22:23:56 DEBUG "status": "success", 22:23:56 DEBUG "result": [ 22:23:56 DEBUG "resultType": "vector", 22:23:56 DEBUG { 22:23:56 DEBUG "result": [ 22:23:56 DEBUG "result": [ 22:23:56 DEBUG "result": [ 22:23:56 DEBUG "result": [ 22:23:56 DEBUG "result": [ 22:23:56 DEBUG "data": { 22:23:56 DEBUG "metric": { 22:23:56 DEBUG "resultType": "vector", 22:23:56 DEBUG "resultType": "vector", 22:23:56 DEBUG "result": [ 22:23:56 DEBUG "result": [ 22:23:56 DEBUG "resultType": "vector", 22:23:56 DEBUG "result": [ 22:23:56 DEBUG "result": [ 22:23:56 DEBUG "result": [ 22:23:56 DEBUG { 22:23:56 DEBUG { 22:23:56 DEBUG "resultType": "vector", 22:23:56 DEBUG "resultType": "vector", 22:23:56 DEBUG { 22:23:56 DEBUG "resultType": "vector", 22:23:56 DEBUG "data": { 22:23:56 DEBUG { 22:23:56 DEBUG "result": [ 22:23:56 DEBUG "metric": { 22:23:56 DEBUG { 22:23:56 DEBUG { 22:23:56 DEBUG { 22:23:56 DEBUG { 22:23:56 DEBUG { 22:23:56 DEBUG "resultType": "vector", 22:23:56 DEBUG "pod": "am-555f4db595-fz8sn" 22:23:56 DEBUG "result": [ 22:23:56 DEBUG "result": [ 22:23:56 DEBUG { 22:23:56 DEBUG { 22:23:56 DEBUG "result": [ 22:23:56 DEBUG { 22:23:56 DEBUG { 22:23:56 DEBUG { 22:23:56 DEBUG "metric": { 22:23:56 DEBUG "metric": { 22:23:56 DEBUG "result": [ 22:23:56 DEBUG "result": [ 22:23:56 DEBUG "metric": { 22:23:56 DEBUG "result": [ 22:23:56 DEBUG "resultType": "vector", 22:23:56 DEBUG "metric": { 22:23:56 DEBUG { 22:23:56 DEBUG "pod": "ds-cts-1" 22:23:56 DEBUG "metric": { 22:23:56 DEBUG "metric": { 22:23:56 DEBUG "metric": { 22:23:56 DEBUG "metric": { 22:23:56 DEBUG "metric": { 22:23:56 DEBUG "result": [ 22:23:56 DEBUG }, 22:23:56 DEBUG { 22:23:56 DEBUG { 22:23:56 DEBUG "metric": { 22:23:56 DEBUG "metric": { 22:23:56 DEBUG { 22:23:56 DEBUG "metric": { 22:23:56 DEBUG "metric": { 22:23:56 DEBUG "metric": { 22:23:56 DEBUG "instance": "10.142.0.107:9100" 22:23:56 DEBUG "pod": "am-555f4db595-fz8sn" 22:23:56 DEBUG { 22:23:56 DEBUG { 22:23:56 DEBUG "pod": "am-555f4db595-fz8sn" 22:23:56 DEBUG { 22:23:56 DEBUG "result": [ 22:23:56 DEBUG "instance": "10.142.0.107:9100" 22:23:56 DEBUG "metric": { 22:23:56 DEBUG }, 22:23:56 DEBUG "instance": "10.142.0.107:9100" 22:23:56 DEBUG "pod": "am-555f4db595-fz8sn" 22:23:56 DEBUG "instance": "10.142.0.107:9100" 22:23:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:23:56 DEBUG "pod": "ds-idrepo-2" 22:23:56 DEBUG { 22:23:56 DEBUG "value": [ 22:23:56 DEBUG "metric": { 22:23:56 DEBUG "metric": { 22:23:56 DEBUG "pod": "ds-idrepo-2" 22:23:56 DEBUG "pod": "ds-idrepo-2" 22:23:56 DEBUG "metric": { 22:23:56 DEBUG "pod": "ds-idrepo-2" 22:23:56 DEBUG "instance": "10.142.0.107:9100" 22:23:56 DEBUG "pod": "ds-cts-1" 22:23:56 DEBUG }, 22:23:56 DEBUG }, 22:23:56 DEBUG "metric": { 22:23:56 DEBUG "metric": { 22:23:56 DEBUG }, 22:23:56 DEBUG "metric": { 22:23:56 DEBUG { 22:23:56 DEBUG }, 22:23:56 DEBUG "pod": "ds-idrepo-2" 22:23:56 DEBUG "value": [ 22:23:56 DEBUG }, 22:23:56 DEBUG }, 22:23:56 DEBUG }, 22:23:56 DEBUG }, 22:23:56 DEBUG }, 22:23:56 DEBUG "metric": { 22:23:56 DEBUG 22:23:56 DEBUG "pod": "am-555f4db595-fz8sn" 22:23:56 DEBUG "pod": "am-555f4db595-fz8sn" 22:23:56 DEBUG }, 22:23:56 DEBUG }, 22:23:56 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:23:56 DEBUG }, 22:23:56 DEBUG }, 22:23:56 DEBUG }, 22:23:56 DEBUG "value": [ 22:23:56 DEBUG "value": [ 22:23:56 DEBUG "pod": "ds-cts-1" 22:23:56 DEBUG "pod": "am-555f4db595-fz8sn" 22:23:56 DEBUG "value": [ 22:23:56 DEBUG "pod": "ds-idrepo-2" 22:23:56 DEBUG "metric": { 22:23:56 DEBUG "value": [ 22:23:56 DEBUG }, 22:23:56 DEBUG 22:23:56 DEBUG "value": [ 22:23:56 DEBUG "value": [ 22:23:56 DEBUG "value": [ 22:23:56 DEBUG "value": [ 22:23:56 DEBUG "value": [ 22:23:56 DEBUG "pod": "ds-cts-1" 22:23:56 DEBUG 22:23:56 DEBUG }, 22:23:56 DEBUG "value": [ 22:23:56 DEBUG "value": [ 22:23:56 DEBUG }, 22:23:56 DEBUG }, 22:23:56 DEBUG "value": [ 22:23:56 DEBUG "value": [ 22:23:56 DEBUG "value": [ 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG }, 22:23:56 DEBUG }, 22:23:56 DEBUG 22:23:56 DEBUG }, 22:23:56 DEBUG "pod": "ds-idrepo-2" 22:23:56 DEBUG 22:23:56 DEBUG "value": [ 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG }, 22:23:56 DEBUG 22:23:56 DEBUG "value": [ 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG "value": [ 22:23:56 DEBUG "value": [ 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG "value": [ 22:23:56 DEBUG "value": [ 22:23:56 DEBUG 22:23:56 DEBUG "value": [ 22:23:56 DEBUG }, 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG "value": [ 22:23:56 DEBUG [...] 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG "value": [ 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG [...] 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG [...] 22:23:56 DEBUG [...] 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG [...] 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG [...] 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG [...] 22:23:56 DEBUG [...] 22:23:56 DEBUG [...] 22:23:56 DEBUG [...] 22:23:56 DEBUG [...] 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG [...] 22:23:56 DEBUG [...] 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG [...] 22:23:56 DEBUG [...] 22:23:56 DEBUG [...] 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG [...] 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG [...] 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG [...] 22:23:56 DEBUG [...] 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG [...] 22:23:56 DEBUG [...] 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG [...] 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG [...] 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG [...] 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG "pod": "ds-idrepo-2" 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG "pod": "am-555f4db595-kzxsp" 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG "instance": "10.142.0.43:9100" 22:23:56 DEBUG "pod": "am-555f4db595-kzxsp" 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG "pod": "am-555f4db595-kzxsp" 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG "instance": "10.142.0.43:9100" 22:23:56 DEBUG 22:23:56 DEBUG }, 22:23:56 DEBUG "instance": "10.142.0.43:9100" 22:23:56 DEBUG "pod": "am-555f4db595-kzxsp" 22:23:56 DEBUG "instance": "10.142.0.43:9100" 22:23:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:23:56 DEBUG "pod": "ds-idrepo-0" 22:23:56 DEBUG }, 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG "pod": "am-555f4db595-kzxsp" 22:23:56 DEBUG "pod": "ds-cts-2" 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG "pod": "ds-idrepo-0" 22:23:56 DEBUG "instance": "10.142.0.43:9100" 22:23:56 DEBUG "pod": "ds-idrepo-2" 22:23:56 DEBUG }, 22:23:56 DEBUG }, 22:23:56 DEBUG 22:23:56 DEBUG "pod": "ds-idrepo-2" 22:23:56 DEBUG }, 22:23:56 DEBUG 22:23:56 DEBUG 22:23:56 DEBUG }, 22:23:56 DEBUG "pod": "ds-idrepo-0" 22:23:56 DEBUG "value": [ 22:23:56 DEBUG }, 22:23:56 DEBUG }, 22:23:56 DEBUG }, 22:23:56 DEBUG }, 22:23:56 DEBUG }, 22:23:56 DEBUG "value": [ 22:23:56 DEBUG "pod": "ds-idrepo-2" 22:23:56 DEBUG "pod": "am-555f4db595-kzxsp" 22:23:56 DEBUG }, 22:23:56 DEBUG }, 22:23:56 DEBUG "pod": "ds-idrepo-0" 22:23:56 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:23:56 DEBUG }, 22:23:56 DEBUG }, 22:23:56 DEBUG }, 22:23:56 DEBUG "value": [ 22:23:56 DEBUG "value": [ 22:23:56 DEBUG 22:23:56 DEBUG }, 22:23:56 DEBUG "value": [ 22:23:56 DEBUG "pod": "ds-idrepo-0" 22:23:56 DEBUG 22:23:56 DEBUG "value": [ 22:23:56 DEBUG }, 22:23:56 DEBUG 1713302636, 22:23:56 DEBUG "value": [ 22:23:56 DEBUG "value": [ 22:23:56 DEBUG "value": [ 22:23:56 DEBUG "value": [ 22:23:56 DEBUG "value": [ 22:23:56 DEBUG 1713302636, 22:23:56 DEBUG }, 22:23:56 DEBUG }, 22:23:56 DEBUG "value": [ 22:23:56 DEBUG "value": [ 22:23:56 DEBUG }, 22:23:56 DEBUG }, 22:23:56 DEBUG "value": [ 22:23:56 DEBUG "value": [ 22:23:56 DEBUG "value": [ 22:23:56 DEBUG 1713302636, 22:23:56 DEBUG 1713302636, 22:23:56 DEBUG "pod": "am-555f4db595-kzxsp" 22:23:56 DEBUG "value": [ 22:23:56 DEBUG 1713302636, 22:23:56 DEBUG }, 22:23:56 DEBUG "pod": "ds-idrepo-0" 22:23:56 DEBUG 1713302636, 22:23:56 DEBUG "value": [ 22:23:56 DEBUG "22428.663173905697" 22:23:56 DEBUG 1713302636, 22:23:56 DEBUG 1713302636, 22:23:56 DEBUG 1713302636, 22:23:56 DEBUG 1713302636, 22:23:56 DEBUG "6.313038619184396" 22:23:56 DEBUG "2001.1" 22:23:56 DEBUG "value": [ 22:23:56 DEBUG "value": [ 22:23:56 DEBUG 1713302636, 22:23:56 DEBUG 1713302636, 22:23:56 DEBUG "value": [ 22:23:56 DEBUG "value": [ 22:23:56 DEBUG 1713302636, 22:23:56 DEBUG 1713302636, 22:23:56 DEBUG 1713302636, 22:23:56 DEBUG "5537.94152442116" 22:23:56 DEBUG "0.024" 22:23:56 DEBUG }, 22:23:56 DEBUG 1713302636, 22:23:56 DEBUG "0" 22:23:56 DEBUG "value": [ 22:23:56 DEBUG }, 22:23:56 DEBUG "45417925.266666666" 22:23:56 DEBUG 1713302636, 22:23:56 DEBUG ] 22:23:56 DEBUG "45427612.83333333" 22:23:56 DEBUG "2001.1" 22:23:56 DEBUG "19954021376" 22:23:56 DEBUG 1713302636, 22:23:56 DEBUG ] 22:23:56 DEBUG ] 22:23:56 DEBUG 1713302636, 22:23:56 DEBUG 1713302636, 22:23:56 DEBUG "2001.0666666666666" 22:23:56 DEBUG "0" 22:23:56 DEBUG 1713302636, 22:23:56 DEBUG 1713302636, 22:23:56 DEBUG "0" 22:23:56 DEBUG "0.0006666666666666598" 22:23:56 DEBUG "0.00040000000000001514" 22:23:56 DEBUG ] 22:23:56 DEBUG ] 22:23:56 DEBUG "value": [ 22:23:56 DEBUG "4.439611458509384" 22:23:56 DEBUG ] 22:23:56 DEBUG 1713302636, 22:23:56 DEBUG "value": [ 22:23:56 DEBUG ] 22:23:56 DEBUG "0" 22:23:56 DEBUG } 22:23:56 DEBUG ] 22:23:56 DEBUG ] 22:23:56 DEBUG ] 22:23:56 DEBUG "0" 22:23:56 DEBUG } 22:23:56 DEBUG } 22:23:56 DEBUG "0" 22:23:56 DEBUG "0" 22:23:56 DEBUG ] 22:23:56 DEBUG ] 22:23:56 DEBUG "0" 22:23:56 DEBUG "647168000" 22:23:56 DEBUG ] 22:23:56 DEBUG ] 22:23:56 DEBUG ] 22:23:56 DEBUG } 22:23:56 DEBUG } 22:23:56 DEBUG 1713302636, 22:23:56 DEBUG ] 22:23:56 DEBUG } 22:23:56 DEBUG "0" 22:23:56 DEBUG 1713302636, 22:23:56 DEBUG } 22:23:56 DEBUG ] 22:23:56 DEBUG ] 22:23:56 DEBUG } 22:23:56 DEBUG } 22:23:56 DEBUG } 22:23:56 DEBUG ] 22:23:56 DEBUG ] 22:23:56 DEBUG ] 22:23:56 DEBUG ] 22:23:56 DEBUG ] 22:23:56 DEBUG } 22:23:56 DEBUG } 22:23:56 DEBUG ] 22:23:56 DEBUG ] 22:23:56 DEBUG } 22:23:56 DEBUG } 22:23:56 DEBUG } 22:23:56 DEBUG ] 22:23:56 DEBUG ] 22:23:56 DEBUG "1460084.5" 22:23:56 DEBUG } 22:23:56 DEBUG ] 22:23:56 DEBUG ] 22:23:56 DEBUG "0" 22:23:56 DEBUG ] 22:23:56 DEBUG } 22:23:56 DEBUG } 22:23:56 DEBUG ] 22:23:56 DEBUG ] 22:23:56 DEBUG ] 22:23:56 DEBUG } 22:23:56 DEBUG } 22:23:56 DEBUG } 22:23:56 DEBUG } 22:23:56 DEBUG } 22:23:56 DEBUG ] 22:23:56 DEBUG ] 22:23:56 DEBUG } 22:23:56 DEBUG } 22:23:56 DEBUG ] 22:23:56 DEBUG ] 22:23:56 DEBUG ] 22:23:56 DEBUG } 22:23:56 DEBUG } 22:23:56 DEBUG ] 22:23:56 DEBUG ] 22:23:56 DEBUG } 22:23:56 DEBUG } 22:23:56 DEBUG ] 22:23:56 DEBUG } 22:23:56 DEBUG ] 22:23:56 DEBUG } 22:23:56 DEBUG } 22:23:56 DEBUG } 22:23:56 DEBUG } 22:23:56 DEBUG ] 22:23:56 DEBUG } 22:23:56 DEBUG } 22:23:56 DEBUG ] 22:23:56 DEBUG ] 22:23:56 DEBUG } 22:23:56 DEBUG } 22:23:56 DEBUG ] 22:23:56 DEBUG ] 22:23:56 DEBUG } 22:23:56 DEBUG } 22:23:56 DEBUG } 22:23:56 DEBUG } 22:23:56 DEBUG } 22:23:56 DEBUG } 22:23:56 DEBUG } 22:23:56 DEBUG } 22:23:56 DEBUG ] 22:23:56 DEBUG } 22:23:56 DEBUG } 22:23:56 DEBUG } 22:23:56 DEBUG } 22:23:56 DEBUG } 22:23:56 DEBUG } 22:23:56 DEBUG } 22:23:56 DEBUG } 22:23:56 INFO [loop_until]: OK (rc = 0) 22:23:56 DEBUG } 22:23:56 DEBUG } 22:23:56 DEBUG } 22:23:56 DEBUG } 22:23:56 DEBUG } 22:23:56 DEBUG } 22:23:56 DEBUG } 22:23:56 DEBUG } 22:23:56 DEBUG ] 22:23:56 DEBUG } 22:23:56 DEBUG } 22:23:56 DEBUG ] 22:23:56 DEBUG } 22:23:56 DEBUG } 22:23:56 DEBUG } 22:23:56 DEBUG --- stdout --- 22:23:56 DEBUG } 22:23:56 DEBUG } 22:23:56 DEBUG } 22:23:56 DEBUG } 22:23:56 DEBUG } 22:23:56 DEBUG } 22:23:56 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 63m 0% 5095Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7192m 24% 14725Mi 13% gke-xlou-cdm-default-pool-1b0769c8-nhlf 9414m 31% 10823Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 85m 0% 2646Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 4702m 15% 6452Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 8293m 27% 16605Mi 14% gke-xlou-cdm-default-pool-1b0769c8-xnmp 63m 0% 3157Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 9273m 31% 10361Mi 9% gke-xlou-cdm-ds-0f601751-hmm7 4673m 15% 5578Mi 4% gke-xlou-cdm-ds-0f601751-j488 6912m 23% 14064Mi 12% gke-xlou-cdm-ds-0f601751-mc5d 10678m 35% 3350Mi 3% gke-xlou-cdm-ds-0f601751-nj6q 9481m 31% 10414Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 4792m 16% 7361Mi 6% gke-xlou-cdm-frontend-a8771548-i82n 6464m 40% 5949Mi 10% 22:23:56 DEBUG } 22:23:56 DEBUG } 22:23:56 DEBUG --- stderr --- 22:23:56 DEBUG 22:24:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302695" 22:24:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713302695" 22:24:55 ERROR [http_cmd]: ERROR 22:24:55 ERROR --- status code --- 22:24:55 ERROR http status code is 400 (expected [200, 401, 403, 404]) 22:24:55 ERROR --- http response --- 22:24:55 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 22:24:55 INFO [http_cmd]: http status code OK 22:24:55 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 22:24:55 DEBUG --- status code --- 22:24:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:24:55 DEBUG --- http response --- 22:24:55 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 22:24:56 INFO 22:24:56 INFO [loop_until]: kubectl --namespace=xlou top pods 22:24:56 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:24:56 INFO [loop_until]: OK (rc = 0) 22:24:56 DEBUG --- stdout --- 22:24:56 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 9Mi am-555f4db595-4lbss 9312m 8589Mi am-555f4db595-fz8sn 9204m 8602Mi am-555f4db595-kzxsp 9260m 8553Mi ds-cts-0 8086m 12094Mi ds-cts-1 7059m 12180Mi ds-cts-2 6839m 12145Mi ds-idrepo-0 4725m 5032Mi ds-idrepo-1 4517m 2834Mi ds-idrepo-2 4470m 3115Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 4m 1207Mi idm-b85f9d978-rl76f 3m 1226Mi lodemon-bc756c647-cr7x6 15m 63Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-7c9b667974-9mr2s 6222m 525Mi 22:24:56 DEBUG --- stderr --- 22:24:56 DEBUG 22:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302696" 22:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302696" 22:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713302696" 22:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302696" 22:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713302696" 22:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302696" 22:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302696" 22:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302696" 22:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302696" 22:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302696" 22:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302696" 22:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302696" 22:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302696" 22:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302696" 22:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302696" 22:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713302696" 22:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302696" 22:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302696" 22:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713302696" 22:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302696" 22:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302696" 22:24:56 INFO 22:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302696" 22:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302696" 22:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302696" 22:24:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302696" 22:24:56 INFO [http_cmd]: http status code OK 22:24:56 INFO [http_cmd]: http status code OK 22:24:56 INFO [http_cmd]: http status code OK 22:24:56 INFO [http_cmd]: http status code OK 22:24:56 INFO [http_cmd]: http status code OK 22:24:56 INFO [loop_until]: kubectl --namespace=xlou top node 22:24:56 DEBUG --- status code --- 22:24:56 INFO [http_cmd]: http status code OK 22:24:56 INFO [http_cmd]: http status code OK 22:24:56 INFO [http_cmd]: http status code OK 22:24:56 INFO [http_cmd]: http status code OK 22:24:56 INFO [http_cmd]: http status code OK 22:24:56 INFO [http_cmd]: http status code OK 22:24:56 DEBUG --- status code --- 22:24:56 INFO [http_cmd]: http status code OK 22:24:56 INFO [http_cmd]: http status code OK 22:24:56 INFO [http_cmd]: http status code OK 22:24:56 INFO [http_cmd]: http status code OK 22:24:56 DEBUG --- status code --- 22:24:56 INFO [http_cmd]: http status code OK 22:24:56 DEBUG --- status code --- 22:24:56 INFO [http_cmd]: http status code OK 22:24:56 INFO [http_cmd]: http status code OK 22:24:56 INFO [http_cmd]: http status code OK 22:24:56 DEBUG --- status code --- 22:24:56 INFO [http_cmd]: http status code OK 22:24:56 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:24:56 INFO [http_cmd]: http status code OK 22:24:56 INFO [http_cmd]: http status code OK 22:24:56 INFO [http_cmd]: http status code OK 22:24:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:24:56 DEBUG --- status code --- 22:24:56 DEBUG --- status code --- 22:24:56 DEBUG --- status code --- 22:24:56 DEBUG --- status code --- 22:24:56 DEBUG --- status code --- 22:24:56 DEBUG --- status code --- 22:24:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:24:56 DEBUG --- status code --- 22:24:56 DEBUG --- status code --- 22:24:56 INFO [http_cmd]: http status code OK 22:24:56 INFO [http_cmd]: http status code OK 22:24:56 DEBUG --- status code --- 22:24:56 DEBUG --- status code --- 22:24:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:24:56 DEBUG --- status code --- 22:24:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:24:56 DEBUG --- status code --- 22:24:56 DEBUG --- status code --- 22:24:56 DEBUG --- status code --- 22:24:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:24:56 DEBUG --- status code --- 22:24:56 DEBUG --- status code --- 22:24:56 DEBUG --- status code --- 22:24:56 DEBUG --- status code --- 22:24:56 DEBUG --- http response --- 22:24:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:24:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:24:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:24:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:24:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:24:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:24:57 DEBUG --- http response --- 22:24:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:24:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:24:57 DEBUG --- status code --- 22:24:57 DEBUG --- status code --- 22:24:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:24:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:24:57 DEBUG --- http response --- 22:24:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:24:57 DEBUG --- http response --- 22:24:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:24:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:24:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:24:57 DEBUG --- http response --- 22:24:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:24:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:24:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:24:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:24:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:24:57 DEBUG --- http response --- 22:24:57 DEBUG --- http response --- 22:24:57 DEBUG --- http response --- 22:24:57 DEBUG --- http response --- 22:24:57 DEBUG --- http response --- 22:24:57 DEBUG --- http response --- 22:24:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 22:24:57 DEBUG --- http response --- 22:24:57 DEBUG --- http response --- 22:24:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:24:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:24:57 DEBUG --- http response --- 22:24:57 DEBUG --- http response --- 22:24:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:24:57 DEBUG --- http response --- 22:24:57 DEBUG --- http response --- 22:24:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 22:24:57 DEBUG --- http response --- 22:24:57 DEBUG --- http response --- 22:24:57 DEBUG --- http response --- 22:24:57 DEBUG --- http response --- 22:24:57 DEBUG --- http response --- 22:24:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:24:57 DEBUG --- http response --- 22:24:57 DEBUG ----- output ----- 22:24:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:24:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:24:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:24:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:24:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:24:57 DEBUG ----- output ----- 22:24:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 22:24:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:24:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:24:57 DEBUG --- http response --- 22:24:57 DEBUG --- http response --- 22:24:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:24:57 DEBUG ----- output ----- 22:24:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:24:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:24:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:24:57 DEBUG ----- output ----- 22:24:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:24:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:24:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:24:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 22:24:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:24:57 DEBUG ----- output ----- 22:24:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:24:57 DEBUG { 22:24:57 DEBUG ----- output ----- 22:24:57 DEBUG { 22:24:57 DEBUG ----- output ----- 22:24:57 DEBUG ----- output ----- 22:24:57 DEBUG ----- output ----- 22:24:57 DEBUG { 22:24:57 DEBUG ----- output ----- 22:24:57 DEBUG ----- output ----- 22:24:57 DEBUG ----- output ----- 22:24:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:24:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:24:57 DEBUG ----- output ----- 22:24:57 DEBUG { 22:24:57 DEBUG ----- output ----- 22:24:57 DEBUG ----- output ----- 22:24:57 DEBUG ----- output ----- 22:24:57 DEBUG { 22:24:57 DEBUG ----- output ----- 22:24:57 DEBUG ----- output ----- 22:24:57 DEBUG ----- output ----- 22:24:57 DEBUG ----- output ----- 22:24:57 DEBUG ----- output ----- 22:24:57 DEBUG { 22:24:57 DEBUG ----- output ----- 22:24:57 DEBUG "status": "success", 22:24:57 DEBUG ----- output ----- 22:24:57 DEBUG "status": "success", 22:24:57 DEBUG { 22:24:57 DEBUG { 22:24:57 DEBUG { 22:24:57 DEBUG "status": "success", 22:24:57 DEBUG { 22:24:57 DEBUG { 22:24:57 DEBUG { 22:24:57 DEBUG ----- output ----- 22:24:57 DEBUG ----- output ----- 22:24:57 DEBUG { 22:24:57 DEBUG "status": "success", 22:24:57 DEBUG { 22:24:57 DEBUG { 22:24:57 DEBUG { 22:24:57 DEBUG "status": "success", 22:24:57 DEBUG { 22:24:57 DEBUG { 22:24:57 DEBUG { 22:24:57 DEBUG { 22:24:57 DEBUG { 22:24:57 DEBUG "status": "success", 22:24:57 DEBUG { 22:24:57 DEBUG "data": { 22:24:57 DEBUG { 22:24:57 DEBUG "data": { 22:24:57 DEBUG "status": "success", 22:24:57 DEBUG "status": "success", 22:24:57 DEBUG "status": "success", 22:24:57 DEBUG "data": { 22:24:57 DEBUG "status": "success", 22:24:57 DEBUG "status": "success", 22:24:57 DEBUG "status": "success", 22:24:57 DEBUG { 22:24:57 DEBUG { 22:24:57 DEBUG "status": "success", 22:24:57 DEBUG "data": { 22:24:57 DEBUG "status": "success", 22:24:57 DEBUG "status": "success", 22:24:57 DEBUG "status": "success", 22:24:57 DEBUG "data": { 22:24:57 DEBUG "status": "success", 22:24:57 DEBUG "status": "success", 22:24:57 DEBUG "status": "success", 22:24:57 DEBUG "status": "success", 22:24:57 DEBUG "status": "success", 22:24:57 DEBUG "data": { 22:24:57 DEBUG "status": "success", 22:24:57 DEBUG "resultType": "vector", 22:24:57 DEBUG "status": "success", 22:24:57 DEBUG "resultType": "vector", 22:24:57 DEBUG "data": { 22:24:57 DEBUG "data": { 22:24:57 DEBUG "data": { 22:24:57 DEBUG "resultType": "vector", 22:24:57 DEBUG "result": [ 22:24:57 DEBUG "data": { 22:24:57 DEBUG "data": { 22:24:57 DEBUG "status": "success", 22:24:57 DEBUG "status": "success", 22:24:57 DEBUG "data": { 22:24:57 DEBUG "resultType": "vector", 22:24:57 DEBUG "data": { 22:24:57 DEBUG "data": { 22:24:57 DEBUG "data": { 22:24:57 DEBUG "resultType": "vector", 22:24:57 DEBUG "data": { 22:24:57 DEBUG "data": { 22:24:57 DEBUG "data": { 22:24:57 DEBUG "data": { 22:24:57 DEBUG "data": { 22:24:57 DEBUG "resultType": "vector", 22:24:57 DEBUG "data": { 22:24:57 DEBUG "result": [ 22:24:57 DEBUG "data": { 22:24:57 DEBUG "result": [ 22:24:57 DEBUG "resultType": "vector", 22:24:57 DEBUG "resultType": "vector", 22:24:57 DEBUG "resultType": "vector", 22:24:57 DEBUG "data": { 22:24:57 DEBUG { 22:24:57 DEBUG "resultType": "vector", 22:24:57 DEBUG "resultType": "vector", 22:24:57 DEBUG "data": { 22:24:57 DEBUG "data": { 22:24:57 DEBUG "resultType": "vector", 22:24:57 DEBUG "result": [ 22:24:57 DEBUG "resultType": "vector", 22:24:57 DEBUG "resultType": "vector", 22:24:57 DEBUG "resultType": "vector", 22:24:57 DEBUG "result": [ 22:24:57 DEBUG "resultType": "vector", 22:24:57 DEBUG "resultType": "vector", 22:24:57 DEBUG "resultType": "vector", 22:24:57 DEBUG "resultType": "vector", 22:24:57 DEBUG "resultType": "vector", 22:24:57 DEBUG "result": [ 22:24:57 DEBUG "resultType": "vector", 22:24:57 DEBUG { 22:24:57 DEBUG "resultType": "vector", 22:24:57 DEBUG { 22:24:57 DEBUG "result": [ 22:24:57 DEBUG "result": [ 22:24:57 DEBUG "result": [ 22:24:57 DEBUG "resultType": "vector", 22:24:57 DEBUG "metric": { 22:24:57 DEBUG "result": [ 22:24:57 DEBUG "result": [ 22:24:57 DEBUG "resultType": "vector", 22:24:57 DEBUG "resultType": "vector", 22:24:57 DEBUG "result": [ 22:24:57 DEBUG { 22:24:57 DEBUG "result": [ 22:24:57 DEBUG "result": [ 22:24:57 DEBUG "result": [ 22:24:57 DEBUG { 22:24:57 DEBUG "result": [ 22:24:57 DEBUG "result": [ 22:24:57 DEBUG "result": [ 22:24:57 DEBUG "result": [ 22:24:57 DEBUG "result": [ 22:24:57 DEBUG { 22:24:57 DEBUG "result": [ 22:24:57 DEBUG "metric": { 22:24:57 DEBUG "result": [ 22:24:57 DEBUG "metric": { 22:24:57 DEBUG { 22:24:57 DEBUG { 22:24:57 DEBUG { 22:24:57 DEBUG "result": [ 22:24:57 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:24:57 DEBUG { 22:24:57 DEBUG { 22:24:57 DEBUG "result": [ 22:24:57 DEBUG "result": [ 22:24:57 DEBUG { 22:24:57 DEBUG "metric": { 22:24:57 DEBUG { 22:24:57 DEBUG { 22:24:57 DEBUG { 22:24:57 DEBUG "metric": { 22:24:57 DEBUG { 22:24:57 DEBUG { 22:24:57 DEBUG { 22:24:57 DEBUG { 22:24:57 DEBUG { 22:24:57 DEBUG "metric": { 22:24:57 DEBUG { 22:24:57 DEBUG "pod": "am-555f4db595-fz8sn" 22:24:57 DEBUG { 22:24:57 DEBUG "instance": "10.142.0.107:9100" 22:24:57 DEBUG "metric": { 22:24:57 DEBUG "metric": { 22:24:57 DEBUG "metric": { 22:24:57 DEBUG { 22:24:57 DEBUG }, 22:24:57 DEBUG "metric": { 22:24:57 DEBUG "metric": { 22:24:57 DEBUG { 22:24:57 DEBUG { 22:24:57 DEBUG "metric": { 22:24:57 DEBUG "pod": "am-555f4db595-fz8sn" 22:24:57 DEBUG "metric": { 22:24:57 DEBUG "metric": { 22:24:57 DEBUG "metric": { 22:24:57 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:24:57 DEBUG "metric": { 22:24:57 DEBUG "metric": { 22:24:57 DEBUG "metric": { 22:24:57 DEBUG "metric": { 22:24:57 DEBUG "metric": { 22:24:57 DEBUG "pod": "am-555f4db595-fz8sn" 22:24:57 DEBUG "metric": { 22:24:57 DEBUG }, 22:24:57 DEBUG "metric": { 22:24:57 DEBUG }, 22:24:57 DEBUG "instance": "10.142.0.107:9100" 22:24:57 DEBUG "instance": "10.142.0.107:9100" 22:24:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:24:57 DEBUG "metric": { 22:24:57 DEBUG "value": [ 22:24:57 DEBUG "instance": "10.142.0.107:9100" 22:24:57 DEBUG "pod": "ds-idrepo-2" 22:24:57 DEBUG "metric": { 22:24:57 DEBUG "metric": { 22:24:57 DEBUG "pod": "ds-idrepo-2" 22:24:57 DEBUG }, 22:24:57 DEBUG "instance": "10.142.0.107:9100" 22:24:57 DEBUG "pod": "am-555f4db595-fz8sn" 22:24:57 DEBUG "pod": "am-555f4db595-fz8sn" 22:24:57 DEBUG }, 22:24:57 DEBUG "pod": "ds-idrepo-2" 22:24:57 DEBUG "pod": "ds-idrepo-2" 22:24:57 DEBUG "pod": "ds-idrepo-2" 22:24:57 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:24:57 DEBUG }, 22:24:57 DEBUG }, 22:24:57 DEBUG "value": [ 22:24:57 DEBUG "value": [ 22:24:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:24:57 DEBUG "value": [ 22:24:57 DEBUG }, 22:24:57 DEBUG }, 22:24:57 DEBUG }, 22:24:57 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:24:57 DEBUG 22:24:57 DEBUG }, 22:24:57 DEBUG }, 22:24:57 DEBUG "pod": "am-555f4db595-fz8sn" 22:24:57 DEBUG "pod": "am-555f4db595-fz8sn" 22:24:57 DEBUG }, 22:24:57 DEBUG "value": [ 22:24:57 DEBUG }, 22:24:57 DEBUG }, 22:24:57 DEBUG }, 22:24:57 DEBUG "value": [ 22:24:57 DEBUG }, 22:24:57 DEBUG }, 22:24:57 DEBUG }, 22:24:57 DEBUG "pod": "ds-idrepo-2" 22:24:57 DEBUG "value": [ 22:24:57 DEBUG 22:24:57 DEBUG "pod": "ds-idrepo-2" 22:24:57 DEBUG 22:24:57 DEBUG }, 22:24:57 DEBUG 22:24:57 DEBUG "value": [ 22:24:57 DEBUG "value": [ 22:24:57 DEBUG "value": [ 22:24:57 DEBUG }, 22:24:57 DEBUG 22:24:57 DEBUG "value": [ 22:24:57 DEBUG "value": [ 22:24:57 DEBUG }, 22:24:57 DEBUG }, 22:24:57 DEBUG "value": [ 22:24:57 DEBUG 22:24:57 DEBUG "value": [ 22:24:57 DEBUG "value": [ 22:24:57 DEBUG "value": [ 22:24:57 DEBUG 22:24:57 DEBUG "value": [ 22:24:57 DEBUG "value": [ 22:24:57 DEBUG 22:24:57 DEBUG }, 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG }, 22:24:57 DEBUG 22:24:57 DEBUG "value": [ 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG "value": [ 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG "value": [ 22:24:57 DEBUG "value": [ 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG "value": [ 22:24:57 DEBUG 22:24:57 DEBUG "value": [ 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG "value": [ 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG [...] 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG [...] 22:24:57 DEBUG 22:24:57 DEBUG [...] 22:24:57 DEBUG 22:24:57 DEBUG [...] 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG [...] 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG [...] 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG [...] 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG [...] 22:24:57 DEBUG 22:24:57 DEBUG [...] 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG [...] 22:24:57 DEBUG [...] 22:24:57 DEBUG [...] 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG [...] 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG [...] 22:24:57 DEBUG 22:24:57 DEBUG [...] 22:24:57 DEBUG [...] 22:24:57 DEBUG [...] 22:24:57 DEBUG 22:24:57 DEBUG [...] 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG [...] 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG [...] 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG [...] 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG [...] 22:24:57 DEBUG 22:24:57 DEBUG [...] 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG [...] 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG "pod": "ds-idrepo-2" 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG [...] 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG "pod": "am-555f4db595-kzxsp" 22:24:57 DEBUG 22:24:57 DEBUG "pod": "am-555f4db595-kzxsp" 22:24:57 DEBUG 22:24:57 DEBUG "instance": "10.142.0.43:9100" 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG }, 22:24:57 DEBUG 22:24:57 DEBUG "pod": "ds-idrepo-0" 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG "pod": "am-555f4db595-kzxsp" 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG "pod": "ds-idrepo-2" 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG "pod": "ds-idrepo-0" 22:24:57 DEBUG 22:24:57 DEBUG "pod": "ds-idrepo-2" 22:24:57 DEBUG }, 22:24:57 DEBUG 22:24:57 DEBUG }, 22:24:57 DEBUG 22:24:57 DEBUG }, 22:24:57 DEBUG "instance": "10.142.0.43:9100" 22:24:57 DEBUG "instance": "10.142.0.43:9100" 22:24:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:24:57 DEBUG 22:24:57 DEBUG "value": [ 22:24:57 DEBUG "instance": "10.142.0.43:9100" 22:24:57 DEBUG }, 22:24:57 DEBUG 22:24:57 DEBUG 22:24:57 DEBUG "pod": "ds-idrepo-0" 22:24:57 DEBUG }, 22:24:57 DEBUG "instance": "10.142.0.43:9100" 22:24:57 DEBUG "pod": "am-555f4db595-kzxsp" 22:24:57 DEBUG "pod": "am-555f4db595-kzxsp" 22:24:57 DEBUG }, 22:24:57 DEBUG "pod": "ds-idrepo-0" 22:24:57 DEBUG 22:24:57 DEBUG }, 22:24:57 DEBUG 22:24:57 DEBUG }, 22:24:57 DEBUG "value": [ 22:24:57 DEBUG 22:24:57 DEBUG "value": [ 22:24:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:24:57 DEBUG "value": [ 22:24:57 DEBUG }, 22:24:57 DEBUG }, 22:24:57 DEBUG }, 22:24:57 DEBUG "pod": "ds-idrepo-2" 22:24:57 DEBUG 1713302696, 22:24:57 DEBUG }, 22:24:57 DEBUG "value": [ 22:24:57 DEBUG 22:24:57 DEBUG "pod": "am-555f4db595-kzxsp" 22:24:57 DEBUG }, 22:24:57 DEBUG "value": [ 22:24:57 DEBUG }, 22:24:57 DEBUG }, 22:24:57 DEBUG }, 22:24:57 DEBUG "value": [ 22:24:57 DEBUG }, 22:24:57 DEBUG "pod": "ds-idrepo-0" 22:24:57 DEBUG "value": [ 22:24:57 DEBUG "pod": "ds-cts-2" 22:24:57 DEBUG "value": [ 22:24:57 DEBUG 1713302696, 22:24:57 DEBUG "pod": "ds-idrepo-0" 22:24:57 DEBUG 1713302696, 22:24:57 DEBUG }, 22:24:57 DEBUG 1713302696, 22:24:57 DEBUG "value": [ 22:24:57 DEBUG "value": [ 22:24:57 DEBUG "value": [ 22:24:57 DEBUG }, 22:24:57 DEBUG "435.5135650780325" 22:24:57 DEBUG "value": [ 22:24:57 DEBUG 1713302696, 22:24:57 DEBUG "pod": "am-555f4db595-kzxsp" 22:24:57 DEBUG }, 22:24:57 DEBUG "value": [ 22:24:57 DEBUG 1713302696, 22:24:57 DEBUG "value": [ 22:24:57 DEBUG "value": [ 22:24:57 DEBUG "value": [ 22:24:57 DEBUG 1713302696, 22:24:57 DEBUG "value": [ 22:24:57 DEBUG }, 22:24:57 DEBUG 1713302696, 22:24:57 DEBUG }, 22:24:57 DEBUG 1713302696, 22:24:57 DEBUG "0" 22:24:57 DEBUG }, 22:24:57 DEBUG "0.024" 22:24:57 DEBUG "value": [ 22:24:57 DEBUG "45439477" 22:24:57 DEBUG 1713302696, 22:24:57 DEBUG 1713302696, 22:24:57 DEBUG 1713302696, 22:24:57 DEBUG "value": [ 22:24:57 DEBUG ] 22:24:57 DEBUG 1713302696, 22:24:57 DEBUG "0" 22:24:57 DEBUG }, 22:24:57 DEBUG "value": [ 22:24:57 DEBUG 1713302696, 22:24:57 DEBUG "1996.6333333333332" 22:24:57 DEBUG 1713302696, 22:24:57 DEBUG 1713302696, 22:24:57 DEBUG 1713302696, 22:24:57 DEBUG "4028.5004769718007" 22:24:57 DEBUG 1713302696, 22:24:57 DEBUG "value": [ 22:24:57 DEBUG "0" 22:24:57 DEBUG "value": [ 22:24:57 DEBUG "0" 22:24:57 DEBUG ] 22:24:57 DEBUG "value": [ 22:24:57 DEBUG ] 22:24:57 DEBUG 1713302696, 22:24:57 DEBUG ] 22:24:57 DEBUG "45449364.46666666" 22:24:57 DEBUG "19954531328" 22:24:57 DEBUG "647475200" 22:24:57 DEBUG 1713302696, 22:24:57 DEBUG } 22:24:57 DEBUG "0.000333333333333341" 22:24:57 DEBUG ] 22:24:57 DEBUG "value": [ 22:24:57 DEBUG 1713302696, 22:24:57 DEBUG "0" 22:24:57 DEBUG ] 22:24:57 DEBUG "0.00046666666666700016" 22:24:57 DEBUG "1996.7" 22:24:57 DEBUG "0" 22:24:57 DEBUG ] 22:24:57 DEBUG "0" 22:24:57 DEBUG 1713302696, 22:24:57 DEBUG ] 22:24:57 DEBUG 1713302696, 22:24:57 DEBUG "0" 22:24:57 DEBUG } 22:24:57 DEBUG 1713302696, 22:24:57 DEBUG } 22:24:57 DEBUG "6.292178224713934" 22:24:57 DEBUG } 22:24:57 DEBUG ] 22:24:57 DEBUG ] 22:24:57 DEBUG ] 22:24:57 DEBUG "3.0870632116514427" 22:24:57 DEBUG ] 22:24:57 DEBUG ] 22:24:57 DEBUG } 22:24:57 DEBUG 1713302696, 22:24:57 DEBUG "1996.7666666666667" 22:24:57 DEBUG ] 22:24:57 DEBUG } 22:24:57 DEBUG ] 22:24:57 DEBUG ] 22:24:57 DEBUG ] 22:24:57 DEBUG } 22:24:57 DEBUG ] 22:24:57 DEBUG "0" 22:24:57 DEBUG } 22:24:57 DEBUG ] 22:24:57 DEBUG ] 22:24:57 DEBUG ] 22:24:57 DEBUG "0" 22:24:57 DEBUG ] 22:24:57 DEBUG ] 22:24:57 DEBUG ] 22:24:57 DEBUG } 22:24:57 DEBUG } 22:24:57 DEBUG ] 22:24:57 DEBUG ] 22:24:57 DEBUG } 22:24:57 DEBUG ] 22:24:57 DEBUG } 22:24:57 DEBUG "1580022" 22:24:57 DEBUG ] 22:24:57 DEBUG } 22:24:57 DEBUG ] 22:24:57 DEBUG } 22:24:57 DEBUG } 22:24:57 DEBUG } 22:24:57 DEBUG ] 22:24:57 DEBUG } 22:24:57 DEBUG ] 22:24:57 DEBUG ] 22:24:57 DEBUG } 22:24:57 DEBUG } 22:24:57 DEBUG } 22:24:57 DEBUG ] 22:24:57 DEBUG } 22:24:57 DEBUG } 22:24:57 DEBUG } 22:24:57 DEBUG ] 22:24:57 DEBUG } 22:24:57 DEBUG } 22:24:57 DEBUG } 22:24:57 DEBUG } 22:24:57 DEBUG } 22:24:57 DEBUG ] 22:24:57 DEBUG ] 22:24:57 DEBUG } 22:24:57 DEBUG ] 22:24:57 DEBUG } 22:24:57 DEBUG ] 22:24:57 DEBUG ] 22:24:57 DEBUG ] 22:24:57 DEBUG } 22:24:57 DEBUG ] 22:24:57 DEBUG } 22:24:57 DEBUG } 22:24:57 DEBUG ] 22:24:57 DEBUG ] 22:24:57 DEBUG } 22:24:57 DEBUG } 22:24:57 DEBUG } 22:24:57 DEBUG ] 22:24:57 DEBUG } 22:24:57 DEBUG } 22:24:57 DEBUG ] 22:24:57 DEBUG } 22:24:57 DEBUG ] 22:24:57 DEBUG } 22:24:57 DEBUG } 22:24:57 DEBUG } 22:24:57 DEBUG ] 22:24:57 DEBUG } 22:24:57 DEBUG } 22:24:57 DEBUG } 22:24:57 DEBUG } 22:24:57 DEBUG } 22:24:57 DEBUG } 22:24:57 DEBUG } 22:24:57 DEBUG ] 22:24:57 DEBUG } 22:24:57 DEBUG } 22:24:57 DEBUG } 22:24:57 DEBUG ] 22:24:57 DEBUG } 22:24:57 DEBUG } 22:24:57 DEBUG } 22:24:57 DEBUG } 22:24:57 DEBUG } 22:24:57 DEBUG ] 22:24:57 DEBUG } 22:24:57 DEBUG } 22:24:57 DEBUG } 22:24:57 DEBUG } 22:24:57 DEBUG } 22:24:57 DEBUG } 22:24:57 DEBUG } 22:24:57 DEBUG } 22:24:57 DEBUG } 22:24:57 DEBUG } 22:24:57 DEBUG } 22:24:57 DEBUG } 22:24:57 DEBUG } 22:24:57 DEBUG } 22:24:57 DEBUG } 22:24:57 DEBUG } 22:24:57 DEBUG } 22:24:57 DEBUG } 22:24:57 INFO [loop_until]: OK (rc = 0) 22:24:57 DEBUG --- stdout --- 22:24:57 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 5086Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7091m 23% 14736Mi 13% gke-xlou-cdm-default-pool-1b0769c8-nhlf 9398m 31% 10813Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 89m 0% 2646Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 4728m 15% 6458Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 8490m 28% 16590Mi 14% gke-xlou-cdm-default-pool-1b0769c8-xnmp 65m 0% 3162Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 9323m 31% 10368Mi 9% gke-xlou-cdm-ds-0f601751-hmm7 4714m 15% 5575Mi 4% gke-xlou-cdm-ds-0f601751-j488 6992m 23% 14075Mi 12% gke-xlou-cdm-ds-0f601751-mc5d 10629m 35% 3346Mi 2% gke-xlou-cdm-ds-0f601751-nj6q 9432m 31% 10412Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 4764m 15% 7355Mi 6% gke-xlou-cdm-frontend-a8771548-i82n 6423m 40% 5949Mi 10% 22:24:57 DEBUG --- stderr --- 22:24:57 DEBUG 22:25:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302755" 22:25:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713302755" 22:25:55 ERROR [http_cmd]: ERROR 22:25:55 INFO [http_cmd]: http status code OK 22:25:55 ERROR --- status code --- 22:25:55 DEBUG --- status code --- 22:25:55 ERROR http status code is 400 (expected [200, 401, 403, 404]) 22:25:55 ERROR --- http response --- 22:25:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:25:55 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 22:25:55 DEBUG --- http response --- 22:25:55 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 22:25:55 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 22:25:56 INFO 22:25:56 INFO [loop_until]: kubectl --namespace=xlou top pods 22:25:56 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:25:56 INFO [loop_until]: OK (rc = 0) 22:25:56 DEBUG --- stdout --- 22:25:56 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 9Mi am-555f4db595-4lbss 9282m 8589Mi am-555f4db595-fz8sn 9466m 8602Mi am-555f4db595-kzxsp 8973m 8553Mi ds-cts-0 7648m 12100Mi ds-cts-1 7315m 12187Mi ds-cts-2 7169m 12152Mi ds-idrepo-0 4734m 5032Mi ds-idrepo-1 4386m 2850Mi ds-idrepo-2 4498m 3115Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 3m 1207Mi idm-b85f9d978-rl76f 4m 1226Mi lodemon-bc756c647-cr7x6 15m 63Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-7c9b667974-9mr2s 6353m 527Mi 22:25:56 DEBUG --- stderr --- 22:25:56 DEBUG 22:25:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302757" 22:25:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302757" 22:25:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302757" 22:25:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302757" 22:25:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302757" 22:25:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713302757" 22:25:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302757" 22:25:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302757" 22:25:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302757" 22:25:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302757" 22:25:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302757" 22:25:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302757" 22:25:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302757" 22:25:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302757" 22:25:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302757" 22:25:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302757" 22:25:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302757" 22:25:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302757" 22:25:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713302757" 22:25:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713302757" 22:25:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302757" 22:25:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302757" 22:25:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713302757" 22:25:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302757" 22:25:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302757" 22:25:57 INFO 22:25:57 INFO [http_cmd]: http status code OK 22:25:57 INFO [http_cmd]: http status code OK 22:25:57 INFO [http_cmd]: http status code OK 22:25:57 INFO [http_cmd]: http status code OK 22:25:57 INFO [http_cmd]: http status code OK 22:25:57 INFO [http_cmd]: http status code OK 22:25:57 INFO [http_cmd]: http status code OK 22:25:57 INFO [http_cmd]: http status code OK 22:25:57 INFO [http_cmd]: http status code OK 22:25:57 INFO [http_cmd]: http status code OK 22:25:57 INFO [http_cmd]: http status code OK 22:25:57 INFO [http_cmd]: http status code OK 22:25:57 INFO [http_cmd]: http status code OK 22:25:57 INFO [http_cmd]: http status code OK 22:25:57 INFO [loop_until]: kubectl --namespace=xlou top node 22:25:57 DEBUG --- status code --- 22:25:57 INFO [http_cmd]: http status code OK 22:25:57 DEBUG --- status code --- 22:25:57 INFO [http_cmd]: http status code OK 22:25:57 INFO [http_cmd]: http status code OK 22:25:57 INFO [http_cmd]: http status code OK 22:25:57 INFO [http_cmd]: http status code OK 22:25:57 DEBUG --- status code --- 22:25:57 DEBUG --- status code --- 22:25:57 INFO [http_cmd]: http status code OK 22:25:57 INFO [http_cmd]: http status code OK 22:25:57 DEBUG --- status code --- 22:25:57 DEBUG --- status code --- 22:25:57 DEBUG --- status code --- 22:25:57 DEBUG --- status code --- 22:25:57 INFO [http_cmd]: http status code OK 22:25:57 DEBUG --- status code --- 22:25:57 DEBUG --- status code --- 22:25:57 INFO [http_cmd]: http status code OK 22:25:57 DEBUG --- status code --- 22:25:57 INFO [http_cmd]: http status code OK 22:25:57 DEBUG --- status code --- 22:25:57 DEBUG --- status code --- 22:25:57 DEBUG --- status code --- 22:25:57 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:25:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:25:57 DEBUG --- status code --- 22:25:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:25:57 DEBUG --- status code --- 22:25:57 DEBUG --- status code --- 22:25:57 DEBUG --- status code --- 22:25:57 DEBUG --- status code --- 22:25:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:25:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:25:57 DEBUG --- status code --- 22:25:57 DEBUG --- status code --- 22:25:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:25:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:25:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:25:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:25:57 DEBUG --- status code --- 22:25:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:25:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:25:57 DEBUG --- status code --- 22:25:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:25:57 DEBUG --- status code --- 22:25:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:25:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:25:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:25:57 DEBUG --- http response --- 22:25:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:25:57 DEBUG --- http response --- 22:25:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:25:57 INFO [http_cmd]: http status code OK 22:25:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:25:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:25:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:25:57 DEBUG --- http response --- 22:25:57 DEBUG --- http response --- 22:25:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:25:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:25:57 DEBUG --- http response --- 22:25:57 DEBUG --- http response --- 22:25:57 DEBUG --- http response --- 22:25:57 DEBUG --- http response --- 22:25:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:25:57 DEBUG --- http response --- 22:25:57 DEBUG --- http response --- 22:25:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:25:57 DEBUG --- http response --- 22:25:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:25:57 DEBUG --- http response --- 22:25:57 DEBUG --- http response --- 22:25:57 DEBUG --- http response --- 22:25:57 DEBUG --- http response --- 22:25:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:25:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:25:57 DEBUG --- http response --- 22:25:57 DEBUG --- status code --- 22:25:57 DEBUG --- http response --- 22:25:57 DEBUG --- http response --- 22:25:57 DEBUG --- http response --- 22:25:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:25:57 DEBUG --- http response --- 22:25:57 DEBUG --- http response --- 22:25:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:25:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:25:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:25:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:25:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:25:57 DEBUG --- http response --- 22:25:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:25:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:25:57 DEBUG --- http response --- 22:25:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 22:25:57 DEBUG --- http response --- 22:25:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:25:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:25:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:25:57 DEBUG ----- output ----- 22:25:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:25:57 DEBUG ----- output ----- 22:25:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:25:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:25:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 22:25:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:25:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:25:57 DEBUG ----- output ----- 22:25:57 DEBUG ----- output ----- 22:25:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:25:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:25:57 DEBUG ----- output ----- 22:25:57 DEBUG ----- output ----- 22:25:57 DEBUG ----- output ----- 22:25:57 DEBUG ----- output ----- 22:25:57 DEBUG ----- output ----- 22:25:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:25:57 DEBUG ----- output ----- 22:25:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:25:57 DEBUG ----- output ----- 22:25:57 DEBUG ----- output ----- 22:25:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:25:57 DEBUG ----- output ----- 22:25:57 DEBUG { 22:25:57 DEBUG { 22:25:57 DEBUG ----- output ----- 22:25:57 DEBUG { 22:25:57 DEBUG ----- output ----- 22:25:57 DEBUG --- http response --- 22:25:57 DEBUG ----- output ----- 22:25:57 DEBUG ----- output ----- 22:25:57 DEBUG ----- output ----- 22:25:57 DEBUG { 22:25:57 DEBUG { 22:25:57 DEBUG ----- output ----- 22:25:57 DEBUG ----- output ----- 22:25:57 DEBUG { 22:25:57 DEBUG { 22:25:57 DEBUG { 22:25:57 DEBUG { 22:25:57 DEBUG { 22:25:57 DEBUG ----- output ----- 22:25:57 DEBUG { 22:25:57 DEBUG ----- output ----- 22:25:57 DEBUG { 22:25:57 DEBUG { 22:25:57 DEBUG ----- output ----- 22:25:57 DEBUG "status": "success", 22:25:57 DEBUG ----- output ----- 22:25:57 DEBUG "status": "success", 22:25:57 DEBUG { 22:25:57 DEBUG "status": "success", 22:25:57 DEBUG { 22:25:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:25:57 DEBUG { 22:25:57 DEBUG { 22:25:57 DEBUG { 22:25:57 DEBUG "status": "success", 22:25:57 DEBUG "status": "success", 22:25:57 DEBUG { 22:25:57 DEBUG { 22:25:57 DEBUG "status": "success", 22:25:57 DEBUG "status": "success", 22:25:57 DEBUG "status": "success", 22:25:57 DEBUG "status": "success", 22:25:57 DEBUG "status": "success", 22:25:57 DEBUG { 22:25:57 DEBUG "status": "success", 22:25:57 DEBUG { 22:25:57 DEBUG "status": "success", 22:25:57 DEBUG "status": "success", 22:25:57 DEBUG { 22:25:57 DEBUG "data": { 22:25:57 DEBUG { 22:25:57 DEBUG "data": { 22:25:57 DEBUG "status": "success", 22:25:57 DEBUG "data": { 22:25:57 DEBUG "status": "success", 22:25:57 DEBUG ----- output ----- 22:25:57 DEBUG "status": "success", 22:25:57 DEBUG "status": "success", 22:25:57 DEBUG "status": "success", 22:25:57 DEBUG "data": { 22:25:57 DEBUG "data": { 22:25:57 DEBUG "status": "success", 22:25:57 DEBUG "status": "success", 22:25:57 DEBUG "data": { 22:25:57 DEBUG "data": { 22:25:57 DEBUG "resultType": "vector", 22:25:57 DEBUG "data": { 22:25:57 DEBUG "data": { 22:25:57 DEBUG "status": "success", 22:25:57 DEBUG "data": { 22:25:57 DEBUG "status": "success", 22:25:57 DEBUG "data": { 22:25:57 DEBUG "data": { 22:25:57 DEBUG "status": "success", 22:25:57 DEBUG "resultType": "vector", 22:25:57 DEBUG "status": "success", 22:25:57 DEBUG "data": { 22:25:57 DEBUG "data": { 22:25:57 DEBUG "resultType": "vector", 22:25:57 DEBUG "data": { 22:25:57 DEBUG { 22:25:57 DEBUG "data": { 22:25:57 DEBUG "data": { 22:25:57 DEBUG "data": { 22:25:57 DEBUG "resultType": "vector", 22:25:57 DEBUG "resultType": "vector", 22:25:57 DEBUG "data": { 22:25:57 DEBUG "data": { 22:25:57 DEBUG "resultType": "vector", 22:25:57 DEBUG "data": { 22:25:57 DEBUG "result": [ 22:25:57 DEBUG "resultType": "vector", 22:25:57 DEBUG "resultType": "vector", 22:25:57 DEBUG "data": { 22:25:57 DEBUG "resultType": "vector", 22:25:57 DEBUG "data": { 22:25:57 DEBUG "resultType": "vector", 22:25:57 DEBUG "resultType": "vector", 22:25:57 DEBUG "data": { 22:25:57 DEBUG "result": [ 22:25:57 DEBUG "resultType": "vector", 22:25:57 DEBUG "resultType": "vector", 22:25:57 DEBUG "resultType": "vector", 22:25:57 DEBUG "result": [ 22:25:57 DEBUG "resultType": "vector", 22:25:57 DEBUG "status": "success", 22:25:57 DEBUG "resultType": "vector", 22:25:57 DEBUG "resultType": "vector", 22:25:57 DEBUG "resultType": "vector", 22:25:57 DEBUG "result": [ 22:25:57 DEBUG "result": [ 22:25:57 DEBUG "resultType": "vector", 22:25:57 DEBUG "resultType": "vector", 22:25:57 DEBUG "result": [ 22:25:57 DEBUG "resultType": "vector", 22:25:57 DEBUG { 22:25:57 DEBUG "result": [ 22:25:57 DEBUG "result": [ 22:25:57 DEBUG "resultType": "vector", 22:25:57 DEBUG "result": [ 22:25:57 DEBUG "resultType": "vector", 22:25:57 DEBUG "result": [ 22:25:57 DEBUG "result": [ 22:25:57 DEBUG "resultType": "vector", 22:25:57 DEBUG { 22:25:57 DEBUG "result": [ 22:25:57 DEBUG "result": [ 22:25:57 DEBUG "result": [ 22:25:57 DEBUG { 22:25:57 DEBUG "result": [ 22:25:57 DEBUG "data": { 22:25:57 DEBUG "result": [ 22:25:57 DEBUG "result": [ 22:25:57 DEBUG "result": [ 22:25:57 DEBUG { 22:25:57 DEBUG "metric": { 22:25:57 DEBUG "result": [ 22:25:57 DEBUG "result": [ 22:25:57 DEBUG { 22:25:57 DEBUG "result": [ 22:25:57 DEBUG "metric": { 22:25:57 DEBUG { 22:25:57 DEBUG { 22:25:57 DEBUG "result": [ 22:25:57 DEBUG { 22:25:57 DEBUG "result": [ 22:25:57 DEBUG { 22:25:57 DEBUG { 22:25:57 DEBUG "result": [ 22:25:57 DEBUG "metric": { 22:25:57 DEBUG { 22:25:57 DEBUG { 22:25:57 DEBUG { 22:25:57 DEBUG "metric": { 22:25:57 DEBUG { 22:25:57 DEBUG "resultType": "vector", 22:25:57 DEBUG { 22:25:57 DEBUG { 22:25:57 DEBUG { 22:25:57 DEBUG { 22:25:57 DEBUG "instance": "10.142.0.107:9100" 22:25:57 DEBUG { 22:25:57 DEBUG { 22:25:57 DEBUG "metric": { 22:25:57 DEBUG { 22:25:57 DEBUG "pod": "am-555f4db595-fz8sn" 22:25:57 DEBUG }, 22:25:57 DEBUG "metric": { 22:25:57 DEBUG { 22:25:57 DEBUG "metric": { 22:25:57 DEBUG { 22:25:57 DEBUG "metric": { 22:25:57 DEBUG "metric": { 22:25:57 DEBUG { 22:25:57 DEBUG "instance": "10.142.0.107:9100" 22:25:57 DEBUG "metric": { 22:25:57 DEBUG "metric": { 22:25:57 DEBUG "metric": { 22:25:57 DEBUG "pod": "am-555f4db595-fz8sn" 22:25:57 DEBUG "metric": { 22:25:57 DEBUG "result": [ 22:25:57 DEBUG "metric": { 22:25:57 DEBUG "metric": { 22:25:57 DEBUG "metric": { 22:25:57 DEBUG "metric": { 22:25:57 DEBUG }, 22:25:57 DEBUG "metric": { 22:25:57 DEBUG "metric": { 22:25:57 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:25:57 DEBUG "metric": { 22:25:57 DEBUG "metric": { 22:25:57 DEBUG "value": [ 22:25:57 DEBUG "instance": "10.142.0.107:9100" 22:25:57 DEBUG "metric": { 22:25:57 DEBUG "pod": "ds-idrepo-2" 22:25:57 DEBUG "metric": { 22:25:57 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:25:57 DEBUG "pod": "ds-idrepo-2" 22:25:57 DEBUG "metric": { 22:25:57 DEBUG }, 22:25:57 DEBUG "instance": "10.142.0.107:9100" 22:25:57 DEBUG "pod": "am-555f4db595-fz8sn" 22:25:57 DEBUG "pod": "ds-idrepo-2" 22:25:57 DEBUG }, 22:25:57 DEBUG "pod": "am-555f4db595-fz8sn" 22:25:57 DEBUG { 22:25:57 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:25:57 DEBUG "pod": "am-555f4db595-fz8sn" 22:25:57 DEBUG "pod": "am-555f4db595-fz8sn" 22:25:57 DEBUG "instance": "10.142.0.107:9100" 22:25:57 DEBUG "value": [ 22:25:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:25:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:25:57 DEBUG }, 22:25:57 DEBUG "pod": "am-555f4db595-fz8sn" 22:25:57 DEBUG "pod": "ds-idrepo-2" 22:25:57 DEBUG 22:25:57 DEBUG }, 22:25:57 DEBUG "pod": "ds-idrepo-2" 22:25:57 DEBUG }, 22:25:57 DEBUG "pod": "ds-idrepo-2" 22:25:57 DEBUG }, 22:25:57 DEBUG }, 22:25:57 DEBUG "pod": "ds-idrepo-2" 22:25:57 DEBUG "value": [ 22:25:57 DEBUG }, 22:25:57 DEBUG }, 22:25:57 DEBUG }, 22:25:57 DEBUG "value": [ 22:25:57 DEBUG }, 22:25:57 DEBUG "metric": { 22:25:57 DEBUG }, 22:25:57 DEBUG }, 22:25:57 DEBUG }, 22:25:57 DEBUG }, 22:25:57 DEBUG 22:25:57 DEBUG }, 22:25:57 DEBUG }, 22:25:57 DEBUG "value": [ 22:25:57 DEBUG }, 22:25:57 DEBUG }, 22:25:57 DEBUG 22:25:57 DEBUG "value": [ 22:25:57 DEBUG }, 22:25:57 DEBUG "value": [ 22:25:57 DEBUG }, 22:25:57 DEBUG "value": [ 22:25:57 DEBUG "value": [ 22:25:57 DEBUG }, 22:25:57 DEBUG 22:25:57 DEBUG "value": [ 22:25:57 DEBUG "value": [ 22:25:57 DEBUG "value": [ 22:25:57 DEBUG 22:25:57 DEBUG "value": [ 22:25:57 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:25:57 DEBUG "value": [ 22:25:57 DEBUG "value": [ 22:25:57 DEBUG "value": [ 22:25:57 DEBUG "value": [ 22:25:57 DEBUG 22:25:57 DEBUG "value": [ 22:25:57 DEBUG "value": [ 22:25:57 DEBUG 22:25:57 DEBUG "value": [ 22:25:57 DEBUG "value": [ 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG "value": [ 22:25:57 DEBUG 22:25:57 DEBUG "value": [ 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG "value": [ 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG }, 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG [...] 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG "value": [ 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG [...] 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG [...] 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG [...] 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG [...] 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG [...] 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG [...] 22:25:57 DEBUG 22:25:57 DEBUG [...] 22:25:57 DEBUG [...] 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG [...] 22:25:57 DEBUG [...] 22:25:57 DEBUG [...] 22:25:57 DEBUG 22:25:57 DEBUG [...] 22:25:57 DEBUG 22:25:57 DEBUG [...] 22:25:57 DEBUG [...] 22:25:57 DEBUG [...] 22:25:57 DEBUG [...] 22:25:57 DEBUG 22:25:57 DEBUG [...] 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG [...] 22:25:57 DEBUG [...] 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG [...] 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG [...] 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG [...] 22:25:57 DEBUG 22:25:57 DEBUG "instance": "10.142.0.43:9100" 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG "pod": "am-555f4db595-kzxsp" 22:25:57 DEBUG [...] 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG "pod": "am-555f4db595-kzxsp" 22:25:57 DEBUG [...] 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG }, 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG }, 22:25:57 DEBUG 22:25:57 DEBUG "instance": "10.142.0.43:9100" 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG "instance": "10.142.0.43:9100" 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG }, 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG "value": [ 22:25:57 DEBUG "pod": "am-555f4db595-kzxsp" 22:25:57 DEBUG 22:25:57 DEBUG 22:25:57 DEBUG "value": [ 22:25:57 DEBUG 22:25:57 DEBUG }, 22:25:57 DEBUG "pod": "ds-idrepo-0" 22:25:57 DEBUG 22:25:57 DEBUG "pod": "idm-b85f9d978-j5pbm" 22:25:57 DEBUG "pod": "ds-idrepo-0" 22:25:57 DEBUG 22:25:57 DEBUG }, 22:25:57 DEBUG "instance": "10.142.0.43:9100" 22:25:57 DEBUG "pod": "am-555f4db595-kzxsp" 22:25:57 DEBUG "pod": "ds-idrepo-0" 22:25:57 DEBUG "value": [ 22:25:57 DEBUG 22:25:57 DEBUG "pod": "idm-b85f9d978-j5pbm" 22:25:57 DEBUG 22:25:57 DEBUG "pod": "am-555f4db595-kzxsp" 22:25:57 DEBUG "pod": "am-555f4db595-kzxsp" 22:25:57 DEBUG "instance": "10.142.0.43:9100" 22:25:57 DEBUG 22:25:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:25:57 DEBUG 1713302757, 22:25:57 DEBUG }, 22:25:57 DEBUG "pod": "am-555f4db595-kzxsp" 22:25:57 DEBUG "pod": "ds-idrepo-0" 22:25:57 DEBUG 1713302757, 22:25:57 DEBUG 22:25:57 DEBUG "value": [ 22:25:57 DEBUG }, 22:25:57 DEBUG "pod": "ds-cts-2" 22:25:57 DEBUG }, 22:25:57 DEBUG }, 22:25:57 DEBUG "pod": "ds-idrepo-0" 22:25:57 DEBUG "value": [ 22:25:57 DEBUG }, 22:25:57 DEBUG }, 22:25:57 DEBUG }, 22:25:57 DEBUG 1713302757, 22:25:57 DEBUG 22:25:57 DEBUG }, 22:25:57 DEBUG "pod": "am-555f4db595-kzxsp" 22:25:57 DEBUG }, 22:25:57 DEBUG }, 22:25:57 DEBUG }, 22:25:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:25:57 DEBUG }, 22:25:57 DEBUG "0" 22:25:57 DEBUG "value": [ 22:25:57 DEBUG }, 22:25:57 DEBUG }, 22:25:57 DEBUG "2007.6" 22:25:57 DEBUG "pod": "ds-idrepo-0" 22:25:57 DEBUG 1713302757, 22:25:57 DEBUG "value": [ 22:25:57 DEBUG }, 22:25:57 DEBUG "value": [ 22:25:57 DEBUG "value": [ 22:25:57 DEBUG }, 22:25:57 DEBUG 1713302757, 22:25:57 DEBUG "value": [ 22:25:57 DEBUG "value": [ 22:25:57 DEBUG "value": [ 22:25:57 DEBUG "0" 22:25:57 DEBUG "pod": "am-555f4db595-kzxsp" 22:25:57 DEBUG "value": [ 22:25:57 DEBUG }, 22:25:57 DEBUG "value": [ 22:25:57 DEBUG "value": [ 22:25:57 DEBUG "value": [ 22:25:57 DEBUG }, 22:25:57 DEBUG "value": [ 22:25:57 DEBUG ] 22:25:57 DEBUG 1713302757, 22:25:57 DEBUG "value": [ 22:25:57 DEBUG "value": [ 22:25:57 DEBUG ] 22:25:57 DEBUG }, 22:25:57 DEBUG "45475514.5" 22:25:57 DEBUG 1713302757, 22:25:57 DEBUG "value": [ 22:25:57 DEBUG 1713302757, 22:25:57 DEBUG 1713302757, 22:25:57 DEBUG "value": [ 22:25:57 DEBUG "0.0005000000000004927" 22:25:57 DEBUG 1713302757, 22:25:57 DEBUG 1713302757, 22:25:57 DEBUG 1713302757, 22:25:57 DEBUG ] 22:25:57 DEBUG }, 22:25:57 DEBUG 1713302757, 22:25:57 DEBUG "value": [ 22:25:57 DEBUG 1713302757, 22:25:57 DEBUG 1713302757, 22:25:57 DEBUG 1713302757, 22:25:57 DEBUG "value": [ 22:25:57 DEBUG 1713302757, 22:25:57 DEBUG } 22:25:57 DEBUG "0" 22:25:57 DEBUG 1713302757, 22:25:57 DEBUG 1713302757, 22:25:57 DEBUG } 22:25:57 DEBUG "value": [ 22:25:57 DEBUG ] 22:25:57 DEBUG "0" 22:25:57 DEBUG 1713302757, 22:25:57 DEBUG "1150.7363311490376" 22:25:57 DEBUG "0" 22:25:57 DEBUG 1713302757, 22:25:57 DEBUG ] 22:25:57 DEBUG "19955182592" 22:25:57 DEBUG "0" 22:25:57 DEBUG "0" 22:25:57 DEBUG } 22:25:57 DEBUG "value": [ 22:25:57 DEBUG "0" 22:25:57 DEBUG 1713302757, 22:25:57 DEBUG "2007.5666666666666" 22:25:57 DEBUG "2007.6" 22:25:57 DEBUG "45465733.36666667" 22:25:57 DEBUG 1713302757, 22:25:57 DEBUG "649441280" 22:25:57 DEBUG ] 22:25:57 DEBUG ] 22:25:57 DEBUG "0.024" 22:25:57 DEBUG "0" 22:25:57 DEBUG ] 22:25:57 DEBUG 1713302757, 22:25:57 DEBUG } 22:25:57 DEBUG ] 22:25:57 DEBUG "0" 22:25:57 DEBUG ] 22:25:57 DEBUG ] 22:25:57 DEBUG "0" 22:25:57 DEBUG } 22:25:57 DEBUG ] 22:25:57 DEBUG ] 22:25:57 DEBUG ] 22:25:57 DEBUG ] 22:25:57 DEBUG 1713302757, 22:25:57 DEBUG ] 22:25:57 DEBUG "1700115.5" 22:25:57 DEBUG ] 22:25:57 DEBUG ] 22:25:57 DEBUG ] 22:25:57 DEBUG "6.295427832204636" 22:25:57 DEBUG ] 22:25:57 DEBUG } 22:25:57 DEBUG } 22:25:57 DEBUG ] 22:25:57 DEBUG ] 22:25:57 DEBUG } 22:25:57 DEBUG "0" 22:25:57 DEBUG ] 22:25:57 DEBUG } 22:25:57 DEBUG ] 22:25:57 DEBUG } 22:25:57 DEBUG } 22:25:57 DEBUG ] 22:25:57 DEBUG ] 22:25:57 DEBUG } 22:25:57 DEBUG } 22:25:57 DEBUG } 22:25:57 DEBUG } 22:25:57 DEBUG "0" 22:25:57 DEBUG } 22:25:57 DEBUG ] 22:25:57 DEBUG } 22:25:57 DEBUG } 22:25:57 DEBUG } 22:25:57 DEBUG ] 22:25:57 DEBUG } 22:25:57 DEBUG } 22:25:57 DEBUG ] 22:25:57 DEBUG } 22:25:57 DEBUG } 22:25:57 DEBUG } 22:25:57 DEBUG ] 22:25:57 DEBUG } 22:25:57 DEBUG ] 22:25:57 DEBUG } 22:25:57 DEBUG ] 22:25:57 DEBUG ] 22:25:57 DEBUG } 22:25:57 DEBUG } 22:25:57 DEBUG ] 22:25:57 DEBUG ] 22:25:57 DEBUG ] 22:25:57 DEBUG } 22:25:57 DEBUG ] 22:25:57 DEBUG ] 22:25:57 DEBUG } 22:25:57 DEBUG ] 22:25:57 DEBUG ] 22:25:57 DEBUG ] 22:25:57 DEBUG } 22:25:57 DEBUG ] 22:25:57 DEBUG } 22:25:57 DEBUG ] 22:25:57 DEBUG ] 22:25:57 DEBUG } 22:25:57 DEBUG } 22:25:57 DEBUG } 22:25:57 DEBUG ] 22:25:57 DEBUG } 22:25:57 DEBUG } 22:25:57 DEBUG ] 22:25:57 DEBUG } 22:25:57 DEBUG } 22:25:57 DEBUG } 22:25:57 DEBUG } 22:25:57 DEBUG } 22:25:57 DEBUG } 22:25:57 INFO [loop_until]: OK (rc = 0) 22:25:57 DEBUG ] 22:25:57 DEBUG } 22:25:57 DEBUG } 22:25:57 DEBUG } 22:25:57 DEBUG ] 22:25:57 DEBUG } 22:25:57 DEBUG } 22:25:57 DEBUG } 22:25:57 DEBUG } 22:25:57 DEBUG ] 22:25:57 DEBUG } 22:25:57 DEBUG } 22:25:57 DEBUG } 22:25:57 DEBUG } 22:25:57 DEBUG } 22:25:57 DEBUG } 22:25:57 DEBUG } 22:25:57 DEBUG } 22:25:57 DEBUG ] 22:25:57 DEBUG } 22:25:57 DEBUG --- stdout --- 22:25:57 DEBUG } 22:25:57 DEBUG } 22:25:57 DEBUG } 22:25:57 DEBUG } 22:25:57 DEBUG } 22:25:57 DEBUG } 22:25:57 DEBUG } 22:25:57 DEBUG } 22:25:57 DEBUG } 22:25:57 DEBUG } 22:25:57 DEBUG } 22:25:57 DEBUG } 22:25:57 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 62m 0% 5092Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7491m 25% 14747Mi 13% gke-xlou-cdm-default-pool-1b0769c8-nhlf 9337m 31% 10825Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 83m 0% 2649Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 4744m 15% 6469Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 8623m 28% 16597Mi 14% gke-xlou-cdm-default-pool-1b0769c8-xnmp 59m 0% 3146Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 9303m 31% 10367Mi 9% gke-xlou-cdm-ds-0f601751-hmm7 4664m 15% 5603Mi 5% gke-xlou-cdm-ds-0f601751-j488 7306m 24% 14077Mi 12% gke-xlou-cdm-ds-0f601751-mc5d 10665m 35% 3348Mi 2% gke-xlou-cdm-ds-0f601751-nj6q 9612m 32% 10420Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 4815m 16% 7358Mi 6% gke-xlou-cdm-frontend-a8771548-i82n 6478m 40% 5961Mi 10% 22:25:57 DEBUG } 22:25:57 DEBUG } 22:25:57 DEBUG } 22:25:57 DEBUG } 22:25:57 DEBUG --- stderr --- 22:25:57 DEBUG 22:26:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713302815" 22:26:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302815" 22:26:55 ERROR [http_cmd]: ERROR 22:26:55 INFO [http_cmd]: http status code OK 22:26:55 ERROR --- status code --- 22:26:55 DEBUG --- status code --- 22:26:55 ERROR http status code is 400 (expected [200, 401, 403, 404]) 22:26:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:26:55 ERROR --- http response --- 22:26:55 DEBUG --- http response --- 22:26:55 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 22:26:55 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 22:26:55 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 22:26:56 INFO 22:26:56 INFO [loop_until]: kubectl --namespace=xlou top pods 22:26:56 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:26:56 INFO [loop_until]: OK (rc = 0) 22:26:56 DEBUG --- stdout --- 22:26:56 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 9Mi am-555f4db595-4lbss 9330m 8591Mi am-555f4db595-fz8sn 9606m 8602Mi am-555f4db595-kzxsp 9296m 8555Mi ds-cts-0 8070m 12108Mi ds-cts-1 7277m 12193Mi ds-cts-2 7104m 12157Mi ds-idrepo-0 4617m 5032Mi ds-idrepo-1 4491m 2849Mi ds-idrepo-2 4452m 3126Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 3m 1207Mi idm-b85f9d978-rl76f 4m 1225Mi lodemon-bc756c647-cr7x6 15m 63Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-7c9b667974-9mr2s 6204m 528Mi 22:26:56 DEBUG --- stderr --- 22:26:56 DEBUG 22:26:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302817" 22:26:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302817" 22:26:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302817" 22:26:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302817" 22:26:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302817" 22:26:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302817" 22:26:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302817" 22:26:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302817" 22:26:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302817" 22:26:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302817" 22:26:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302817" 22:26:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302817" 22:26:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302817" 22:26:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302817" 22:26:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713302817" 22:26:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302817" 22:26:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713302817" 22:26:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302817" 22:26:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302817" 22:26:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713302817" 22:26:57 INFO 22:26:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302817" 22:26:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302817" 22:26:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302817" 22:26:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302817" 22:26:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713302817" 22:26:57 INFO [http_cmd]: http status code OK 22:26:57 INFO [http_cmd]: http status code OK 22:26:57 INFO [http_cmd]: http status code OK 22:26:57 INFO [http_cmd]: http status code OK 22:26:57 INFO [http_cmd]: http status code OK 22:26:57 INFO [http_cmd]: http status code OK 22:26:57 INFO [http_cmd]: http status code OK 22:26:57 INFO [http_cmd]: http status code OK 22:26:57 INFO [http_cmd]: http status code OK 22:26:57 INFO [http_cmd]: http status code OK 22:26:57 INFO [http_cmd]: http status code OK 22:26:57 INFO [http_cmd]: http status code OK 22:26:57 INFO [http_cmd]: http status code OK 22:26:57 INFO [loop_until]: kubectl --namespace=xlou top node 22:26:57 INFO [http_cmd]: http status code OK 22:26:57 INFO [http_cmd]: http status code OK 22:26:57 INFO [http_cmd]: http status code OK 22:26:57 INFO [http_cmd]: http status code OK 22:26:57 DEBUG --- status code --- 22:26:57 DEBUG --- status code --- 22:26:57 INFO [http_cmd]: http status code OK 22:26:57 INFO [http_cmd]: http status code OK 22:26:57 INFO [http_cmd]: http status code OK 22:26:57 DEBUG --- status code --- 22:26:57 DEBUG --- status code --- 22:26:57 DEBUG --- status code --- 22:26:57 DEBUG --- status code --- 22:26:57 DEBUG --- status code --- 22:26:57 DEBUG --- status code --- 22:26:57 DEBUG --- status code --- 22:26:57 DEBUG --- status code --- 22:26:57 DEBUG --- status code --- 22:26:57 DEBUG --- status code --- 22:26:57 DEBUG --- status code --- 22:26:57 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:26:57 DEBUG --- status code --- 22:26:57 INFO [http_cmd]: http status code OK 22:26:57 INFO [http_cmd]: http status code OK 22:26:57 INFO [http_cmd]: http status code OK 22:26:57 DEBUG --- status code --- 22:26:57 INFO [http_cmd]: http status code OK 22:26:57 DEBUG --- status code --- 22:26:57 DEBUG --- status code --- 22:26:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:26:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:26:57 DEBUG --- status code --- 22:26:57 DEBUG --- status code --- 22:26:57 DEBUG --- status code --- 22:26:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:26:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:26:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:26:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:26:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:26:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:26:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:26:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:26:57 DEBUG --- http response --- 22:26:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:26:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:26:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:26:57 DEBUG --- status code --- 22:26:57 DEBUG --- status code --- 22:26:57 INFO [http_cmd]: http status code OK 22:26:57 DEBUG --- status code --- 22:26:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:26:57 DEBUG --- status code --- 22:26:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:26:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:26:57 DEBUG --- http response --- 22:26:57 DEBUG --- http response --- 22:26:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:26:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:26:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:26:57 DEBUG --- http response --- 22:26:57 DEBUG --- http response --- 22:26:57 DEBUG --- http response --- 22:26:57 DEBUG --- http response --- 22:26:57 DEBUG --- http response --- 22:26:57 DEBUG --- http response --- 22:26:57 DEBUG --- http response --- 22:26:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:26:57 DEBUG --- http response --- 22:26:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:26:57 DEBUG --- http response --- 22:26:57 DEBUG --- http response --- 22:26:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:26:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:26:57 DEBUG --- status code --- 22:26:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:26:57 DEBUG --- http response --- 22:26:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:26:57 DEBUG --- http response --- 22:26:57 DEBUG --- http response --- 22:26:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:26:57 DEBUG --- http response --- 22:26:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:26:57 DEBUG --- http response --- 22:26:57 DEBUG --- http response --- 22:26:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:26:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:26:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:26:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:26:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:26:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:26:57 DEBUG --- http response --- 22:26:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:26:57 DEBUG ----- output ----- 22:26:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:26:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:26:57 DEBUG --- http response --- 22:26:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:26:57 DEBUG --- http response --- 22:26:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:26:57 DEBUG --- http response --- 22:26:57 DEBUG --- http response --- 22:26:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:26:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:26:57 DEBUG ----- output ----- 22:26:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:26:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:26:57 DEBUG ----- output ----- 22:26:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:26:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:26:57 DEBUG ----- output ----- 22:26:57 DEBUG ----- output ----- 22:26:57 DEBUG ----- output ----- 22:26:57 DEBUG { 22:26:57 DEBUG "status": "success", 22:26:57 DEBUG ----- output ----- 22:26:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:26:57 DEBUG ----- output ----- 22:26:57 DEBUG { 22:26:57 DEBUG ----- output ----- 22:26:57 DEBUG ----- output ----- 22:26:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:26:57 DEBUG ----- output ----- 22:26:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:26:57 DEBUG --- http response --- 22:26:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:26:57 DEBUG ----- output ----- 22:26:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:26:57 DEBUG ----- output ----- 22:26:57 DEBUG { 22:26:57 DEBUG ----- output ----- 22:26:57 DEBUG ----- output ----- 22:26:57 DEBUG { 22:26:57 DEBUG "status": "success", 22:26:57 DEBUG ----- output ----- 22:26:57 DEBUG { 22:26:57 DEBUG { 22:26:57 DEBUG "data": { 22:26:57 DEBUG ----- output ----- 22:26:57 DEBUG ----- output ----- 22:26:57 DEBUG { 22:26:57 DEBUG ----- output ----- 22:26:57 DEBUG { 22:26:57 DEBUG "status": "success", 22:26:57 DEBUG { 22:26:57 DEBUG { 22:26:57 DEBUG ----- output ----- 22:26:57 DEBUG { 22:26:57 DEBUG ----- output ----- 22:26:57 DEBUG ----- output ----- 22:26:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:26:57 DEBUG { 22:26:57 DEBUG ----- output ----- 22:26:57 DEBUG { 22:26:57 DEBUG "status": "success", 22:26:57 DEBUG { 22:26:57 DEBUG { 22:26:57 DEBUG ----- output ----- 22:26:57 DEBUG "data": { 22:26:57 DEBUG { 22:26:57 DEBUG "status": "success", 22:26:57 DEBUG "status": "success", 22:26:57 DEBUG "resultType": "vector", 22:26:57 DEBUG { 22:26:57 DEBUG { 22:26:57 DEBUG "status": "success", 22:26:57 DEBUG { 22:26:57 DEBUG "status": "success", 22:26:57 DEBUG "data": { 22:26:57 DEBUG "status": "success", 22:26:57 DEBUG "status": "success", 22:26:57 DEBUG { 22:26:57 DEBUG "status": "success", 22:26:57 DEBUG { 22:26:57 DEBUG { 22:26:57 DEBUG ----- output ----- 22:26:57 DEBUG "status": "success", 22:26:57 DEBUG { 22:26:57 DEBUG "status": "success", 22:26:57 DEBUG "data": { 22:26:57 DEBUG "status": "success", 22:26:57 DEBUG "status": "success", 22:26:57 DEBUG { 22:26:57 DEBUG "resultType": "vector", 22:26:57 DEBUG "status": "success", 22:26:57 DEBUG "data": { 22:26:57 DEBUG "data": { 22:26:57 DEBUG "result": [ 22:26:57 DEBUG "status": "success", 22:26:57 DEBUG "status": "success", 22:26:57 DEBUG "data": { 22:26:57 DEBUG "status": "success", 22:26:57 DEBUG "data": { 22:26:57 DEBUG "resultType": "vector", 22:26:57 DEBUG "data": { 22:26:57 DEBUG "data": { 22:26:57 DEBUG "status": "success", 22:26:57 DEBUG "data": { 22:26:57 DEBUG "status": "success", 22:26:57 DEBUG "status": "success", 22:26:57 DEBUG { 22:26:57 DEBUG "data": { 22:26:57 DEBUG "status": "success", 22:26:57 DEBUG "data": { 22:26:57 DEBUG "resultType": "vector", 22:26:57 DEBUG "data": { 22:26:57 DEBUG "data": { 22:26:57 DEBUG "status": "success", 22:26:57 DEBUG "result": [ 22:26:57 DEBUG "data": { 22:26:57 DEBUG "resultType": "vector", 22:26:57 DEBUG "resultType": "vector", 22:26:57 DEBUG { 22:26:57 DEBUG "data": { 22:26:57 DEBUG "data": { 22:26:57 DEBUG "resultType": "vector", 22:26:57 DEBUG "data": { 22:26:57 DEBUG "resultType": "vector", 22:26:57 DEBUG "result": [ 22:26:57 DEBUG "resultType": "vector", 22:26:57 DEBUG "resultType": "vector", 22:26:57 DEBUG "data": { 22:26:57 DEBUG "resultType": "vector", 22:26:57 DEBUG "data": { 22:26:57 DEBUG "data": { 22:26:57 DEBUG "status": "success", 22:26:57 DEBUG "resultType": "vector", 22:26:57 DEBUG "data": { 22:26:57 DEBUG "resultType": "vector", 22:26:57 DEBUG "result": [ 22:26:57 DEBUG "resultType": "vector", 22:26:57 DEBUG "resultType": "vector", 22:26:57 DEBUG "data": { 22:26:57 DEBUG { 22:26:57 DEBUG "resultType": "vector", 22:26:57 DEBUG "result": [ 22:26:57 DEBUG "result": [ 22:26:57 DEBUG "metric": { 22:26:57 DEBUG "resultType": "vector", 22:26:57 DEBUG "result": [ 22:26:57 DEBUG "result": [ 22:26:57 DEBUG "resultType": "vector", 22:26:57 DEBUG "result": [ 22:26:57 DEBUG { 22:26:57 DEBUG "result": [ 22:26:57 DEBUG "result": [ 22:26:57 DEBUG "resultType": "vector", 22:26:57 DEBUG "result": [ 22:26:57 DEBUG "resultType": "vector", 22:26:57 DEBUG "resultType": "vector", 22:26:57 DEBUG "data": { 22:26:57 DEBUG "result": [ 22:26:57 DEBUG "resultType": "vector", 22:26:57 DEBUG "result": [ 22:26:57 DEBUG { 22:26:57 DEBUG "result": [ 22:26:57 DEBUG "result": [ 22:26:57 DEBUG "resultType": "vector", 22:26:57 DEBUG "metric": { 22:26:57 DEBUG "result": [ 22:26:57 DEBUG { 22:26:57 DEBUG { 22:26:57 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:26:57 DEBUG "resultType": "vector", 22:26:57 DEBUG { 22:26:57 DEBUG { 22:26:57 DEBUG "result": [ 22:26:57 DEBUG { 22:26:57 DEBUG "metric": { 22:26:57 DEBUG { 22:26:57 DEBUG { 22:26:57 DEBUG "result": [ 22:26:57 DEBUG { 22:26:57 DEBUG "result": [ 22:26:57 DEBUG "result": [ 22:26:57 DEBUG "resultType": "vector", 22:26:57 DEBUG { 22:26:57 DEBUG "result": [ 22:26:57 DEBUG { 22:26:57 DEBUG "metric": { 22:26:57 DEBUG { 22:26:57 DEBUG { 22:26:57 DEBUG "result": [ 22:26:57 DEBUG "pod": "am-555f4db595-fz8sn" 22:26:57 DEBUG { 22:26:57 DEBUG "metric": { 22:26:57 DEBUG "metric": { 22:26:57 DEBUG }, 22:26:57 DEBUG "result": [ 22:26:57 DEBUG "metric": { 22:26:57 DEBUG "metric": { 22:26:57 DEBUG { 22:26:57 DEBUG "metric": { 22:26:57 DEBUG "pod": "ds-idrepo-2" 22:26:57 DEBUG "metric": { 22:26:57 DEBUG "metric": { 22:26:57 DEBUG { 22:26:57 DEBUG "metric": { 22:26:57 DEBUG { 22:26:57 DEBUG { 22:26:57 DEBUG "result": [ 22:26:57 DEBUG "metric": { 22:26:57 DEBUG { 22:26:57 DEBUG "metric": { 22:26:57 DEBUG "pod": "am-555f4db595-fz8sn" 22:26:57 DEBUG "metric": { 22:26:57 DEBUG "metric": { 22:26:57 DEBUG { 22:26:57 DEBUG }, 22:26:57 DEBUG "metric": { 22:26:57 DEBUG "instance": "10.142.0.107:9100" 22:26:57 DEBUG "instance": "10.142.0.107:9100" 22:26:57 DEBUG "value": [ 22:26:57 DEBUG { 22:26:57 DEBUG "instance": "10.142.0.107:9100" 22:26:57 DEBUG "instance": "10.142.0.107:9100" 22:26:57 DEBUG "metric": { 22:26:57 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:26:57 DEBUG }, 22:26:57 DEBUG "pod": "am-555f4db595-fz8sn" 22:26:57 DEBUG "instance": "10.142.0.107:9100" 22:26:57 DEBUG "metric": { 22:26:57 DEBUG "pod": "ds-idrepo-2" 22:26:57 DEBUG "metric": { 22:26:57 DEBUG "metric": { 22:26:57 DEBUG { 22:26:57 DEBUG "pod": "ds-idrepo-2" 22:26:57 DEBUG "metric": { 22:26:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:26:57 DEBUG }, 22:26:57 DEBUG "pod": "am-555f4db595-fz8sn" 22:26:57 DEBUG "pod": "am-555f4db595-fz8sn" 22:26:57 DEBUG "metric": { 22:26:57 DEBUG "value": [ 22:26:57 DEBUG "pod": "am-555f4db595-fz8sn" 22:26:57 DEBUG }, 22:26:57 DEBUG "value": [ 22:26:57 DEBUG 22:26:57 DEBUG "metric": { 22:26:57 DEBUG }, 22:26:57 DEBUG }, 22:26:57 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:26:57 DEBUG }, 22:26:57 DEBUG "value": [ 22:26:57 DEBUG }, 22:26:57 DEBUG }, 22:26:57 DEBUG "pod": "ds-idrepo-2" 22:26:57 DEBUG }, 22:26:57 DEBUG "pod": "ds-idrepo-2" 22:26:57 DEBUG "pod": "am-555f4db595-fz8sn" 22:26:57 DEBUG "metric": { 22:26:57 DEBUG }, 22:26:57 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:26:57 DEBUG }, 22:26:57 DEBUG "value": [ 22:26:57 DEBUG }, 22:26:57 DEBUG }, 22:26:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:26:57 DEBUG 22:26:57 DEBUG }, 22:26:57 DEBUG 22:26:57 DEBUG }, 22:26:57 DEBUG 22:26:57 DEBUG "pod": "ds-idrepo-2" 22:26:57 DEBUG "value": [ 22:26:57 DEBUG "value": [ 22:26:57 DEBUG }, 22:26:57 DEBUG "value": [ 22:26:57 DEBUG 22:26:57 DEBUG "value": [ 22:26:57 DEBUG "value": [ 22:26:57 DEBUG }, 22:26:57 DEBUG "value": [ 22:26:57 DEBUG }, 22:26:57 DEBUG }, 22:26:57 DEBUG "pod": "ds-idrepo-2" 22:26:57 DEBUG "value": [ 22:26:57 DEBUG }, 22:26:57 DEBUG "value": [ 22:26:57 DEBUG 22:26:57 DEBUG "value": [ 22:26:57 DEBUG "value": [ 22:26:57 DEBUG }, 22:26:57 DEBUG 22:26:57 DEBUG "value": [ 22:26:57 DEBUG 22:26:57 DEBUG "value": [ 22:26:57 DEBUG 22:26:57 DEBUG }, 22:26:57 DEBUG "value": [ 22:26:57 DEBUG 22:26:57 DEBUG "value": [ 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG "value": [ 22:26:57 DEBUG 22:26:57 DEBUG "value": [ 22:26:57 DEBUG "value": [ 22:26:57 DEBUG }, 22:26:57 DEBUG 22:26:57 DEBUG "value": [ 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG "value": [ 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG [...] 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG "value": [ 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG [...] 22:26:57 DEBUG 22:26:57 DEBUG [...] 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG [...] 22:26:57 DEBUG 22:26:57 DEBUG [...] 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG [...] 22:26:57 DEBUG [...] 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG [...] 22:26:57 DEBUG 22:26:57 DEBUG [...] 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG [...] 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG [...] 22:26:57 DEBUG 22:26:57 DEBUG [...] 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG [...] 22:26:57 DEBUG 22:26:57 DEBUG [...] 22:26:57 DEBUG 22:26:57 DEBUG [...] 22:26:57 DEBUG [...] 22:26:57 DEBUG 22:26:57 DEBUG [...] 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG [...] 22:26:57 DEBUG [...] 22:26:57 DEBUG 22:26:57 DEBUG [...] 22:26:57 DEBUG [...] 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG [...] 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG [...] 22:26:57 DEBUG [...] 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG "pod": "overseer-0-7c9b667974-9mr2s" 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG [...] 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG "pod": "am-555f4db595-kzxsp" 22:26:57 DEBUG 22:26:57 DEBUG "instance": "10.142.0.43:9100" 22:26:57 DEBUG 22:26:57 DEBUG }, 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG "pod": "ds-idrepo-0" 22:26:57 DEBUG "pod": "am-555f4db595-kzxsp" 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG "pod": "am-555f4db595-kzxsp" 22:26:57 DEBUG "pod": "am-555f4db595-kzxsp" 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG }, 22:26:57 DEBUG 22:26:57 DEBUG }, 22:26:57 DEBUG 22:26:57 DEBUG "value": [ 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG "instance": "10.142.0.43:9100" 22:26:57 DEBUG 22:26:57 DEBUG "pod": "overseer-0-7c9b667974-9mr2s" 22:26:57 DEBUG }, 22:26:57 DEBUG }, 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG "pod": "ds-idrepo-0" 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG "pod": "ds-idrepo-0" 22:26:57 DEBUG 22:26:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:26:57 DEBUG }, 22:26:57 DEBUG }, 22:26:57 DEBUG 22:26:57 DEBUG 22:26:57 DEBUG "value": [ 22:26:57 DEBUG "pod": "am-555f4db595-kzxsp" 22:26:57 DEBUG "value": [ 22:26:57 DEBUG "instance": "10.142.0.43:9100" 22:26:57 DEBUG 1713302817, 22:26:57 DEBUG "instance": "10.142.0.43:9100" 22:26:57 DEBUG "pod": "ds-idrepo-0" 22:26:57 DEBUG }, 22:26:57 DEBUG "pod": "overseer-0-7c9b667974-9mr2s" 22:26:57 DEBUG }, 22:26:57 DEBUG "value": [ 22:26:57 DEBUG "value": [ 22:26:57 DEBUG "instance": "10.142.0.43:9100" 22:26:57 DEBUG "pod": "ds-idrepo-0" 22:26:57 DEBUG }, 22:26:57 DEBUG "pod": "ds-idrepo-0" 22:26:57 DEBUG "pod": "am-555f4db595-kzxsp" 22:26:57 DEBUG 22:26:57 DEBUG }, 22:26:57 DEBUG "pod": "overseer-0-7c9b667974-9mr2s" 22:26:57 DEBUG }, 22:26:57 DEBUG "value": [ 22:26:57 DEBUG "value": [ 22:26:57 DEBUG "pod": "am-555f4db595-kzxsp" 22:26:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:26:57 DEBUG 1713302817, 22:26:57 DEBUG }, 22:26:57 DEBUG 1713302817, 22:26:57 DEBUG }, 22:26:57 DEBUG "0" 22:26:57 DEBUG }, 22:26:57 DEBUG }, 22:26:57 DEBUG "value": [ 22:26:57 DEBUG }, 22:26:57 DEBUG "value": [ 22:26:57 DEBUG 1713302817, 22:26:57 DEBUG 1713302817, 22:26:57 DEBUG }, 22:26:57 DEBUG }, 22:26:57 DEBUG "value": [ 22:26:57 DEBUG }, 22:26:57 DEBUG }, 22:26:57 DEBUG "pod": "ds-cts-2" 22:26:57 DEBUG "value": [ 22:26:57 DEBUG }, 22:26:57 DEBUG "value": [ 22:26:57 DEBUG 1713302817, 22:26:57 DEBUG "0" 22:26:57 DEBUG }, 22:26:57 DEBUG }, 22:26:57 DEBUG "2003.0333333333333" 22:26:57 DEBUG "value": [ 22:26:57 DEBUG "45542055.16666666" 22:26:57 DEBUG "value": [ 22:26:57 DEBUG ] 22:26:57 DEBUG "value": [ 22:26:57 DEBUG "value": [ 22:26:57 DEBUG 1713302817, 22:26:57 DEBUG "value": [ 22:26:57 DEBUG 1713302817, 22:26:57 DEBUG "0" 22:26:57 DEBUG "0" 22:26:57 DEBUG "value": [ 22:26:57 DEBUG "value": [ 22:26:57 DEBUG 1713302817, 22:26:57 DEBUG "value": [ 22:26:57 DEBUG "value": [ 22:26:57 DEBUG }, 22:26:57 DEBUG 1713302817, 22:26:57 DEBUG "0" 22:26:57 DEBUG 1713302817, 22:26:57 DEBUG ] 22:26:57 DEBUG "value": [ 22:26:57 DEBUG "value": [ 22:26:57 DEBUG ] 22:26:57 DEBUG 1713302817, 22:26:57 DEBUG 1713302817, 22:26:57 DEBUG ] 22:26:57 DEBUG 1713302817, 22:26:57 DEBUG } 22:26:57 DEBUG 1713302817, 22:26:57 DEBUG 1713302817, 22:26:57 DEBUG "0.00043333333333350765" 22:26:57 DEBUG 1713302817, 22:26:57 DEBUG "22711.686597938147" 22:26:57 DEBUG ] 22:26:57 DEBUG ] 22:26:57 DEBUG 1713302817, 22:26:57 DEBUG 1713302817, 22:26:57 DEBUG "0" 22:26:57 DEBUG 1713302817, 22:26:57 DEBUG 1713302817, 22:26:57 DEBUG "value": [ 22:26:57 DEBUG ] 22:26:57 DEBUG "value": [ 22:26:57 DEBUG "648904704" 22:26:57 DEBUG } 22:26:57 DEBUG 1713302817, 22:26:57 DEBUG 1713302817, 22:26:57 DEBUG } 22:26:57 DEBUG "2002.9666666666667" 22:26:57 DEBUG "0.024" 22:26:57 DEBUG } 22:26:57 DEBUG "0" 22:26:57 DEBUG ] 22:26:57 DEBUG "19955856384" 22:26:57 DEBUG "0" 22:26:57 DEBUG ] 22:26:57 DEBUG "0" 22:26:57 DEBUG ] 22:26:57 DEBUG } 22:26:57 DEBUG } 22:26:57 DEBUG "45532959.3" 22:26:57 DEBUG "0" 22:26:57 DEBUG ] 22:26:57 DEBUG "0" 22:26:57 DEBUG "2002.8999999999999" 22:26:57 DEBUG ] 22:26:57 DEBUG } 22:26:57 DEBUG 1713302817, 22:26:57 DEBUG ] 22:26:57 DEBUG ] 22:26:57 DEBUG "1820159.5" 22:26:57 DEBUG "6.293925481644772" 22:26:57 DEBUG ] 22:26:57 DEBUG ] 22:26:57 DEBUG ] 22:26:57 DEBUG ] 22:26:57 DEBUG ] 22:26:57 DEBUG } 22:26:57 DEBUG ] 22:26:57 DEBUG ] 22:26:57 DEBUG } 22:26:57 DEBUG ] 22:26:57 DEBUG } 22:26:57 DEBUG ] 22:26:57 DEBUG ] 22:26:57 DEBUG ] 22:26:57 DEBUG ] 22:26:57 DEBUG } 22:26:57 DEBUG ] 22:26:57 DEBUG 1713302817, 22:26:57 DEBUG } 22:26:57 DEBUG ] 22:26:57 DEBUG "3.2772579178399353" 22:26:57 DEBUG } 22:26:57 DEBUG } 22:26:57 DEBUG ] 22:26:57 DEBUG ] 22:26:57 DEBUG } 22:26:57 DEBUG } 22:26:57 DEBUG } 22:26:57 DEBUG } 22:26:57 DEBUG } 22:26:57 DEBUG } 22:26:57 DEBUG } 22:26:57 DEBUG } 22:26:57 DEBUG ] 22:26:57 DEBUG } 22:26:57 DEBUG ] 22:26:57 DEBUG } 22:26:57 DEBUG } 22:26:57 DEBUG } 22:26:57 DEBUG } 22:26:57 DEBUG ] 22:26:57 DEBUG } 22:26:57 DEBUG "0" 22:26:57 DEBUG ] 22:26:57 DEBUG } 22:26:57 DEBUG ] 22:26:57 DEBUG ] 22:26:57 DEBUG } 22:26:57 DEBUG } 22:26:57 DEBUG } 22:26:57 DEBUG } 22:26:57 DEBUG ] 22:26:57 DEBUG ] 22:26:57 DEBUG } 22:26:57 DEBUG ] 22:26:57 DEBUG ] 22:26:57 DEBUG ] 22:26:57 DEBUG } 22:26:57 DEBUG ] 22:26:57 DEBUG } 22:26:57 DEBUG } 22:26:57 DEBUG } 22:26:57 DEBUG ] 22:26:57 DEBUG ] 22:26:57 DEBUG } 22:26:57 DEBUG ] 22:26:57 DEBUG ] 22:26:57 DEBUG } 22:26:57 DEBUG } 22:26:57 DEBUG } 22:26:57 DEBUG } 22:26:57 DEBUG ] 22:26:57 DEBUG ] 22:26:57 DEBUG } 22:26:57 DEBUG } 22:26:57 DEBUG } 22:26:57 DEBUG } 22:26:57 DEBUG } 22:26:57 DEBUG } 22:26:57 DEBUG } 22:26:57 DEBUG } 22:26:57 DEBUG } 22:26:57 DEBUG } 22:26:57 DEBUG } 22:26:57 DEBUG } 22:26:57 DEBUG } 22:26:57 DEBUG } 22:26:57 DEBUG ] 22:26:57 DEBUG } 22:26:57 DEBUG } 22:26:57 INFO [loop_until]: OK (rc = 0) 22:26:57 DEBUG } 22:26:57 DEBUG } 22:26:57 DEBUG } 22:26:57 DEBUG } 22:26:57 DEBUG } 22:26:57 DEBUG } 22:26:57 DEBUG } 22:26:57 DEBUG } 22:26:57 DEBUG } 22:26:57 DEBUG } 22:26:57 DEBUG ] 22:26:57 DEBUG } 22:26:57 DEBUG } 22:26:57 DEBUG --- stdout --- 22:26:57 DEBUG } 22:26:57 DEBUG } 22:26:57 DEBUG } 22:26:57 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 5087Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7439m 24% 14756Mi 13% gke-xlou-cdm-default-pool-1b0769c8-nhlf 9310m 31% 10812Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 82m 0% 2647Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 4574m 15% 6487Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 8299m 27% 16610Mi 14% gke-xlou-cdm-default-pool-1b0769c8-xnmp 66m 0% 3148Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 9251m 30% 10359Mi 9% gke-xlou-cdm-ds-0f601751-hmm7 4686m 15% 5591Mi 5% gke-xlou-cdm-ds-0f601751-j488 7364m 24% 14088Mi 12% gke-xlou-cdm-ds-0f601751-mc5d 10673m 35% 3348Mi 2% gke-xlou-cdm-ds-0f601751-nj6q 9622m 32% 10423Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 4717m 15% 7370Mi 6% gke-xlou-cdm-frontend-a8771548-i82n 6512m 40% 5959Mi 10% 22:26:57 DEBUG } 22:26:57 DEBUG --- stderr --- 22:26:57 DEBUG 22:27:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713302875" 22:27:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302875" 22:27:55 ERROR [http_cmd]: ERROR 22:27:55 ERROR --- status code --- 22:27:55 INFO [http_cmd]: http status code OK 22:27:55 ERROR http status code is 400 (expected [200, 401, 403, 404]) 22:27:55 DEBUG --- status code --- 22:27:55 ERROR --- http response --- 22:27:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:27:55 DEBUG --- http response --- 22:27:55 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 22:27:55 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 22:27:55 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 22:27:56 INFO 22:27:56 INFO [loop_until]: kubectl --namespace=xlou top pods 22:27:56 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:27:56 INFO [loop_until]: OK (rc = 0) 22:27:56 DEBUG --- stdout --- 22:27:56 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 9Mi am-555f4db595-4lbss 9362m 8591Mi am-555f4db595-fz8sn 9379m 8603Mi am-555f4db595-kzxsp 9365m 8557Mi ds-cts-0 7804m 12114Mi ds-cts-1 7469m 12199Mi ds-cts-2 7448m 12166Mi ds-idrepo-0 4819m 5031Mi ds-idrepo-1 4436m 2850Mi ds-idrepo-2 4408m 3141Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 3m 1207Mi idm-b85f9d978-rl76f 4m 1225Mi lodemon-bc756c647-cr7x6 11m 63Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-7c9b667974-9mr2s 6351m 530Mi 22:27:56 DEBUG --- stderr --- 22:27:56 DEBUG 22:27:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302877" 22:27:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302877" 22:27:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302877" 22:27:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302877" 22:27:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302877" 22:27:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302877" 22:27:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302877" 22:27:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302877" 22:27:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302877" 22:27:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302877" 22:27:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713302877" 22:27:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302877" 22:27:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302877" 22:27:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302877" 22:27:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302877" 22:27:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302877" 22:27:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302877" 22:27:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713302877" 22:27:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302877" 22:27:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302877" 22:27:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713302877" 22:27:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302877" 22:27:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302877" 22:27:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713302877" 22:27:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302877" 22:27:57 INFO 22:27:57 INFO [http_cmd]: http status code OK 22:27:57 INFO [http_cmd]: http status code OK 22:27:57 INFO [http_cmd]: http status code OK 22:27:57 INFO [http_cmd]: http status code OK 22:27:57 INFO [http_cmd]: http status code OK 22:27:57 INFO [http_cmd]: http status code OK 22:27:57 INFO [http_cmd]: http status code OK 22:27:57 INFO [http_cmd]: http status code OK 22:27:57 INFO [http_cmd]: http status code OK 22:27:57 INFO [http_cmd]: http status code OK 22:27:57 INFO [http_cmd]: http status code OK 22:27:57 INFO [http_cmd]: http status code OK 22:27:57 INFO [http_cmd]: http status code OK 22:27:57 INFO [http_cmd]: http status code OK 22:27:57 INFO [http_cmd]: http status code OK 22:27:57 INFO [http_cmd]: http status code OK 22:27:57 INFO [http_cmd]: http status code OK 22:27:57 INFO [loop_until]: kubectl --namespace=xlou top node 22:27:57 INFO [http_cmd]: http status code OK 22:27:57 DEBUG --- status code --- 22:27:57 INFO [http_cmd]: http status code OK 22:27:57 DEBUG --- status code --- 22:27:57 DEBUG --- status code --- 22:27:57 DEBUG --- status code --- 22:27:57 INFO [http_cmd]: http status code OK 22:27:57 INFO [http_cmd]: http status code OK 22:27:57 INFO [http_cmd]: http status code OK 22:27:57 DEBUG --- status code --- 22:27:57 DEBUG --- status code --- 22:27:57 DEBUG --- status code --- 22:27:57 DEBUG --- status code --- 22:27:57 DEBUG --- status code --- 22:27:57 DEBUG --- status code --- 22:27:57 DEBUG --- status code --- 22:27:57 INFO [http_cmd]: http status code OK 22:27:57 INFO [http_cmd]: http status code OK 22:27:57 DEBUG --- status code --- 22:27:57 INFO [http_cmd]: http status code OK 22:27:57 DEBUG --- status code --- 22:27:57 DEBUG --- status code --- 22:27:57 DEBUG --- status code --- 22:27:57 DEBUG --- status code --- 22:27:57 DEBUG --- status code --- 22:27:57 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:27:57 DEBUG --- status code --- 22:27:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:27:57 DEBUG --- status code --- 22:27:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:27:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:27:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:27:57 DEBUG --- status code --- 22:27:57 DEBUG --- status code --- 22:27:57 DEBUG --- status code --- 22:27:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:27:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:27:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:27:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:27:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:27:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:27:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:27:57 DEBUG --- status code --- 22:27:57 DEBUG --- status code --- 22:27:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:27:57 DEBUG --- status code --- 22:27:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:27:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:27:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:27:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:27:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:27:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:27:57 DEBUG --- http response --- 22:27:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:27:57 DEBUG --- http response --- 22:27:57 DEBUG --- http response --- 22:27:57 DEBUG --- http response --- 22:27:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:27:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:27:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:27:57 DEBUG --- http response --- 22:27:57 DEBUG --- http response --- 22:27:57 DEBUG --- http response --- 22:27:57 DEBUG --- http response --- 22:27:57 DEBUG --- http response --- 22:27:57 DEBUG --- http response --- 22:27:57 DEBUG --- http response --- 22:27:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:27:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:27:57 DEBUG --- http response --- 22:27:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:27:57 DEBUG --- http response --- 22:27:57 DEBUG --- http response --- 22:27:57 DEBUG --- http response --- 22:27:57 DEBUG --- http response --- 22:27:57 DEBUG --- http response --- 22:27:57 DEBUG --- http response --- 22:27:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:27:57 DEBUG --- http response --- 22:27:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:27:57 DEBUG ----- output ----- 22:27:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:27:57 DEBUG --- http response --- 22:27:57 DEBUG --- http response --- 22:27:57 DEBUG --- http response --- 22:27:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:27:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:27:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:27:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:27:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:27:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:27:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 22:27:57 DEBUG --- http response --- 22:27:57 DEBUG --- http response --- 22:27:57 DEBUG --- http response --- 22:27:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:27:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:27:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 22:27:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:27:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:27:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:27:57 DEBUG ----- output ----- 22:27:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:27:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 22:27:57 DEBUG { 22:27:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:27:57 DEBUG ----- output ----- 22:27:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:27:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 22:27:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:27:57 DEBUG ----- output ----- 22:27:57 DEBUG ----- output ----- 22:27:57 DEBUG ----- output ----- 22:27:57 DEBUG ----- output ----- 22:27:57 DEBUG ----- output ----- 22:27:57 DEBUG ----- output ----- 22:27:57 DEBUG ----- output ----- 22:27:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:27:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:27:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:27:57 DEBUG ----- output ----- 22:27:57 DEBUG ----- output ----- 22:27:57 DEBUG ----- output ----- 22:27:57 DEBUG ----- output ----- 22:27:57 DEBUG ----- output ----- 22:27:57 DEBUG ----- output ----- 22:27:57 DEBUG { 22:27:57 DEBUG ----- output ----- 22:27:57 DEBUG ----- output ----- 22:27:57 DEBUG "status": "success", 22:27:57 DEBUG ----- output ----- 22:27:57 DEBUG { 22:27:57 DEBUG ----- output ----- 22:27:57 DEBUG ----- output ----- 22:27:57 DEBUG ----- output ----- 22:27:57 DEBUG { 22:27:57 DEBUG { 22:27:57 DEBUG { 22:27:57 DEBUG { 22:27:57 DEBUG { 22:27:57 DEBUG { 22:27:57 DEBUG { 22:27:57 DEBUG ----- output ----- 22:27:57 DEBUG ----- output ----- 22:27:57 DEBUG ----- output ----- 22:27:57 DEBUG { 22:27:57 DEBUG { 22:27:57 DEBUG { 22:27:57 DEBUG { 22:27:57 DEBUG { 22:27:57 DEBUG { 22:27:57 DEBUG "status": "success", 22:27:57 DEBUG { 22:27:57 DEBUG { 22:27:57 DEBUG "data": { 22:27:57 DEBUG { 22:27:57 DEBUG "status": "success", 22:27:57 DEBUG { 22:27:57 DEBUG { 22:27:57 DEBUG { 22:27:57 DEBUG "status": "success", 22:27:57 DEBUG "status": "success", 22:27:57 DEBUG "status": "success", 22:27:57 DEBUG "status": "success", 22:27:57 DEBUG "status": "success", 22:27:57 DEBUG "status": "success", 22:27:57 DEBUG "status": "success", 22:27:57 DEBUG { 22:27:57 DEBUG { 22:27:57 DEBUG { 22:27:57 DEBUG "status": "success", 22:27:57 DEBUG "status": "success", 22:27:57 DEBUG "status": "success", 22:27:57 DEBUG "status": "success", 22:27:57 DEBUG "status": "success", 22:27:57 DEBUG "status": "success", 22:27:57 DEBUG "data": { 22:27:57 DEBUG "status": "success", 22:27:57 DEBUG "status": "success", 22:27:57 DEBUG "resultType": "vector", 22:27:57 DEBUG "status": "success", 22:27:57 DEBUG "data": { 22:27:57 DEBUG "status": "success", 22:27:57 DEBUG "status": "success", 22:27:57 DEBUG "status": "success", 22:27:57 DEBUG "data": { 22:27:57 DEBUG "data": { 22:27:57 DEBUG "data": { 22:27:57 DEBUG "data": { 22:27:57 DEBUG "data": { 22:27:57 DEBUG "data": { 22:27:57 DEBUG "data": { 22:27:57 DEBUG "status": "success", 22:27:57 DEBUG "status": "success", 22:27:57 DEBUG "status": "success", 22:27:57 DEBUG "data": { 22:27:57 DEBUG "data": { 22:27:57 DEBUG "data": { 22:27:57 DEBUG "data": { 22:27:57 DEBUG "data": { 22:27:57 DEBUG "data": { 22:27:57 DEBUG "resultType": "vector", 22:27:57 DEBUG "data": { 22:27:57 DEBUG "data": { 22:27:57 DEBUG "result": [ 22:27:57 DEBUG "data": { 22:27:57 DEBUG "resultType": "vector", 22:27:57 DEBUG "data": { 22:27:57 DEBUG "data": { 22:27:57 DEBUG "data": { 22:27:57 DEBUG "resultType": "vector", 22:27:57 DEBUG "resultType": "vector", 22:27:57 DEBUG "resultType": "vector", 22:27:57 DEBUG "resultType": "vector", 22:27:57 DEBUG "resultType": "vector", 22:27:57 DEBUG "resultType": "vector", 22:27:57 DEBUG "resultType": "vector", 22:27:57 DEBUG "data": { 22:27:57 DEBUG "data": { 22:27:57 DEBUG "data": { 22:27:57 DEBUG "resultType": "vector", 22:27:57 DEBUG "resultType": "vector", 22:27:57 DEBUG "resultType": "vector", 22:27:57 DEBUG "resultType": "vector", 22:27:57 DEBUG "resultType": "vector", 22:27:57 DEBUG "resultType": "vector", 22:27:57 DEBUG "result": [ 22:27:57 DEBUG "resultType": "vector", 22:27:57 DEBUG "resultType": "vector", 22:27:57 DEBUG { 22:27:57 DEBUG "resultType": "vector", 22:27:57 DEBUG "result": [ 22:27:57 DEBUG "resultType": "vector", 22:27:57 DEBUG "resultType": "vector", 22:27:57 DEBUG "resultType": "vector", 22:27:57 DEBUG "result": [ 22:27:57 DEBUG "result": [ 22:27:57 DEBUG "result": [ 22:27:57 DEBUG "result": [ 22:27:57 DEBUG "result": [ 22:27:57 DEBUG "result": [ 22:27:57 DEBUG "result": [ 22:27:57 DEBUG "resultType": "vector", 22:27:57 DEBUG "resultType": "vector", 22:27:57 DEBUG "resultType": "vector", 22:27:57 DEBUG "result": [ 22:27:57 DEBUG "result": [ 22:27:57 DEBUG "result": [ 22:27:57 DEBUG "result": [ 22:27:57 DEBUG "result": [ 22:27:57 DEBUG { 22:27:57 DEBUG { 22:27:57 DEBUG "result": [ 22:27:57 DEBUG "result": [ 22:27:57 DEBUG "metric": { 22:27:57 DEBUG "result": [ 22:27:57 DEBUG { 22:27:57 DEBUG "result": [ 22:27:57 DEBUG "result": [ 22:27:57 DEBUG "result": [ 22:27:57 DEBUG { 22:27:57 DEBUG { 22:27:57 DEBUG { 22:27:57 DEBUG { 22:27:57 DEBUG { 22:27:57 DEBUG { 22:27:57 DEBUG { 22:27:57 DEBUG "result": [ 22:27:57 DEBUG "result": [ 22:27:57 DEBUG "result": [ 22:27:57 DEBUG { 22:27:57 DEBUG { 22:27:57 DEBUG { 22:27:57 DEBUG { 22:27:57 DEBUG "result": [ 22:27:57 DEBUG "metric": { 22:27:57 DEBUG "metric": { 22:27:57 DEBUG { 22:27:57 DEBUG { 22:27:57 DEBUG "pod": "am-555f4db595-fz8sn" 22:27:57 DEBUG { 22:27:57 DEBUG "metric": { 22:27:57 DEBUG { 22:27:57 DEBUG { 22:27:57 DEBUG { 22:27:57 DEBUG "metric": { 22:27:57 DEBUG "metric": { 22:27:57 DEBUG "metric": { 22:27:57 DEBUG "metric": { 22:27:57 DEBUG "metric": { 22:27:57 DEBUG "metric": { 22:27:57 DEBUG "metric": { 22:27:57 DEBUG { 22:27:57 DEBUG { 22:27:57 DEBUG { 22:27:57 DEBUG "metric": { 22:27:57 DEBUG "metric": { 22:27:57 DEBUG "metric": { 22:27:57 DEBUG "metric": { 22:27:57 DEBUG { 22:27:57 DEBUG "instance": "10.142.0.107:9100" 22:27:57 DEBUG "pod": "ds-idrepo-2" 22:27:57 DEBUG "metric": { 22:27:57 DEBUG "metric": { 22:27:57 DEBUG }, 22:27:57 DEBUG "metric": { 22:27:57 DEBUG "pod": "am-555f4db595-fz8sn" 22:27:57 DEBUG "metric": { 22:27:57 DEBUG "metric": { 22:27:57 DEBUG "metric": { 22:27:57 DEBUG "pod": "am-555f4db595-fz8sn" 22:27:57 DEBUG "instance": "10.142.0.107:9100" 22:27:57 DEBUG "pod": "ds-idrepo-2" 22:27:57 DEBUG "instance": "10.142.0.107:9100" 22:27:57 DEBUG "pod": "am-555f4db595-fz8sn" 22:27:57 DEBUG "pod": "am-555f4db595-fz8sn" 22:27:57 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:27:57 DEBUG "metric": { 22:27:57 DEBUG "metric": { 22:27:57 DEBUG "metric": { 22:27:57 DEBUG "instance": "10.142.0.107:9100" 22:27:57 DEBUG "pod": "am-555f4db595-fz8sn" 22:27:57 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:27:57 DEBUG "pod": "ds-idrepo-2" 22:27:57 DEBUG "metric": { 22:27:57 DEBUG }, 22:27:57 DEBUG }, 22:27:57 DEBUG "pod": "ds-idrepo-2" 22:27:57 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:27:57 DEBUG "value": [ 22:27:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:27:57 DEBUG }, 22:27:57 DEBUG "pod": "am-555f4db595-fz8sn" 22:27:57 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:27:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:27:57 DEBUG }, 22:27:57 DEBUG }, 22:27:57 DEBUG }, 22:27:57 DEBUG }, 22:27:57 DEBUG }, 22:27:57 DEBUG }, 22:27:57 DEBUG }, 22:27:57 DEBUG "pod": "ds-idrepo-2" 22:27:57 DEBUG "pod": "ds-idrepo-2" 22:27:57 DEBUG "instance": "10.142.0.107:9100" 22:27:57 DEBUG }, 22:27:57 DEBUG }, 22:27:57 DEBUG }, 22:27:57 DEBUG }, 22:27:57 DEBUG "pod": "ds-idrepo-2" 22:27:57 DEBUG "value": [ 22:27:57 DEBUG "value": [ 22:27:57 DEBUG }, 22:27:57 DEBUG }, 22:27:57 DEBUG 22:27:57 DEBUG }, 22:27:57 DEBUG "value": [ 22:27:57 DEBUG }, 22:27:57 DEBUG }, 22:27:57 DEBUG }, 22:27:57 DEBUG "value": [ 22:27:57 DEBUG "value": [ 22:27:57 DEBUG 22:27:57 DEBUG "value": [ 22:27:57 DEBUG "value": [ 22:27:57 DEBUG "value": [ 22:27:57 DEBUG "value": [ 22:27:57 DEBUG }, 22:27:57 DEBUG }, 22:27:57 DEBUG }, 22:27:57 DEBUG "value": [ 22:27:57 DEBUG "value": [ 22:27:57 DEBUG "value": [ 22:27:57 DEBUG "value": [ 22:27:57 DEBUG }, 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG "value": [ 22:27:57 DEBUG "value": [ 22:27:57 DEBUG 22:27:57 DEBUG "value": [ 22:27:57 DEBUG 22:27:57 DEBUG "value": [ 22:27:57 DEBUG "value": [ 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG "value": [ 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG "value": [ 22:27:57 DEBUG "value": [ 22:27:57 DEBUG "value": [ 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG "value": [ 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG "value": [ 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG [...] 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG [...] 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG [...] 22:27:57 DEBUG [...] 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG [...] 22:27:57 DEBUG 22:27:57 DEBUG [...] 22:27:57 DEBUG 22:27:57 DEBUG [...] 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG [...] 22:27:57 DEBUG [...] 22:27:57 DEBUG [...] 22:27:57 DEBUG [...] 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG [...] 22:27:57 DEBUG [...] 22:27:57 DEBUG [...] 22:27:57 DEBUG [...] 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG [...] 22:27:57 DEBUG [...] 22:27:57 DEBUG 22:27:57 DEBUG [...] 22:27:57 DEBUG 22:27:57 DEBUG [...] 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG [...] 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG [...] 22:27:57 DEBUG [...] 22:27:57 DEBUG [...] 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG [...] 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG [...] 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG "pod": "am-555f4db595-kzxsp" 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG "instance": "10.142.0.43:9100" 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG "instance": "10.142.0.43:9100" 22:27:57 DEBUG "pod": "ds-idrepo-0" 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG }, 22:27:57 DEBUG 22:27:57 DEBUG "pod": "am-555f4db595-kzxsp" 22:27:57 DEBUG 22:27:57 DEBUG "pod": "overseer-0-7c9b667974-9mr2s" 22:27:57 DEBUG 22:27:57 DEBUG "pod": "am-555f4db595-kzxsp" 22:27:57 DEBUG 22:27:57 DEBUG }, 22:27:57 DEBUG "value": [ 22:27:57 DEBUG "pod": "am-555f4db595-kzxsp" 22:27:57 DEBUG "pod": "am-555f4db595-kzxsp" 22:27:57 DEBUG "pod": "overseer-0-7c9b667974-9mr2s" 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG 22:27:57 DEBUG "instance": "10.142.0.43:9100" 22:27:57 DEBUG "pod": "am-555f4db595-kzxsp" 22:27:57 DEBUG "pod": "overseer-0-7c9b667974-9mr2s" 22:27:57 DEBUG "pod": "ds-idrepo-0" 22:27:57 DEBUG 22:27:57 DEBUG }, 22:27:57 DEBUG }, 22:27:57 DEBUG "pod": "ds-idrepo-0" 22:27:57 DEBUG "pod": "overseer-0-7c9b667974-9mr2s" 22:27:57 DEBUG "value": [ 22:27:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:27:57 DEBUG }, 22:27:57 DEBUG "pod": "am-555f4db595-kzxsp" 22:27:57 DEBUG }, 22:27:57 DEBUG 22:27:57 DEBUG }, 22:27:57 DEBUG "pod": "ds-idrepo-0" 22:27:57 DEBUG "instance": "10.142.0.43:9100" 22:27:57 DEBUG 1713302877, 22:27:57 DEBUG }, 22:27:57 DEBUG }, 22:27:57 DEBUG }, 22:27:57 DEBUG "pod": "ds-cts-2" 22:27:57 DEBUG "pod": "ds-idrepo-0" 22:27:57 DEBUG "instance": "10.142.0.43:9100" 22:27:57 DEBUG }, 22:27:57 DEBUG }, 22:27:57 DEBUG }, 22:27:57 DEBUG }, 22:27:57 DEBUG "pod": "ds-idrepo-0" 22:27:57 DEBUG "value": [ 22:27:57 DEBUG "value": [ 22:27:57 DEBUG }, 22:27:57 DEBUG }, 22:27:57 DEBUG 1713302877, 22:27:57 DEBUG }, 22:27:57 DEBUG "value": [ 22:27:57 DEBUG }, 22:27:57 DEBUG "value": [ 22:27:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:27:57 DEBUG "value": [ 22:27:57 DEBUG }, 22:27:57 DEBUG }, 22:27:57 DEBUG "45452004.03333333" 22:27:57 DEBUG "value": [ 22:27:57 DEBUG "value": [ 22:27:57 DEBUG "value": [ 22:27:57 DEBUG }, 22:27:57 DEBUG }, 22:27:57 DEBUG }, 22:27:57 DEBUG "value": [ 22:27:57 DEBUG "value": [ 22:27:57 DEBUG "value": [ 22:27:57 DEBUG "value": [ 22:27:57 DEBUG }, 22:27:57 DEBUG 1713302877, 22:27:57 DEBUG 1713302877, 22:27:57 DEBUG "value": [ 22:27:57 DEBUG "value": [ 22:27:57 DEBUG "0" 22:27:57 DEBUG "value": [ 22:27:57 DEBUG 1713302877, 22:27:57 DEBUG "value": [ 22:27:57 DEBUG 1713302877, 22:27:57 DEBUG }, 22:27:57 DEBUG 1713302877, 22:27:57 DEBUG "value": [ 22:27:57 DEBUG "value": [ 22:27:57 DEBUG ] 22:27:57 DEBUG 1713302877, 22:27:57 DEBUG 1713302877, 22:27:57 DEBUG 1713302877, 22:27:57 DEBUG "value": [ 22:27:57 DEBUG "value": [ 22:27:57 DEBUG "value": [ 22:27:57 DEBUG 1713302877, 22:27:57 DEBUG 1713302877, 22:27:57 DEBUG 1713302877, 22:27:57 DEBUG 1713302877, 22:27:57 DEBUG "value": [ 22:27:57 DEBUG 1713302877, 22:27:57 DEBUG "0" 22:27:57 DEBUG 1713302877, 22:27:57 DEBUG 1713302877, 22:27:57 DEBUG ] 22:27:57 DEBUG 1713302877, 22:27:57 DEBUG "1991.8333333333333" 22:27:57 DEBUG 1713302877, 22:27:57 DEBUG "0" 22:27:57 DEBUG "value": [ 22:27:57 DEBUG "0" 22:27:57 DEBUG 1713302877, 22:27:57 DEBUG 1713302877, 22:27:57 DEBUG } 22:27:57 DEBUG "0.024" 22:27:57 DEBUG "1991.8" 22:27:57 DEBUG "11715.557400297463" 22:27:57 DEBUG 1713302877, 22:27:57 DEBUG 1713302877, 22:27:57 DEBUG 1713302877, 22:27:57 DEBUG "0" 22:27:57 DEBUG "1991.8333333333333" 22:27:57 DEBUG "3.464897837015046" 22:27:57 DEBUG "0" 22:27:57 DEBUG "0" 22:27:57 DEBUG ] 22:27:57 DEBUG "0" 22:27:57 DEBUG "0" 22:27:57 DEBUG } 22:27:57 DEBUG "652009472" 22:27:57 DEBUG ] 22:27:57 DEBUG "1940255" 22:27:57 DEBUG ] 22:27:57 DEBUG 1713302877, 22:27:57 DEBUG ] 22:27:57 DEBUG "0" 22:27:57 DEBUG "0.0005666666666674776" 22:27:57 DEBUG ] 22:27:57 DEBUG ] 22:27:57 DEBUG ] 22:27:57 DEBUG ] 22:27:57 DEBUG "45442168.86666666" 22:27:57 DEBUG "0" 22:27:57 DEBUG "0" 22:27:57 DEBUG "19956683776" 22:27:57 DEBUG ] 22:27:57 DEBUG ] 22:27:57 DEBUG ] 22:27:57 DEBUG ] 22:27:57 DEBUG ] 22:27:57 DEBUG } 22:27:57 DEBUG ] 22:27:57 DEBUG ] 22:27:57 DEBUG ] 22:27:57 DEBUG ] 22:27:57 DEBUG } 22:27:57 DEBUG ] 22:27:57 DEBUG } 22:27:57 DEBUG "6.340460362182714" 22:27:57 DEBUG } 22:27:57 DEBUG ] 22:27:57 DEBUG ] 22:27:57 DEBUG } 22:27:57 DEBUG } 22:27:57 DEBUG } 22:27:57 DEBUG } 22:27:57 DEBUG ] 22:27:57 DEBUG ] 22:27:57 DEBUG ] 22:27:57 DEBUG ] 22:27:57 DEBUG } 22:27:57 DEBUG } 22:27:57 DEBUG } 22:27:57 DEBUG } 22:27:57 DEBUG } 22:27:57 DEBUG ] 22:27:57 DEBUG } 22:27:57 DEBUG } 22:27:57 DEBUG } 22:27:57 DEBUG } 22:27:57 DEBUG ] 22:27:57 DEBUG } 22:27:57 DEBUG ] 22:27:57 DEBUG ] 22:27:57 DEBUG ] 22:27:57 DEBUG } 22:27:57 DEBUG } 22:27:57 DEBUG } 22:27:57 DEBUG ] 22:27:57 DEBUG ] 22:27:57 DEBUG ] 22:27:57 DEBUG } 22:27:57 DEBUG } 22:27:57 DEBUG } 22:27:57 DEBUG } 22:27:57 DEBUG ] 22:27:57 DEBUG ] 22:27:57 DEBUG ] 22:27:57 DEBUG ] 22:27:57 DEBUG ] 22:27:57 DEBUG } 22:27:57 DEBUG ] 22:27:57 DEBUG ] 22:27:57 DEBUG } 22:27:57 DEBUG ] 22:27:57 DEBUG } 22:27:57 DEBUG } 22:27:57 DEBUG } 22:27:57 DEBUG } 22:27:57 DEBUG } 22:27:57 DEBUG ] 22:27:57 DEBUG ] 22:27:57 DEBUG } 22:27:57 DEBUG } 22:27:57 DEBUG } 22:27:57 DEBUG ] 22:27:57 DEBUG ] 22:27:57 DEBUG ] 22:27:57 DEBUG ] 22:27:57 DEBUG } 22:27:57 DEBUG } 22:27:57 DEBUG } 22:27:57 DEBUG } 22:27:57 DEBUG } 22:27:57 DEBUG } 22:27:57 DEBUG } 22:27:57 DEBUG } 22:27:57 DEBUG } 22:27:57 DEBUG ] 22:27:57 DEBUG } 22:27:57 DEBUG ] 22:27:57 DEBUG } 22:27:57 DEBUG } 22:27:57 DEBUG } 22:27:57 DEBUG } 22:27:57 DEBUG } 22:27:57 DEBUG } 22:27:57 DEBUG } 22:27:57 DEBUG } 22:27:57 DEBUG } 22:27:57 DEBUG } 22:27:57 DEBUG } 22:27:57 DEBUG } 22:27:57 DEBUG } 22:27:57 DEBUG } 22:27:57 DEBUG } 22:27:57 DEBUG } 22:27:57 DEBUG } 22:27:57 DEBUG } 22:27:57 DEBUG } 22:27:57 DEBUG } 22:27:57 DEBUG } 22:27:57 DEBUG } 22:27:57 DEBUG } 22:27:57 INFO [loop_until]: OK (rc = 0) 22:27:57 DEBUG } 22:27:57 DEBUG } 22:27:57 DEBUG } 22:27:57 DEBUG } 22:27:57 DEBUG } 22:27:57 DEBUG --- stdout --- 22:27:57 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 62m 0% 5083Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7460m 24% 14755Mi 13% gke-xlou-cdm-default-pool-1b0769c8-nhlf 9237m 30% 10815Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 82m 0% 2641Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 4718m 15% 6488Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 8560m 28% 16602Mi 14% gke-xlou-cdm-default-pool-1b0769c8-xnmp 61m 0% 3148Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 9306m 31% 10372Mi 9% gke-xlou-cdm-ds-0f601751-hmm7 4620m 15% 5579Mi 4% gke-xlou-cdm-ds-0f601751-j488 7431m 24% 14097Mi 12% gke-xlou-cdm-ds-0f601751-mc5d 10625m 35% 3347Mi 2% gke-xlou-cdm-ds-0f601751-nj6q 9340m 31% 10420Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 4732m 15% 7355Mi 6% gke-xlou-cdm-frontend-a8771548-i82n 6411m 40% 5952Mi 10% 22:27:57 DEBUG --- stderr --- 22:27:57 DEBUG 22:28:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713302935" 22:28:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302935" 22:28:55 INFO [http_cmd]: http status code OK 22:28:55 DEBUG --- status code --- 22:28:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:28:55 DEBUG --- http response --- 22:28:55 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 22:28:55 ERROR [http_cmd]: ERROR 22:28:55 ERROR --- status code --- 22:28:55 ERROR http status code is 400 (expected [200, 401, 403, 404]) 22:28:55 ERROR --- http response --- 22:28:55 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 22:28:55 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 22:28:56 INFO 22:28:56 INFO [loop_until]: kubectl --namespace=xlou top pods 22:28:56 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:28:56 INFO [loop_until]: OK (rc = 0) 22:28:56 DEBUG --- stdout --- 22:28:56 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 9Mi am-555f4db595-4lbss 9208m 8591Mi am-555f4db595-fz8sn 9243m 8604Mi am-555f4db595-kzxsp 9414m 8555Mi ds-cts-0 8150m 12120Mi ds-cts-1 7548m 12204Mi ds-cts-2 7190m 12170Mi ds-idrepo-0 4719m 5032Mi ds-idrepo-1 4578m 2850Mi ds-idrepo-2 4389m 3140Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 4m 1207Mi idm-b85f9d978-rl76f 3m 1225Mi lodemon-bc756c647-cr7x6 12m 63Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-7c9b667974-9mr2s 6281m 531Mi 22:28:56 DEBUG --- stderr --- 22:28:56 DEBUG 22:28:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302937" 22:28:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302937" 22:28:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302937" 22:28:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302937" 22:28:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302937" 22:28:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302937" 22:28:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713302937" 22:28:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302937" 22:28:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302937" 22:28:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302937" 22:28:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713302937" 22:28:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302937" 22:28:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302937" 22:28:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302937" 22:28:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302937" 22:28:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302937" 22:28:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302937" 22:28:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302937" 22:28:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713302937" 22:28:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302937" 22:28:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713302937" 22:28:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302937" 22:28:57 INFO 22:28:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302937" 22:28:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302937" 22:28:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302937" 22:28:57 INFO [http_cmd]: http status code OK 22:28:57 INFO [http_cmd]: http status code OK 22:28:57 INFO [http_cmd]: http status code OK 22:28:57 INFO [http_cmd]: http status code OK 22:28:57 INFO [http_cmd]: http status code OK 22:28:57 INFO [http_cmd]: http status code OK 22:28:57 INFO [http_cmd]: http status code OK 22:28:57 INFO [http_cmd]: http status code OK 22:28:57 INFO [http_cmd]: http status code OK 22:28:57 INFO [http_cmd]: http status code OK 22:28:57 INFO [http_cmd]: http status code OK 22:28:57 INFO [http_cmd]: http status code OK 22:28:57 INFO [http_cmd]: http status code OK 22:28:57 INFO [loop_until]: kubectl --namespace=xlou top node 22:28:57 INFO [http_cmd]: http status code OK 22:28:57 INFO [http_cmd]: http status code OK 22:28:57 INFO [http_cmd]: http status code OK 22:28:57 INFO [http_cmd]: http status code OK 22:28:57 INFO [http_cmd]: http status code OK 22:28:57 DEBUG --- status code --- 22:28:57 INFO [http_cmd]: http status code OK 22:28:57 DEBUG --- status code --- 22:28:57 INFO [http_cmd]: http status code OK 22:28:57 DEBUG --- status code --- 22:28:57 DEBUG --- status code --- 22:28:57 INFO [http_cmd]: http status code OK 22:28:57 DEBUG --- status code --- 22:28:57 DEBUG --- status code --- 22:28:57 DEBUG --- status code --- 22:28:57 DEBUG --- status code --- 22:28:57 DEBUG --- status code --- 22:28:57 DEBUG --- status code --- 22:28:57 DEBUG --- status code --- 22:28:57 DEBUG --- status code --- 22:28:57 DEBUG --- status code --- 22:28:57 INFO [http_cmd]: http status code OK 22:28:57 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:28:57 INFO [http_cmd]: http status code OK 22:28:57 INFO [http_cmd]: http status code OK 22:28:57 DEBUG --- status code --- 22:28:57 DEBUG --- status code --- 22:28:57 DEBUG --- status code --- 22:28:57 DEBUG --- status code --- 22:28:57 DEBUG --- status code --- 22:28:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:28:57 DEBUG --- status code --- 22:28:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:28:57 DEBUG --- status code --- 22:28:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:28:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:28:57 DEBUG --- status code --- 22:28:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:28:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:28:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:28:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:28:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:28:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:28:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:28:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:28:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:28:57 DEBUG --- status code --- 22:28:57 DEBUG --- status code --- 22:28:57 DEBUG --- status code --- 22:28:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:28:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:28:57 INFO [http_cmd]: http status code OK 22:28:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:28:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:28:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:28:57 DEBUG --- http response --- 22:28:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:28:57 DEBUG --- http response --- 22:28:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:28:57 DEBUG --- http response --- 22:28:57 DEBUG --- http response --- 22:28:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:28:57 DEBUG --- http response --- 22:28:57 DEBUG --- http response --- 22:28:57 DEBUG --- http response --- 22:28:57 DEBUG --- http response --- 22:28:57 DEBUG --- http response --- 22:28:57 DEBUG --- http response --- 22:28:57 DEBUG --- http response --- 22:28:57 DEBUG --- http response --- 22:28:57 DEBUG --- http response --- 22:28:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:28:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:28:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:28:57 DEBUG --- http response --- 22:28:57 DEBUG --- http response --- 22:28:57 DEBUG --- status code --- 22:28:57 DEBUG --- http response --- 22:28:57 DEBUG --- http response --- 22:28:57 DEBUG --- http response --- 22:28:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:28:57 DEBUG --- http response --- 22:28:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:28:57 DEBUG --- http response --- 22:28:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:28:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:28:57 DEBUG --- http response --- 22:28:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:28:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:28:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:28:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:28:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:28:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:28:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:28:57 DEBUG --- http response --- 22:28:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:28:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:28:57 DEBUG --- http response --- 22:28:57 DEBUG --- http response --- 22:28:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:28:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:28:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:28:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:28:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:28:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:28:57 DEBUG ----- output ----- 22:28:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:28:57 DEBUG ----- output ----- 22:28:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:28:57 DEBUG ----- output ----- 22:28:57 DEBUG ----- output ----- 22:28:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:28:57 DEBUG ----- output ----- 22:28:57 DEBUG ----- output ----- 22:28:57 DEBUG ----- output ----- 22:28:57 DEBUG ----- output ----- 22:28:57 DEBUG ----- output ----- 22:28:57 DEBUG ----- output ----- 22:28:57 DEBUG ----- output ----- 22:28:57 DEBUG ----- output ----- 22:28:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:28:57 DEBUG ----- output ----- 22:28:57 DEBUG ----- output ----- 22:28:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:28:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:28:57 DEBUG ----- output ----- 22:28:57 DEBUG --- http response --- 22:28:57 DEBUG ----- output ----- 22:28:57 DEBUG ----- output ----- 22:28:57 DEBUG ----- output ----- 22:28:57 DEBUG { 22:28:57 DEBUG ----- output ----- 22:28:57 DEBUG { 22:28:57 DEBUG ----- output ----- 22:28:57 DEBUG { 22:28:57 DEBUG { 22:28:57 DEBUG ----- output ----- 22:28:57 DEBUG { 22:28:57 DEBUG { 22:28:57 DEBUG { 22:28:57 DEBUG { 22:28:57 DEBUG { 22:28:57 DEBUG { 22:28:57 DEBUG { 22:28:57 DEBUG { 22:28:57 DEBUG ----- output ----- 22:28:57 DEBUG { 22:28:57 DEBUG { 22:28:57 DEBUG "status": "success", 22:28:57 DEBUG ----- output ----- 22:28:57 DEBUG { 22:28:57 DEBUG { 22:28:57 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:28:57 DEBUG { 22:28:57 DEBUG { 22:28:57 DEBUG "status": "success", 22:28:57 DEBUG { 22:28:57 DEBUG "status": "success", 22:28:57 DEBUG { 22:28:57 DEBUG "status": "success", 22:28:57 DEBUG "status": "success", 22:28:57 DEBUG { 22:28:57 DEBUG "status": "success", 22:28:57 DEBUG "status": "success", 22:28:57 DEBUG "status": "success", 22:28:57 DEBUG "status": "success", 22:28:57 DEBUG "status": "success", 22:28:57 DEBUG "status": "success", 22:28:57 DEBUG "status": "success", 22:28:57 DEBUG "status": "success", 22:28:57 DEBUG { 22:28:57 DEBUG "status": "success", 22:28:57 DEBUG "data": { 22:28:57 DEBUG { 22:28:57 DEBUG "status": "success", 22:28:57 DEBUG "status": "success", 22:28:57 DEBUG ----- output ----- 22:28:57 DEBUG "status": "success", 22:28:57 DEBUG "status": "success", 22:28:57 DEBUG ----- output ----- 22:28:57 DEBUG "data": { 22:28:57 DEBUG "status": "success", 22:28:57 DEBUG "data": { 22:28:57 DEBUG "status": "success", 22:28:57 DEBUG "data": { 22:28:57 DEBUG "data": { 22:28:57 DEBUG "status": "success", 22:28:57 DEBUG "data": { 22:28:57 DEBUG "data": { 22:28:57 DEBUG "data": { 22:28:57 DEBUG "data": { 22:28:57 DEBUG "data": { 22:28:57 DEBUG "data": { 22:28:57 DEBUG "data": { 22:28:57 DEBUG "data": { 22:28:57 DEBUG "status": "success", 22:28:57 DEBUG "data": { 22:28:57 DEBUG "resultType": "vector", 22:28:57 DEBUG "status": "success", 22:28:57 DEBUG "data": { 22:28:57 DEBUG "data": { 22:28:57 DEBUG { 22:28:57 DEBUG "data": { 22:28:57 DEBUG "data": { 22:28:57 DEBUG { 22:28:57 DEBUG "resultType": "vector", 22:28:57 DEBUG "data": { 22:28:57 DEBUG "resultType": "vector", 22:28:57 DEBUG "data": { 22:28:57 DEBUG "resultType": "vector", 22:28:57 DEBUG "resultType": "vector", 22:28:57 DEBUG "data": { 22:28:57 DEBUG "resultType": "vector", 22:28:57 DEBUG "resultType": "vector", 22:28:57 DEBUG "resultType": "vector", 22:28:57 DEBUG "resultType": "vector", 22:28:57 DEBUG "resultType": "vector", 22:28:57 DEBUG "resultType": "vector", 22:28:57 DEBUG "resultType": "vector", 22:28:57 DEBUG "resultType": "vector", 22:28:57 DEBUG "data": { 22:28:57 DEBUG "resultType": "vector", 22:28:57 DEBUG "result": [ 22:28:57 DEBUG "data": { 22:28:57 DEBUG "resultType": "vector", 22:28:57 DEBUG "resultType": "vector", 22:28:57 DEBUG "status": "success", 22:28:57 DEBUG "resultType": "vector", 22:28:57 DEBUG "resultType": "vector", 22:28:57 DEBUG "status": "success", 22:28:57 DEBUG "result": [ 22:28:57 DEBUG "resultType": "vector", 22:28:57 DEBUG "result": [ 22:28:57 DEBUG "resultType": "vector", 22:28:57 DEBUG "result": [ 22:28:57 DEBUG "result": [ 22:28:57 DEBUG "resultType": "vector", 22:28:57 DEBUG "result": [ 22:28:57 DEBUG "result": [ 22:28:57 DEBUG "result": [ 22:28:57 DEBUG "result": [ 22:28:57 DEBUG "result": [ 22:28:57 DEBUG "result": [ 22:28:57 DEBUG "result": [ 22:28:57 DEBUG "result": [ 22:28:57 DEBUG "resultType": "vector", 22:28:57 DEBUG "result": [ 22:28:57 DEBUG { 22:28:57 DEBUG "resultType": "vector", 22:28:57 DEBUG "result": [ 22:28:57 DEBUG "result": [ 22:28:57 DEBUG "data": { 22:28:57 DEBUG "result": [ 22:28:57 DEBUG "result": [ 22:28:57 DEBUG "data": { 22:28:57 DEBUG { 22:28:57 DEBUG "result": [ 22:28:57 DEBUG { 22:28:57 DEBUG "result": [ 22:28:57 DEBUG { 22:28:57 DEBUG { 22:28:57 DEBUG "result": [ 22:28:57 DEBUG { 22:28:57 DEBUG { 22:28:57 DEBUG { 22:28:57 DEBUG { 22:28:57 DEBUG { 22:28:57 DEBUG { 22:28:57 DEBUG { 22:28:57 DEBUG { 22:28:57 DEBUG "result": [ 22:28:57 DEBUG { 22:28:57 DEBUG "metric": { 22:28:57 DEBUG "result": [ 22:28:57 DEBUG { 22:28:57 DEBUG { 22:28:57 DEBUG "resultType": "vector", 22:28:57 DEBUG { 22:28:57 DEBUG { 22:28:57 DEBUG "metric": { 22:28:57 DEBUG "metric": { 22:28:57 DEBUG { 22:28:57 DEBUG "metric": { 22:28:57 DEBUG { 22:28:57 DEBUG "metric": { 22:28:57 DEBUG "metric": { 22:28:57 DEBUG { 22:28:57 DEBUG "metric": { 22:28:57 DEBUG "metric": { 22:28:57 DEBUG "metric": { 22:28:57 DEBUG "metric": { 22:28:57 DEBUG "metric": { 22:28:57 DEBUG "metric": { 22:28:57 DEBUG "metric": { 22:28:57 DEBUG "metric": { 22:28:57 DEBUG { 22:28:57 DEBUG "metric": { 22:28:57 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:28:57 DEBUG { 22:28:57 DEBUG "metric": { 22:28:57 DEBUG "metric": { 22:28:57 DEBUG "result": [ 22:28:57 DEBUG "metric": { 22:28:57 DEBUG "resultType": "vector", 22:28:57 DEBUG "pod": "ds-idrepo-2" 22:28:57 DEBUG "pod": "am-555f4db595-fz8sn" 22:28:57 DEBUG "metric": { 22:28:57 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:28:57 DEBUG "metric": { 22:28:57 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:28:57 DEBUG "instance": "10.142.0.107:9100" 22:28:57 DEBUG "metric": { 22:28:57 DEBUG "pod": "ds-idrepo-2" 22:28:57 DEBUG "pod": "am-555f4db595-fz8sn" 22:28:57 DEBUG "pod": "am-555f4db595-fz8sn" 22:28:57 DEBUG "pod": "am-555f4db595-fz8sn" 22:28:57 DEBUG "pod": "ds-idrepo-2" 22:28:57 DEBUG "pod": "am-555f4db595-fz8sn" 22:28:57 DEBUG "pod": "am-555f4db595-fz8sn" 22:28:57 DEBUG "pod": "ds-idrepo-2" 22:28:57 DEBUG "metric": { 22:28:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:28:57 DEBUG }, 22:28:57 DEBUG "metric": { 22:28:57 DEBUG "instance": "10.142.0.107:9100" 22:28:57 DEBUG "instance": "10.142.0.107:9100" 22:28:57 DEBUG }, 22:28:57 DEBUG "instance": "10.142.0.107:9100" 22:28:57 DEBUG "result": [ 22:28:57 DEBUG }, 22:28:57 DEBUG }, 22:28:57 DEBUG "pod": "ds-idrepo-2" 22:28:57 DEBUG }, 22:28:57 DEBUG "pod": "ds-idrepo-2" 22:28:57 DEBUG }, 22:28:57 DEBUG }, 22:28:57 DEBUG "pod": "am-555f4db595-fz8sn" 22:28:57 DEBUG }, 22:28:57 DEBUG }, 22:28:57 DEBUG }, 22:28:57 DEBUG }, 22:28:57 DEBUG }, 22:28:57 DEBUG }, 22:28:57 DEBUG }, 22:28:57 DEBUG }, 22:28:57 DEBUG "pod": "ds-idrepo-2" 22:28:57 DEBUG }, 22:28:57 DEBUG "value": [ 22:28:57 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:28:57 DEBUG }, 22:28:57 DEBUG "value": [ 22:28:57 DEBUG }, 22:28:57 DEBUG { 22:28:57 DEBUG "value": [ 22:28:57 DEBUG "value": [ 22:28:57 DEBUG }, 22:28:57 DEBUG "value": [ 22:28:57 DEBUG }, 22:28:57 DEBUG "value": [ 22:28:57 DEBUG "value": [ 22:28:57 DEBUG }, 22:28:57 DEBUG "value": [ 22:28:57 DEBUG "value": [ 22:28:57 DEBUG "value": [ 22:28:57 DEBUG "value": [ 22:28:57 DEBUG "value": [ 22:28:57 DEBUG "value": [ 22:28:57 DEBUG "value": [ 22:28:57 DEBUG "value": [ 22:28:57 DEBUG { 22:28:57 DEBUG }, 22:28:57 DEBUG "value": [ 22:28:57 DEBUG 22:28:57 DEBUG }, 22:28:57 DEBUG "value": [ 22:28:57 DEBUG 22:28:57 DEBUG "value": [ 22:28:57 DEBUG "metric": { 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG "value": [ 22:28:57 DEBUG 22:28:57 DEBUG "value": [ 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG "value": [ 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG "metric": { 22:28:57 DEBUG "value": [ 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG "value": [ 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG "instance": "10.142.0.107:9100" 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG }, 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG }, 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG [...] 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG [...] 22:28:57 DEBUG 22:28:57 DEBUG "value": [ 22:28:57 DEBUG [...] 22:28:57 DEBUG [...] 22:28:57 DEBUG 22:28:57 DEBUG [...] 22:28:57 DEBUG 22:28:57 DEBUG [...] 22:28:57 DEBUG [...] 22:28:57 DEBUG 22:28:57 DEBUG [...] 22:28:57 DEBUG [...] 22:28:57 DEBUG [...] 22:28:57 DEBUG [...] 22:28:57 DEBUG [...] 22:28:57 DEBUG [...] 22:28:57 DEBUG [...] 22:28:57 DEBUG [...] 22:28:57 DEBUG "value": [ 22:28:57 DEBUG 22:28:57 DEBUG [...] 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG [...] 22:28:57 DEBUG 22:28:57 DEBUG [...] 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG [...] 22:28:57 DEBUG 22:28:57 DEBUG [...] 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG [...] 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG [...] 22:28:57 DEBUG [...] 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG "pod": "am-555f4db595-kzxsp" 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG "pod": "ds-idrepo-0" 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG [...] 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG "instance": "10.142.0.43:9100" 22:28:57 DEBUG 22:28:57 DEBUG [...] 22:28:57 DEBUG "pod": "ds-idrepo-0" 22:28:57 DEBUG "pod": "am-555f4db595-kzxsp" 22:28:57 DEBUG 22:28:57 DEBUG "pod": "overseer-0-7c9b667974-9mr2s" 22:28:57 DEBUG 22:28:57 DEBUG "pod": "ds-cts-2" 22:28:57 DEBUG "instance": "10.142.0.43:9100" 22:28:57 DEBUG 22:28:57 DEBUG "pod": "ds-idrepo-0" 22:28:57 DEBUG }, 22:28:57 DEBUG "pod": "am-555f4db595-kzxsp" 22:28:57 DEBUG 22:28:57 DEBUG }, 22:28:57 DEBUG "pod": "am-555f4db595-kzxsp" 22:28:57 DEBUG "pod": "ds-idrepo-0" 22:28:57 DEBUG 22:28:57 DEBUG "pod": "overseer-0-7c9b667974-9mr2s" 22:28:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG 22:28:57 DEBUG "instance": "10.142.0.43:9100" 22:28:57 DEBUG }, 22:28:57 DEBUG "instance": "10.142.0.43:9100" 22:28:57 DEBUG 22:28:57 DEBUG }, 22:28:57 DEBUG }, 22:28:57 DEBUG "pod": "ds-idrepo-0" 22:28:57 DEBUG }, 22:28:57 DEBUG "pod": "ds-cts-2" 22:28:57 DEBUG }, 22:28:57 DEBUG }, 22:28:57 DEBUG "pod": "am-555f4db595-kzxsp" 22:28:57 DEBUG }, 22:28:57 DEBUG "value": [ 22:28:57 DEBUG }, 22:28:57 DEBUG "pod": "am-555f4db595-kzxsp" 22:28:57 DEBUG "value": [ 22:28:57 DEBUG }, 22:28:57 DEBUG }, 22:28:57 DEBUG 22:28:57 DEBUG }, 22:28:57 DEBUG }, 22:28:57 DEBUG "pod": "ds-idrepo-0" 22:28:57 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:28:57 DEBUG "pod": "am-555f4db595-kzxsp" 22:28:57 DEBUG }, 22:28:57 DEBUG "value": [ 22:28:57 DEBUG }, 22:28:57 DEBUG 22:28:57 DEBUG "value": [ 22:28:57 DEBUG "value": [ 22:28:57 DEBUG }, 22:28:57 DEBUG "value": [ 22:28:57 DEBUG }, 22:28:57 DEBUG "value": [ 22:28:57 DEBUG "value": [ 22:28:57 DEBUG }, 22:28:57 DEBUG "value": [ 22:28:57 DEBUG 1713302937, 22:28:57 DEBUG "value": [ 22:28:57 DEBUG }, 22:28:57 DEBUG 1713302937, 22:28:57 DEBUG "value": [ 22:28:57 DEBUG "value": [ 22:28:57 DEBUG 22:28:57 DEBUG "value": [ 22:28:57 DEBUG "value": [ 22:28:57 DEBUG }, 22:28:57 DEBUG }, 22:28:57 DEBUG }, 22:28:57 DEBUG "value": [ 22:28:57 DEBUG 1713302937, 22:28:57 DEBUG "value": [ 22:28:57 DEBUG 22:28:57 DEBUG "pod": "ds-cts-2" 22:28:57 DEBUG 1713302937, 22:28:57 DEBUG "value": [ 22:28:57 DEBUG 1713302937, 22:28:57 DEBUG "0" 22:28:57 DEBUG 1713302937, 22:28:57 DEBUG 1713302937, 22:28:57 DEBUG "value": [ 22:28:57 DEBUG 1713302937, 22:28:57 DEBUG "1999.1333333333332" 22:28:57 DEBUG 1713302937, 22:28:57 DEBUG "value": [ 22:28:57 DEBUG "0" 22:28:57 DEBUG 1713302937, 22:28:57 DEBUG 1713302937, 22:28:57 DEBUG "instance": "10.142.0.43:9100" 22:28:57 DEBUG 1713302937, 22:28:57 DEBUG 1713302937, 22:28:57 DEBUG "value": [ 22:28:57 DEBUG "value": [ 22:28:57 DEBUG "value": [ 22:28:57 DEBUG 1713302937, 22:28:57 DEBUG "45385313.33333333" 22:28:57 DEBUG 1713302937, 22:28:57 DEBUG 1713302937, 22:28:57 DEBUG }, 22:28:57 DEBUG "0.024" 22:28:57 DEBUG 1713302937, 22:28:57 DEBUG "value": [ 22:28:57 DEBUG ] 22:28:57 DEBUG "16215.360253365006" 22:28:57 DEBUG "45386053.86666666" 22:28:57 DEBUG 1713302937, 22:28:57 DEBUG "0" 22:28:57 DEBUG ] 22:28:57 DEBUG "1999.1" 22:28:57 DEBUG ] 22:28:57 DEBUG ] 22:28:57 DEBUG "1999.1666666666667" 22:28:57 DEBUG "0" 22:28:57 DEBUG }, 22:28:57 DEBUG "0" 22:28:57 DEBUG "657465344" 22:28:57 DEBUG 1713302937, 22:28:57 DEBUG 1713302937, 22:28:57 DEBUG 1713302937, 22:28:57 DEBUG "19957505024" 22:28:57 DEBUG ] 22:28:57 DEBUG "0.00040000000000001514" 22:28:57 DEBUG "0" 22:28:57 DEBUG "value": [ 22:28:57 DEBUG ] 22:28:57 DEBUG "0" 22:28:57 DEBUG 1713302937, 22:28:57 DEBUG } 22:28:57 DEBUG ] 22:28:57 DEBUG ] 22:28:57 DEBUG "2059943.5" 22:28:57 DEBUG ] 22:28:57 DEBUG } 22:28:57 DEBUG 1713302937, 22:28:57 DEBUG } 22:28:57 DEBUG } 22:28:57 DEBUG ] 22:28:57 DEBUG ] 22:28:57 DEBUG "value": [ 22:28:57 DEBUG ] 22:28:57 DEBUG ] 22:28:57 DEBUG "0" 22:28:57 DEBUG "6.292916088147783" 22:28:57 DEBUG "0" 22:28:57 DEBUG ] 22:28:57 DEBUG } 22:28:57 DEBUG ] 22:28:57 DEBUG ] 22:28:57 DEBUG 1713302937, 22:28:57 DEBUG } 22:28:57 DEBUG ] 22:28:57 DEBUG "0" 22:28:57 DEBUG ] 22:28:57 DEBUG } 22:28:57 DEBUG } 22:28:57 DEBUG ] 22:28:57 DEBUG } 22:28:57 DEBUG ] 22:28:57 DEBUG "0" 22:28:57 DEBUG ] 22:28:57 DEBUG ] 22:28:57 DEBUG } 22:28:57 DEBUG ] 22:28:57 DEBUG 1713302937, 22:28:57 DEBUG } 22:28:57 DEBUG } 22:28:57 DEBUG ] 22:28:57 DEBUG ] 22:28:57 DEBUG ] 22:28:57 DEBUG } 22:28:57 DEBUG ] 22:28:57 DEBUG } 22:28:57 DEBUG } 22:28:57 DEBUG "37428553.98258116" 22:28:57 DEBUG ] 22:28:57 DEBUG } 22:28:57 DEBUG ] 22:28:57 DEBUG } 22:28:57 DEBUG ] 22:28:57 DEBUG ] 22:28:57 DEBUG } 22:28:57 DEBUG ] 22:28:57 DEBUG } 22:28:57 DEBUG ] 22:28:57 DEBUG } 22:28:57 DEBUG } 22:28:57 DEBUG } 22:28:57 DEBUG } 22:28:57 DEBUG "0.00033333333333333365" 22:28:57 DEBUG ] 22:28:57 DEBUG ] 22:28:57 DEBUG } 22:28:57 DEBUG } 22:28:57 DEBUG } 22:28:57 DEBUG ] 22:28:57 DEBUG } 22:28:57 DEBUG ] 22:28:57 DEBUG ] 22:28:57 DEBUG ] 22:28:57 DEBUG } 22:28:57 DEBUG ] 22:28:57 DEBUG } 22:28:57 DEBUG } 22:28:57 DEBUG } 22:28:57 DEBUG } 22:28:57 DEBUG ] 22:28:57 DEBUG } 22:28:57 DEBUG } 22:28:57 DEBUG } 22:28:57 DEBUG } 22:28:57 DEBUG } 22:28:57 DEBUG ] 22:28:57 DEBUG } 22:28:57 DEBUG ] 22:28:57 DEBUG } 22:28:57 DEBUG } 22:28:57 DEBUG ] 22:28:57 DEBUG ] 22:28:57 DEBUG ] 22:28:57 DEBUG } 22:28:57 DEBUG } 22:28:57 DEBUG } 22:28:57 DEBUG } 22:28:57 DEBUG } 22:28:57 DEBUG } 22:28:57 DEBUG } 22:28:57 DEBUG ] 22:28:57 DEBUG } 22:28:57 DEBUG } 22:28:57 DEBUG } 22:28:57 DEBUG } 22:28:57 DEBUG ] 22:28:57 DEBUG } 22:28:57 DEBUG } 22:28:57 DEBUG } 22:28:57 DEBUG } 22:28:57 DEBUG } 22:28:57 DEBUG } 22:28:57 DEBUG } 22:28:57 DEBUG } 22:28:57 DEBUG } 22:28:57 DEBUG } 22:28:57 DEBUG ] 22:28:57 DEBUG } 22:28:57 DEBUG } 22:28:57 DEBUG } 22:28:57 DEBUG } 22:28:57 DEBUG } 22:28:57 DEBUG ] 22:28:57 DEBUG } 22:28:57 DEBUG } 22:28:57 DEBUG } 22:28:57 DEBUG } 22:28:57 DEBUG } 22:28:57 DEBUG } 22:28:57 DEBUG } 22:28:57 DEBUG } 22:28:57 DEBUG } 22:28:58 INFO [loop_until]: OK (rc = 0) 22:28:58 DEBUG --- stdout --- 22:28:58 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 5092Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7629m 25% 14761Mi 13% gke-xlou-cdm-default-pool-1b0769c8-nhlf 9386m 31% 10820Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 83m 0% 2642Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 4752m 15% 6492Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 8584m 28% 16613Mi 14% gke-xlou-cdm-default-pool-1b0769c8-xnmp 61m 0% 3147Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 9252m 30% 10373Mi 9% gke-xlou-cdm-ds-0f601751-hmm7 4514m 15% 5588Mi 5% gke-xlou-cdm-ds-0f601751-j488 7584m 25% 14104Mi 12% gke-xlou-cdm-ds-0f601751-mc5d 10318m 34% 3344Mi 2% gke-xlou-cdm-ds-0f601751-nj6q 9664m 32% 10424Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 4715m 15% 7354Mi 6% gke-xlou-cdm-frontend-a8771548-i82n 6432m 40% 5950Mi 10% 22:28:58 DEBUG --- stderr --- 22:28:58 DEBUG 22:29:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302995" 22:29:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713302995" 22:29:55 ERROR [http_cmd]: ERROR 22:29:55 ERROR --- status code --- 22:29:55 ERROR http status code is 400 (expected [200, 401, 403, 404]) 22:29:55 ERROR --- http response --- 22:29:55 INFO [http_cmd]: http status code OK 22:29:55 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 22:29:55 DEBUG --- status code --- 22:29:55 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 22:29:55 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:29:55 DEBUG --- http response --- 22:29:55 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 22:29:56 INFO 22:29:56 INFO [loop_until]: kubectl --namespace=xlou top pods 22:29:56 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:29:56 INFO [loop_until]: OK (rc = 0) 22:29:56 DEBUG --- stdout --- 22:29:56 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 9Mi am-555f4db595-4lbss 9011m 8592Mi am-555f4db595-fz8sn 9287m 8603Mi am-555f4db595-kzxsp 9221m 8556Mi ds-cts-0 8046m 12129Mi ds-cts-1 7435m 12211Mi ds-cts-2 7499m 12175Mi ds-idrepo-0 4716m 5032Mi ds-idrepo-1 4446m 2850Mi ds-idrepo-2 4523m 3139Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 3m 1208Mi idm-b85f9d978-rl76f 4m 1225Mi lodemon-bc756c647-cr7x6 24m 63Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-7c9b667974-9mr2s 6178m 533Mi 22:29:56 DEBUG --- stderr --- 22:29:56 DEBUG 22:29:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302997" 22:29:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302997" 22:29:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302997" 22:29:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713302997" 22:29:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302997" 22:29:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302997" 22:29:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713302997" 22:29:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302997" 22:29:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302997" 22:29:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302997" 22:29:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302997" 22:29:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713302997" 22:29:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302997" 22:29:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302997" 22:29:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302997" 22:29:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302997" 22:29:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302997" 22:29:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302997" 22:29:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302997" 22:29:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302997" 22:29:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302997" 22:29:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713302997" 22:29:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302997" 22:29:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713302997" 22:29:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713302997" 22:29:57 INFO [http_cmd]: http status code OK 22:29:57 INFO [http_cmd]: http status code OK 22:29:57 INFO [http_cmd]: http status code OK 22:29:57 INFO [http_cmd]: http status code OK 22:29:57 INFO [http_cmd]: http status code OK 22:29:57 INFO [http_cmd]: http status code OK 22:29:57 INFO [http_cmd]: http status code OK 22:29:57 INFO [http_cmd]: http status code OK 22:29:58 INFO [http_cmd]: http status code OK 22:29:58 INFO [http_cmd]: http status code OK 22:29:58 INFO [http_cmd]: http status code OK 22:29:58 INFO [http_cmd]: http status code OK 22:29:58 INFO [http_cmd]: http status code OK 22:29:58 INFO [http_cmd]: http status code OK 22:29:58 DEBUG --- status code --- 22:29:58 DEBUG --- status code --- 22:29:58 DEBUG --- status code --- 22:29:58 DEBUG --- status code --- 22:29:58 DEBUG --- status code --- 22:29:58 DEBUG --- status code --- 22:29:58 DEBUG --- status code --- 22:29:58 DEBUG --- status code --- 22:29:58 DEBUG --- status code --- 22:29:58 DEBUG --- status code --- 22:29:58 DEBUG --- status code --- 22:29:58 DEBUG --- status code --- 22:29:58 DEBUG --- status code --- 22:29:58 DEBUG --- status code --- 22:29:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:29:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:29:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:29:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:29:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:29:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:29:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:29:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:29:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:29:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:29:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:29:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:29:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:29:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:29:58 DEBUG --- http response --- 22:29:58 DEBUG --- http response --- 22:29:58 DEBUG --- http response --- 22:29:58 DEBUG --- http response --- 22:29:58 DEBUG --- http response --- 22:29:58 DEBUG --- http response --- 22:29:58 DEBUG --- http response --- 22:29:58 DEBUG --- http response --- 22:29:58 DEBUG --- http response --- 22:29:58 DEBUG --- http response --- 22:29:58 DEBUG --- http response --- 22:29:58 DEBUG --- http response --- 22:29:58 DEBUG --- http response --- 22:29:58 DEBUG --- http response --- 22:29:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:29:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:29:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:29:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:29:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:29:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:29:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:29:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:29:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:29:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:29:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:29:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:29:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:29:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:29:58 INFO [http_cmd]: http status code OK 22:29:58 DEBUG ----- output ----- 22:29:58 INFO [http_cmd]: http status code OK 22:29:58 DEBUG ----- output ----- 22:29:58 DEBUG ----- output ----- 22:29:58 DEBUG ----- output ----- 22:29:58 INFO [http_cmd]: http status code OK 22:29:58 DEBUG ----- output ----- 22:29:58 INFO [http_cmd]: http status code OK 22:29:58 INFO [http_cmd]: http status code OK 22:29:58 DEBUG ----- output ----- 22:29:58 INFO [http_cmd]: http status code OK 22:29:58 INFO [http_cmd]: http status code OK 22:29:58 INFO [http_cmd]: http status code OK 22:29:58 INFO [http_cmd]: http status code OK 22:29:58 INFO [http_cmd]: http status code OK 22:29:58 INFO [http_cmd]: http status code OK 22:29:58 DEBUG ----- output ----- 22:29:58 DEBUG ----- output ----- 22:29:58 DEBUG ----- output ----- 22:29:58 DEBUG ----- output ----- 22:29:58 DEBUG ----- output ----- 22:29:58 DEBUG ----- output ----- 22:29:58 DEBUG ----- output ----- 22:29:58 DEBUG ----- output ----- 22:29:58 DEBUG --- status code --- 22:29:58 DEBUG { 22:29:58 DEBUG --- status code --- 22:29:58 DEBUG { 22:29:58 DEBUG { 22:29:58 DEBUG { 22:29:58 DEBUG --- status code --- 22:29:58 DEBUG { 22:29:58 DEBUG --- status code --- 22:29:58 DEBUG --- status code --- 22:29:58 DEBUG { 22:29:58 DEBUG --- status code --- 22:29:58 DEBUG --- status code --- 22:29:58 DEBUG --- status code --- 22:29:58 DEBUG --- status code --- 22:29:58 DEBUG --- status code --- 22:29:58 DEBUG --- status code --- 22:29:58 DEBUG { 22:29:58 DEBUG { 22:29:58 DEBUG { 22:29:58 DEBUG { 22:29:58 DEBUG { 22:29:58 DEBUG { 22:29:58 DEBUG { 22:29:58 DEBUG { 22:29:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:29:58 DEBUG "status": "success", 22:29:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:29:58 DEBUG "status": "success", 22:29:58 DEBUG "status": "success", 22:29:58 DEBUG "status": "success", 22:29:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:29:58 DEBUG "status": "success", 22:29:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:29:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:29:58 DEBUG "status": "success", 22:29:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:29:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:29:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:29:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:29:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:29:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:29:58 DEBUG "status": "success", 22:29:58 DEBUG "status": "success", 22:29:58 DEBUG "status": "success", 22:29:58 DEBUG "status": "success", 22:29:58 DEBUG "status": "success", 22:29:58 DEBUG "status": "success", 22:29:58 DEBUG "status": "success", 22:29:58 DEBUG "status": "success", 22:29:58 DEBUG --- http response --- 22:29:58 DEBUG "data": { 22:29:58 DEBUG --- http response --- 22:29:58 DEBUG "data": { 22:29:58 DEBUG "data": { 22:29:58 DEBUG "data": { 22:29:58 DEBUG --- http response --- 22:29:58 DEBUG "data": { 22:29:58 DEBUG --- http response --- 22:29:58 DEBUG --- http response --- 22:29:58 DEBUG "data": { 22:29:58 DEBUG --- http response --- 22:29:58 DEBUG --- http response --- 22:29:58 DEBUG --- http response --- 22:29:58 DEBUG --- http response --- 22:29:58 DEBUG --- http response --- 22:29:58 DEBUG --- http response --- 22:29:58 DEBUG "data": { 22:29:58 DEBUG "data": { 22:29:58 DEBUG "data": { 22:29:58 DEBUG "data": { 22:29:58 DEBUG "data": { 22:29:58 DEBUG "data": { 22:29:58 DEBUG "data": { 22:29:58 DEBUG "data": { 22:29:58 DEBUG "resultType": "vector", 22:29:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:29:58 DEBUG "resultType": "vector", 22:29:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:29:58 DEBUG "resultType": "vector", 22:29:58 DEBUG "resultType": "vector", 22:29:58 DEBUG "resultType": "vector", 22:29:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:29:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:29:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:29:58 DEBUG "resultType": "vector", 22:29:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:29:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:29:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:29:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:29:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:29:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:29:58 DEBUG "resultType": "vector", 22:29:58 DEBUG "resultType": "vector", 22:29:58 DEBUG "resultType": "vector", 22:29:58 DEBUG "resultType": "vector", 22:29:58 DEBUG "resultType": "vector", 22:29:58 DEBUG "resultType": "vector", 22:29:58 DEBUG "resultType": "vector", 22:29:58 DEBUG "resultType": "vector", 22:29:58 DEBUG "result": [ 22:29:58 DEBUG ----- output ----- 22:29:58 DEBUG "result": [ 22:29:58 DEBUG ----- output ----- 22:29:58 DEBUG "result": [ 22:29:58 DEBUG "result": [ 22:29:58 DEBUG "result": [ 22:29:58 DEBUG ----- output ----- 22:29:58 DEBUG ----- output ----- 22:29:58 DEBUG ----- output ----- 22:29:58 DEBUG "result": [ 22:29:58 DEBUG ----- output ----- 22:29:58 DEBUG ----- output ----- 22:29:58 DEBUG ----- output ----- 22:29:58 DEBUG ----- output ----- 22:29:58 DEBUG ----- output ----- 22:29:58 DEBUG ----- output ----- 22:29:58 DEBUG "result": [ 22:29:58 DEBUG "result": [ 22:29:58 DEBUG "result": [ 22:29:58 DEBUG "result": [ 22:29:58 DEBUG "result": [ 22:29:58 DEBUG "result": [ 22:29:58 DEBUG "result": [ 22:29:58 DEBUG "result": [ 22:29:58 DEBUG { 22:29:58 DEBUG { 22:29:58 DEBUG { 22:29:58 DEBUG { 22:29:58 DEBUG { 22:29:58 DEBUG { 22:29:58 DEBUG { 22:29:58 DEBUG { 22:29:58 DEBUG { 22:29:58 DEBUG { 22:29:58 DEBUG { 22:29:58 DEBUG { 22:29:58 DEBUG { 22:29:58 DEBUG { 22:29:58 DEBUG { 22:29:58 DEBUG { 22:29:58 DEBUG { 22:29:58 DEBUG { 22:29:58 DEBUG { 22:29:58 DEBUG { 22:29:58 DEBUG { 22:29:58 DEBUG { 22:29:58 DEBUG { 22:29:58 DEBUG { 22:29:58 DEBUG { 22:29:58 DEBUG "metric": { 22:29:58 DEBUG "status": "success", 22:29:58 DEBUG "metric": { 22:29:58 DEBUG "status": "success", 22:29:58 DEBUG "metric": { 22:29:58 DEBUG "metric": { 22:29:58 DEBUG "metric": { 22:29:58 DEBUG "status": "success", 22:29:58 DEBUG "status": "success", 22:29:58 DEBUG "status": "success", 22:29:58 DEBUG "metric": { 22:29:58 DEBUG "status": "success", 22:29:58 DEBUG "status": "success", 22:29:58 DEBUG "status": "success", 22:29:58 DEBUG "status": "success", 22:29:58 DEBUG "status": "success", 22:29:58 DEBUG "status": "success", 22:29:58 DEBUG "metric": { 22:29:58 DEBUG "metric": { 22:29:58 DEBUG "metric": { 22:29:58 DEBUG "metric": { 22:29:58 DEBUG "metric": { 22:29:58 DEBUG "metric": { 22:29:58 DEBUG "metric": { 22:29:58 DEBUG "metric": { 22:29:58 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:29:58 DEBUG "data": { 22:29:58 DEBUG "pod": "am-555f4db595-fz8sn" 22:29:58 DEBUG "data": { 22:29:58 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:29:58 DEBUG "pod": "ds-idrepo-2" 22:29:58 DEBUG "pod": "am-555f4db595-fz8sn" 22:29:58 DEBUG "data": { 22:29:58 DEBUG "data": { 22:29:58 DEBUG "data": { 22:29:58 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:29:58 DEBUG "data": { 22:29:58 DEBUG "data": { 22:29:58 DEBUG "data": { 22:29:58 DEBUG "data": { 22:29:58 DEBUG "data": { 22:29:58 DEBUG "data": { 22:29:58 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:29:58 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:29:58 DEBUG "instance": "10.142.0.107:9100" 22:29:58 DEBUG "instance": "10.142.0.107:9100" 22:29:58 DEBUG "pod": "am-555f4db595-fz8sn" 22:29:58 DEBUG "pod": "am-555f4db595-fz8sn" 22:29:58 DEBUG "pod": "ds-idrepo-2" 22:29:58 DEBUG "pod": "am-555f4db595-fz8sn" 22:29:58 DEBUG }, 22:29:58 DEBUG "resultType": "vector", 22:29:58 DEBUG }, 22:29:58 DEBUG "resultType": "vector", 22:29:58 DEBUG }, 22:29:58 DEBUG }, 22:29:58 DEBUG }, 22:29:58 DEBUG "resultType": "vector", 22:29:58 DEBUG "resultType": "vector", 22:29:58 DEBUG "resultType": "vector", 22:29:58 DEBUG }, 22:29:58 DEBUG "resultType": "vector", 22:29:58 DEBUG "resultType": "vector", 22:29:58 DEBUG "resultType": "vector", 22:29:58 DEBUG "resultType": "vector", 22:29:58 DEBUG "resultType": "vector", 22:29:58 DEBUG "resultType": "vector", 22:29:58 DEBUG }, 22:29:58 DEBUG }, 22:29:58 DEBUG }, 22:29:58 DEBUG }, 22:29:58 DEBUG }, 22:29:58 DEBUG }, 22:29:58 DEBUG }, 22:29:58 DEBUG }, 22:29:58 DEBUG "value": [ 22:29:58 DEBUG "result": [ 22:29:58 DEBUG "value": [ 22:29:58 DEBUG "result": [ 22:29:58 DEBUG "value": [ 22:29:58 DEBUG "value": [ 22:29:58 DEBUG "value": [ 22:29:58 DEBUG "result": [ 22:29:58 DEBUG "result": [ 22:29:58 DEBUG "result": [ 22:29:58 DEBUG "value": [ 22:29:58 DEBUG "result": [ 22:29:58 DEBUG "result": [ 22:29:58 DEBUG "result": [ 22:29:58 DEBUG "result": [ 22:29:58 DEBUG "result": [ 22:29:58 DEBUG "result": [ 22:29:58 DEBUG "value": [ 22:29:58 DEBUG "value": [ 22:29:58 DEBUG "value": [ 22:29:58 DEBUG "value": [ 22:29:58 DEBUG "value": [ 22:29:58 DEBUG "value": [ 22:29:58 DEBUG "value": [ 22:29:58 DEBUG "value": [ 22:29:58 DEBUG 22:29:58 DEBUG { 22:29:58 DEBUG 22:29:58 DEBUG { 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG { 22:29:58 DEBUG { 22:29:58 DEBUG { 22:29:58 DEBUG 22:29:58 DEBUG { 22:29:58 DEBUG { 22:29:58 DEBUG { 22:29:58 DEBUG { 22:29:58 DEBUG { 22:29:58 DEBUG { 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG "metric": { 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG "metric": { 22:29:58 DEBUG "metric": { 22:29:58 DEBUG "metric": { 22:29:58 DEBUG 22:29:58 DEBUG "metric": { 22:29:58 DEBUG "metric": { 22:29:58 DEBUG "metric": { 22:29:58 DEBUG "metric": { 22:29:58 DEBUG "metric": { 22:29:58 DEBUG "metric": { 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG [...] 22:29:58 DEBUG "metric": { 22:29:58 DEBUG 22:29:58 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG "pod": "ds-idrepo-2" 22:29:58 DEBUG "instance": "10.142.0.107:9100" 22:29:58 DEBUG "pod": "ds-idrepo-2" 22:29:58 DEBUG 22:29:58 DEBUG "instance": "10.142.0.107:9100" 22:29:58 DEBUG "pod": "ds-idrepo-2" 22:29:58 DEBUG "pod": "ds-idrepo-2" 22:29:58 DEBUG "pod": "am-555f4db595-fz8sn" 22:29:58 DEBUG "pod": "am-555f4db595-fz8sn" 22:29:58 DEBUG "instance": "10.142.0.107:9100" 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG "pod": "ds-idrepo-2" 22:29:58 DEBUG [...] 22:29:58 DEBUG }, 22:29:58 DEBUG [...] 22:29:58 DEBUG [...] 22:29:58 DEBUG [...] 22:29:58 DEBUG }, 22:29:58 DEBUG }, 22:29:58 DEBUG }, 22:29:58 DEBUG [...] 22:29:58 DEBUG }, 22:29:58 DEBUG }, 22:29:58 DEBUG }, 22:29:58 DEBUG }, 22:29:58 DEBUG }, 22:29:58 DEBUG }, 22:29:58 DEBUG [...] 22:29:58 DEBUG [...] 22:29:58 DEBUG [...] 22:29:58 DEBUG [...] 22:29:58 DEBUG 22:29:58 DEBUG [...] 22:29:58 DEBUG [...] 22:29:58 DEBUG [...] 22:29:58 DEBUG 22:29:58 DEBUG }, 22:29:58 DEBUG 22:29:58 DEBUG "value": [ 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG "value": [ 22:29:58 DEBUG "value": [ 22:29:58 DEBUG "value": [ 22:29:58 DEBUG 22:29:58 DEBUG "value": [ 22:29:58 DEBUG "value": [ 22:29:58 DEBUG "value": [ 22:29:58 DEBUG "value": [ 22:29:58 DEBUG "value": [ 22:29:58 DEBUG "value": [ 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG [...] 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG "value": [ 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG "pod": "end-user-ui-58878d7668-zg4h2" 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG "instance": "10.142.0.43:9100" 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG }, 22:29:58 DEBUG 22:29:58 DEBUG "pod": "am-555f4db595-kzxsp" 22:29:58 DEBUG 22:29:58 DEBUG "pod": "end-user-ui-58878d7668-zg4h2" 22:29:58 DEBUG "pod": "ds-idrepo-0" 22:29:58 DEBUG "pod": "am-555f4db595-kzxsp" 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:29:58 DEBUG "pod": "end-user-ui-58878d7668-zg4h2" 22:29:58 DEBUG "instance": "10.142.0.43:9100" 22:29:58 DEBUG 22:29:58 DEBUG }, 22:29:58 DEBUG "pod": "am-555f4db595-kzxsp" 22:29:58 DEBUG "pod": "ds-idrepo-0" 22:29:58 DEBUG "pod": "am-555f4db595-kzxsp" 22:29:58 DEBUG "value": [ 22:29:58 DEBUG 22:29:58 DEBUG }, 22:29:58 DEBUG [...] 22:29:58 DEBUG }, 22:29:58 DEBUG }, 22:29:58 DEBUG }, 22:29:58 DEBUG [...] 22:29:58 DEBUG [...] 22:29:58 DEBUG [...] 22:29:58 DEBUG }, 22:29:58 DEBUG [...] 22:29:58 DEBUG [...] 22:29:58 DEBUG [...] 22:29:58 DEBUG [...] 22:29:58 DEBUG [...] 22:29:58 DEBUG [...] 22:29:58 DEBUG }, 22:29:58 DEBUG }, 22:29:58 DEBUG }, 22:29:58 DEBUG "pod": "am-555f4db595-kzxsp" 22:29:58 DEBUG "value": [ 22:29:58 DEBUG }, 22:29:58 DEBUG }, 22:29:58 DEBUG }, 22:29:58 DEBUG 1713302997, 22:29:58 DEBUG [...] 22:29:58 DEBUG "value": [ 22:29:58 DEBUG 22:29:58 DEBUG "value": [ 22:29:58 DEBUG "value": [ 22:29:58 DEBUG "value": [ 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG "value": [ 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG "value": [ 22:29:58 DEBUG "value": [ 22:29:58 DEBUG "value": [ 22:29:58 DEBUG }, 22:29:58 DEBUG 1713302997, 22:29:58 DEBUG "value": [ 22:29:58 DEBUG "value": [ 22:29:58 DEBUG "value": [ 22:29:58 DEBUG "0" 22:29:58 DEBUG 22:29:58 DEBUG 1713302997, 22:29:58 DEBUG 22:29:58 DEBUG 1713302997, 22:29:58 DEBUG 1713302997, 22:29:58 DEBUG 1713302997, 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 1713302997, 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 1713302997, 22:29:58 DEBUG 1713302997, 22:29:58 DEBUG 1713302997, 22:29:58 DEBUG "value": [ 22:29:58 DEBUG "45525976.56666666" 22:29:58 DEBUG 1713302997, 22:29:58 DEBUG 1713302997, 22:29:58 DEBUG 1713302997, 22:29:58 DEBUG ] 22:29:58 DEBUG 22:29:58 DEBUG "1997.7333333333333" 22:29:58 DEBUG 22:29:58 DEBUG "0" 22:29:58 DEBUG "0" 22:29:58 DEBUG "1997.6" 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG "6.302160233505236" 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG 22:29:58 DEBUG "654557184" 22:29:58 DEBUG "0" 22:29:58 DEBUG "0.00046666666666700016" 22:29:58 DEBUG 1713302997, 22:29:58 DEBUG ] 22:29:58 DEBUG "0" 22:29:58 DEBUG "0" 22:29:58 DEBUG "0.024" 22:29:58 DEBUG } 22:29:58 DEBUG 22:29:58 DEBUG ] 22:29:58 DEBUG "pod": "end-user-ui-58878d7668-zg4h2" 22:29:58 DEBUG ] 22:29:58 DEBUG ] 22:29:58 DEBUG ] 22:29:58 DEBUG "pod": "ds-idrepo-0" 22:29:58 DEBUG "instance": "10.142.0.43:9100" 22:29:58 DEBUG "pod": "ds-idrepo-0" 22:29:58 DEBUG ] 22:29:58 DEBUG "instance": "10.142.0.43:9100" 22:29:58 DEBUG }, 22:29:58 DEBUG "pod": "ds-idrepo-0" 22:29:58 DEBUG "pod": "am-555f4db595-kzxsp" 22:29:58 DEBUG "pod": "am-555f4db595-kzxsp" 22:29:58 DEBUG "instance": "10.142.0.43:9100" 22:29:58 DEBUG ] 22:29:58 DEBUG ] 22:29:58 DEBUG ] 22:29:58 DEBUG "0" 22:29:58 DEBUG } 22:29:58 DEBUG ] 22:29:58 DEBUG ] 22:29:58 DEBUG } 22:29:58 DEBUG ] 22:29:58 DEBUG "pod": "ds-cts-2" 22:29:58 DEBUG } 22:29:58 DEBUG }, 22:29:58 DEBUG } 22:29:58 DEBUG } 22:29:58 DEBUG } 22:29:58 DEBUG }, 22:29:58 DEBUG }, 22:29:58 DEBUG }, 22:29:58 DEBUG } 22:29:58 DEBUG "pod": "ds-idrepo-0" 22:29:58 DEBUG "value": [ 22:29:58 DEBUG }, 22:29:58 DEBUG }, 22:29:58 DEBUG }, 22:29:58 DEBUG }, 22:29:58 DEBUG } 22:29:58 DEBUG } 22:29:58 DEBUG ] 22:29:58 DEBUG ] 22:29:58 DEBUG ] 22:29:58 DEBUG } 22:29:58 DEBUG ] 22:29:58 DEBUG ] 22:29:58 DEBUG } 22:29:58 DEBUG }, 22:29:58 DEBUG "value": [ 22:29:58 DEBUG "value": [ 22:29:58 DEBUG ] 22:29:58 DEBUG ] 22:29:58 DEBUG ] 22:29:58 DEBUG "value": [ 22:29:58 DEBUG "value": [ 22:29:58 DEBUG "value": [ 22:29:58 DEBUG ] 22:29:58 DEBUG }, 22:29:58 DEBUG 1713302997, 22:29:58 DEBUG "value": [ 22:29:58 DEBUG "value": [ 22:29:58 DEBUG "value": [ 22:29:58 DEBUG "value": [ 22:29:58 DEBUG ] 22:29:58 DEBUG } 22:29:58 DEBUG } 22:29:58 DEBUG ] 22:29:58 DEBUG } 22:29:58 DEBUG ] 22:29:58 DEBUG } 22:29:58 DEBUG } 22:29:58 DEBUG } 22:29:58 DEBUG ] 22:29:58 DEBUG 1713302997, 22:29:58 DEBUG 1713302997, 22:29:58 DEBUG } 22:29:58 DEBUG } 22:29:58 DEBUG } 22:29:58 DEBUG 1713302997, 22:29:58 DEBUG 1713302997, 22:29:58 DEBUG 1713302997, 22:29:58 DEBUG } 22:29:58 DEBUG "value": [ 22:29:58 DEBUG "45534995.93333332" 22:29:58 DEBUG 1713302997, 22:29:58 DEBUG 1713302997, 22:29:58 DEBUG 1713302997, 22:29:58 DEBUG 1713302997, 22:29:58 DEBUG } 22:29:58 DEBUG } 22:29:58 DEBUG } 22:29:58 DEBUG } 22:29:58 DEBUG ] 22:29:58 DEBUG } 22:29:58 DEBUG ] 22:29:58 DEBUG } 22:29:58 DEBUG } 22:29:58 DEBUG "0" 22:29:58 DEBUG "0.00003113397450236968" 22:29:58 DEBUG } 22:29:58 DEBUG } 22:29:58 DEBUG } 22:29:58 DEBUG "0" 22:29:58 DEBUG "19958010880" 22:29:58 DEBUG "0" 22:29:58 DEBUG } 22:29:58 DEBUG 1713302997, 22:29:58 DEBUG ] 22:29:58 DEBUG "0" 22:29:58 DEBUG "1997.7333333333333" 22:29:58 DEBUG "2179913" 22:29:58 DEBUG "0.0003333333333333262" 22:29:58 DEBUG } 22:29:58 DEBUG } 22:29:58 DEBUG } 22:29:58 DEBUG } 22:29:58 DEBUG } 22:29:58 DEBUG } 22:29:58 DEBUG } 22:29:58 DEBUG ] 22:29:58 DEBUG ] 22:29:58 DEBUG ] 22:29:58 DEBUG ] 22:29:58 DEBUG ] 22:29:58 DEBUG "0" 22:29:58 DEBUG } 22:29:58 DEBUG ] 22:29:58 DEBUG ] 22:29:58 DEBUG ] 22:29:58 DEBUG ] 22:29:58 DEBUG } 22:29:58 DEBUG } 22:29:58 DEBUG } 22:29:58 DEBUG } 22:29:58 DEBUG } 22:29:58 DEBUG } 22:29:58 DEBUG } 22:29:58 DEBUG ] 22:29:58 DEBUG ] 22:29:58 DEBUG } 22:29:58 DEBUG } 22:29:58 DEBUG } 22:29:58 DEBUG } 22:29:58 DEBUG ] 22:29:58 DEBUG ] 22:29:58 DEBUG ] 22:29:58 DEBUG ] 22:29:58 DEBUG ] 22:29:58 DEBUG } 22:29:58 DEBUG } 22:29:58 DEBUG ] 22:29:58 DEBUG ] 22:29:58 DEBUG ] 22:29:58 DEBUG ] 22:29:58 DEBUG } 22:29:58 DEBUG } 22:29:58 DEBUG } 22:29:58 DEBUG } 22:29:58 DEBUG } 22:29:58 DEBUG ] 22:29:58 DEBUG } 22:29:58 DEBUG } 22:29:58 DEBUG } 22:29:58 DEBUG } 22:29:58 DEBUG } 22:29:58 DEBUG } 22:29:58 DEBUG } 22:29:58 DEBUG } 22:29:58 DEBUG } 22:29:58 DEBUG } 22:29:58 DEBUG } 22:29:58 DEBUG } 22:29:58 DEBUG } 22:29:58 DEBUG } 22:29:58 DEBUG } 22:29:58 DEBUG } 22:29:58 INFO 22:29:58 INFO [loop_until]: kubectl --namespace=xlou top node 22:29:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:29:58 INFO [loop_until]: OK (rc = 0) 22:29:58 DEBUG --- stdout --- 22:29:58 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 66m 0% 5096Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7670m 25% 14768Mi 13% gke-xlou-cdm-default-pool-1b0769c8-nhlf 9135m 30% 10818Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 77m 0% 2651Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 4763m 15% 6500Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 8570m 28% 16641Mi 14% gke-xlou-cdm-default-pool-1b0769c8-xnmp 60m 0% 3146Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 9451m 31% 10366Mi 9% gke-xlou-cdm-ds-0f601751-hmm7 4645m 15% 5591Mi 5% gke-xlou-cdm-ds-0f601751-j488 7648m 25% 14106Mi 12% gke-xlou-cdm-ds-0f601751-mc5d 10663m 35% 3344Mi 2% gke-xlou-cdm-ds-0f601751-nj6q 9621m 32% 10425Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 4808m 16% 7365Mi 6% gke-xlou-cdm-frontend-a8771548-i82n 6438m 40% 5950Mi 10% 22:29:58 DEBUG --- stderr --- 22:29:58 DEBUG 22:30:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713303055" 22:30:55 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303055" 22:30:56 ERROR [http_cmd]: ERROR 22:30:56 ERROR --- status code --- 22:30:56 ERROR http status code is 400 (expected [200, 401, 403, 404]) 22:30:56 ERROR --- http response --- 22:30:56 INFO [http_cmd]: http status code OK 22:30:56 DEBUG --- status code --- 22:30:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:30:56 DEBUG --- http response --- 22:30:56 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 22:30:56 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 22:30:56 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 22:30:57 INFO 22:30:57 INFO [loop_until]: kubectl --namespace=xlou top pods 22:30:57 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:30:57 INFO [loop_until]: OK (rc = 0) 22:30:57 DEBUG --- stdout --- 22:30:57 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 9Mi am-555f4db595-4lbss 9427m 8594Mi am-555f4db595-fz8sn 9428m 8605Mi am-555f4db595-kzxsp 9460m 8559Mi ds-cts-0 8129m 12142Mi ds-cts-1 7438m 12224Mi ds-cts-2 7566m 12187Mi ds-idrepo-0 4645m 5032Mi ds-idrepo-1 4349m 2850Mi ds-idrepo-2 4413m 3160Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 4m 1207Mi idm-b85f9d978-rl76f 4m 1225Mi lodemon-bc756c647-cr7x6 2m 63Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-7c9b667974-9mr2s 6350m 535Mi 22:30:57 DEBUG --- stderr --- 22:30:57 DEBUG 22:30:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303058" 22:30:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303058" 22:30:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303058" 22:30:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303058" 22:30:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303058" 22:30:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303058" 22:30:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713303058" 22:30:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713303058" 22:30:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303058" 22:30:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303058" 22:30:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303058" 22:30:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303058" 22:30:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713303058" 22:30:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303058" 22:30:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303058" 22:30:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713303058" 22:30:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303058" 22:30:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303058" 22:30:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303058" 22:30:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303058" 22:30:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303058" 22:30:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303058" 22:30:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303058" 22:30:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303058" 22:30:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303058" 22:30:58 INFO [http_cmd]: http status code OK 22:30:58 INFO [http_cmd]: http status code OK 22:30:58 INFO [http_cmd]: http status code OK 22:30:58 INFO [http_cmd]: http status code OK 22:30:58 INFO [http_cmd]: http status code OK 22:30:58 INFO [http_cmd]: http status code OK 22:30:58 INFO [http_cmd]: http status code OK 22:30:58 INFO [http_cmd]: http status code OK 22:30:58 INFO [http_cmd]: http status code OK 22:30:58 INFO [http_cmd]: http status code OK 22:30:58 INFO [http_cmd]: http status code OK 22:30:58 INFO [http_cmd]: http status code OK 22:30:58 INFO [http_cmd]: http status code OK 22:30:58 INFO [http_cmd]: http status code OK 22:30:58 INFO [http_cmd]: http status code OK 22:30:58 INFO [http_cmd]: http status code OK 22:30:58 DEBUG --- status code --- 22:30:58 INFO [http_cmd]: http status code OK 22:30:58 INFO [http_cmd]: http status code OK 22:30:58 INFO [http_cmd]: http status code OK 22:30:58 DEBUG --- status code --- 22:30:58 DEBUG --- status code --- 22:30:58 DEBUG --- status code --- 22:30:58 INFO [http_cmd]: http status code OK 22:30:58 DEBUG --- status code --- 22:30:58 DEBUG --- status code --- 22:30:58 INFO [http_cmd]: http status code OK 22:30:58 DEBUG --- status code --- 22:30:58 DEBUG --- status code --- 22:30:58 DEBUG --- status code --- 22:30:58 DEBUG --- status code --- 22:30:58 DEBUG --- status code --- 22:30:58 DEBUG --- status code --- 22:30:58 DEBUG --- status code --- 22:30:58 INFO [http_cmd]: http status code OK 22:30:58 INFO [http_cmd]: http status code OK 22:30:58 DEBUG --- status code --- 22:30:58 DEBUG --- status code --- 22:30:58 DEBUG --- status code --- 22:30:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:30:58 DEBUG --- status code --- 22:30:58 DEBUG --- status code --- 22:30:58 DEBUG --- status code --- 22:30:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:30:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:30:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:30:58 DEBUG --- status code --- 22:30:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:30:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:30:58 DEBUG --- status code --- 22:30:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:30:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:30:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:30:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:30:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:30:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:30:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:30:58 DEBUG --- status code --- 22:30:58 DEBUG --- status code --- 22:30:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:30:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:30:58 DEBUG --- http response --- 22:30:58 DEBUG --- http response --- 22:30:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:30:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:30:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:30:58 DEBUG --- http response --- 22:30:58 DEBUG --- http response --- 22:30:58 DEBUG --- http response --- 22:30:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:30:58 DEBUG --- http response --- 22:30:58 DEBUG --- http response --- 22:30:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:30:58 DEBUG --- http response --- 22:30:58 DEBUG --- http response --- 22:30:58 INFO [http_cmd]: http status code OK 22:30:58 DEBUG --- http response --- 22:30:58 INFO [http_cmd]: http status code OK 22:30:58 DEBUG --- http response --- 22:30:58 DEBUG --- http response --- 22:30:58 DEBUG --- http response --- 22:30:58 DEBUG --- http response --- 22:30:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:30:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:30:58 DEBUG --- http response --- 22:30:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:30:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:30:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:30:58 DEBUG --- http response --- 22:30:58 DEBUG --- http response --- 22:30:58 DEBUG --- http response --- 22:30:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:30:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:30:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:30:58 DEBUG --- http response --- 22:30:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:30:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:30:58 DEBUG --- http response --- 22:30:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:30:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:30:58 DEBUG --- status code --- 22:30:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:30:58 DEBUG --- status code --- 22:30:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:30:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:30:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:30:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:30:58 DEBUG --- http response --- 22:30:58 DEBUG --- http response --- 22:30:58 DEBUG --- http response --- 22:30:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:30:58 DEBUG ----- output ----- 22:30:58 DEBUG ----- output ----- 22:30:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:30:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:30:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:30:58 DEBUG ----- output ----- 22:30:58 DEBUG ----- output ----- 22:30:58 DEBUG ----- output ----- 22:30:58 DEBUG ----- output ----- 22:30:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:30:58 DEBUG ----- output ----- 22:30:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:30:58 DEBUG ----- output ----- 22:30:58 DEBUG ----- output ----- 22:30:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:30:58 DEBUG ----- output ----- 22:30:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:30:58 DEBUG ----- output ----- 22:30:58 DEBUG ----- output ----- 22:30:58 DEBUG ----- output ----- 22:30:58 DEBUG ----- output ----- 22:30:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:30:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:30:58 DEBUG ----- output ----- 22:30:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:30:58 DEBUG { 22:30:58 DEBUG { 22:30:58 DEBUG ----- output ----- 22:30:58 DEBUG ----- output ----- 22:30:58 DEBUG ----- output ----- 22:30:58 DEBUG { 22:30:58 DEBUG { 22:30:58 DEBUG { 22:30:58 DEBUG { 22:30:58 DEBUG ----- output ----- 22:30:58 DEBUG { 22:30:58 DEBUG ----- output ----- 22:30:58 DEBUG { 22:30:58 DEBUG { 22:30:58 DEBUG --- http response --- 22:30:58 DEBUG { 22:30:58 DEBUG --- http response --- 22:30:58 DEBUG { 22:30:58 DEBUG { 22:30:58 DEBUG { 22:30:58 DEBUG { 22:30:58 DEBUG ----- output ----- 22:30:58 DEBUG ----- output ----- 22:30:58 DEBUG { 22:30:58 DEBUG ----- output ----- 22:30:58 DEBUG "status": "success", 22:30:58 DEBUG "status": "success", 22:30:58 DEBUG { 22:30:58 DEBUG { 22:30:58 DEBUG { 22:30:58 DEBUG "status": "success", 22:30:58 DEBUG "status": "success", 22:30:58 DEBUG "status": "success", 22:30:58 DEBUG "status": "success", 22:30:58 DEBUG { 22:30:58 DEBUG "status": "success", 22:30:58 DEBUG { 22:30:58 DEBUG "status": "success", 22:30:58 DEBUG "status": "success", 22:30:58 DEBUG "status": "success", 22:30:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:30:58 DEBUG "status": "success", 22:30:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:30:58 DEBUG "status": "success", 22:30:58 DEBUG "status": "success", 22:30:58 DEBUG "status": "success", 22:30:58 DEBUG { 22:30:58 DEBUG { 22:30:58 DEBUG "status": "success", 22:30:58 DEBUG { 22:30:58 DEBUG "data": { 22:30:58 DEBUG "data": { 22:30:58 DEBUG "status": "success", 22:30:58 DEBUG "status": "success", 22:30:58 DEBUG "status": "success", 22:30:58 DEBUG "data": { 22:30:58 DEBUG "data": { 22:30:58 DEBUG "data": { 22:30:58 DEBUG "data": { 22:30:58 DEBUG "status": "success", 22:30:58 DEBUG "data": { 22:30:58 DEBUG "status": "success", 22:30:58 DEBUG "data": { 22:30:58 DEBUG "data": { 22:30:58 DEBUG "data": { 22:30:58 DEBUG ----- output ----- 22:30:58 DEBUG "data": { 22:30:58 DEBUG ----- output ----- 22:30:58 DEBUG "data": { 22:30:58 DEBUG "data": { 22:30:58 DEBUG "data": { 22:30:58 DEBUG "status": "success", 22:30:58 DEBUG "status": "success", 22:30:58 DEBUG "data": { 22:30:58 DEBUG "status": "success", 22:30:58 DEBUG "resultType": "vector", 22:30:58 DEBUG "resultType": "vector", 22:30:58 DEBUG "data": { 22:30:58 DEBUG "data": { 22:30:58 DEBUG "data": { 22:30:58 DEBUG "resultType": "vector", 22:30:58 DEBUG "resultType": "vector", 22:30:58 DEBUG "resultType": "vector", 22:30:58 DEBUG "resultType": "vector", 22:30:58 DEBUG "data": { 22:30:58 DEBUG "resultType": "vector", 22:30:58 DEBUG "data": { 22:30:58 DEBUG "resultType": "vector", 22:30:58 DEBUG "resultType": "vector", 22:30:58 DEBUG "resultType": "vector", 22:30:58 DEBUG { 22:30:58 DEBUG "resultType": "vector", 22:30:58 DEBUG { 22:30:58 DEBUG "resultType": "vector", 22:30:58 DEBUG "resultType": "vector", 22:30:58 DEBUG "resultType": "vector", 22:30:58 DEBUG "data": { 22:30:58 DEBUG "data": { 22:30:58 DEBUG "resultType": "vector", 22:30:58 DEBUG "data": { 22:30:58 DEBUG "result": [ 22:30:58 DEBUG "result": [ 22:30:58 DEBUG "resultType": "vector", 22:30:58 DEBUG "resultType": "vector", 22:30:58 DEBUG "resultType": "vector", 22:30:58 DEBUG "result": [ 22:30:58 DEBUG "result": [ 22:30:58 DEBUG "result": [ 22:30:58 DEBUG "result": [ 22:30:58 DEBUG "resultType": "vector", 22:30:58 DEBUG "result": [ 22:30:58 DEBUG "resultType": "vector", 22:30:58 DEBUG "result": [ 22:30:58 DEBUG "result": [ 22:30:58 DEBUG "result": [ 22:30:58 DEBUG "status": "success", 22:30:58 DEBUG "result": [ 22:30:58 DEBUG "status": "success", 22:30:58 DEBUG "result": [ 22:30:58 DEBUG "result": [ 22:30:58 DEBUG "result": [ 22:30:58 DEBUG "resultType": "vector", 22:30:58 DEBUG "resultType": "vector", 22:30:58 DEBUG "result": [ 22:30:58 DEBUG "resultType": "vector", 22:30:58 DEBUG { 22:30:58 DEBUG { 22:30:58 DEBUG "result": [ 22:30:58 DEBUG "result": [ 22:30:58 DEBUG "result": [ 22:30:58 DEBUG { 22:30:58 DEBUG { 22:30:58 DEBUG { 22:30:58 DEBUG { 22:30:58 DEBUG "result": [ 22:30:58 DEBUG { 22:30:58 DEBUG "result": [ 22:30:58 DEBUG { 22:30:58 DEBUG { 22:30:58 DEBUG { 22:30:58 DEBUG "data": { 22:30:58 DEBUG { 22:30:58 DEBUG "data": { 22:30:58 DEBUG { 22:30:58 DEBUG { 22:30:58 DEBUG { 22:30:58 DEBUG "result": [ 22:30:58 DEBUG "result": [ 22:30:58 DEBUG { 22:30:58 DEBUG "result": [ 22:30:58 DEBUG "metric": { 22:30:58 DEBUG "metric": { 22:30:58 DEBUG { 22:30:58 DEBUG { 22:30:58 DEBUG { 22:30:58 DEBUG "metric": { 22:30:58 DEBUG "metric": { 22:30:58 DEBUG "metric": { 22:30:58 DEBUG "metric": { 22:30:58 DEBUG { 22:30:58 DEBUG "metric": { 22:30:58 DEBUG { 22:30:58 DEBUG "metric": { 22:30:58 DEBUG "metric": { 22:30:58 DEBUG "metric": { 22:30:58 DEBUG "resultType": "vector", 22:30:58 DEBUG "metric": { 22:30:58 DEBUG "resultType": "vector", 22:30:58 DEBUG "metric": { 22:30:58 DEBUG "metric": { 22:30:58 DEBUG "metric": { 22:30:58 DEBUG { 22:30:58 DEBUG { 22:30:58 DEBUG "metric": { 22:30:58 DEBUG { 22:30:58 DEBUG "instance": "10.142.0.107:9100" 22:30:58 DEBUG "pod": "ds-idrepo-2" 22:30:58 DEBUG "metric": { 22:30:58 DEBUG "metric": { 22:30:58 DEBUG "metric": { 22:30:58 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:30:58 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:30:58 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:30:58 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:30:58 DEBUG "metric": { 22:30:58 DEBUG "pod": "ds-idrepo-2" 22:30:58 DEBUG "metric": { 22:30:58 DEBUG "pod": "am-555f4db595-fz8sn" 22:30:58 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:30:58 DEBUG "pod": "am-555f4db595-fz8sn" 22:30:58 DEBUG "result": [ 22:30:58 DEBUG "instance": "10.142.0.107:9100" 22:30:58 DEBUG "result": [ 22:30:58 DEBUG "pod": "am-555f4db595-fz8sn" 22:30:58 DEBUG "pod": "am-555f4db595-fz8sn" 22:30:58 DEBUG "pod": "am-555f4db595-fz8sn" 22:30:58 DEBUG "metric": { 22:30:58 DEBUG "metric": { 22:30:58 DEBUG "instance": "10.142.0.107:9100" 22:30:58 DEBUG }, 22:30:58 DEBUG "value": [ 22:30:58 DEBUG }, 22:30:58 DEBUG "instance": "10.142.0.107:9100" 22:30:58 DEBUG "instance": "10.142.0.107:9100" 22:30:58 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:30:58 DEBUG }, 22:30:58 DEBUG }, 22:30:58 DEBUG }, 22:30:58 DEBUG }, 22:30:58 DEBUG "pod": "am-555f4db595-fz8sn" 22:30:58 DEBUG }, 22:30:58 DEBUG "pod": "am-555f4db595-fz8sn" 22:30:58 DEBUG }, 22:30:58 DEBUG }, 22:30:58 DEBUG }, 22:30:58 DEBUG { 22:30:58 DEBUG }, 22:30:58 DEBUG { 22:30:58 DEBUG }, 22:30:58 DEBUG }, 22:30:58 DEBUG }, 22:30:58 DEBUG "pod": "ds-idrepo-2" 22:30:58 DEBUG "pod": "ds-idrepo-2" 22:30:58 DEBUG "metric": { 22:30:58 DEBUG 22:30:58 DEBUG "value": [ 22:30:58 DEBUG }, 22:30:58 DEBUG }, 22:30:58 DEBUG }, 22:30:58 DEBUG "value": [ 22:30:58 DEBUG "value": [ 22:30:58 DEBUG "value": [ 22:30:58 DEBUG "value": [ 22:30:58 DEBUG }, 22:30:58 DEBUG "value": [ 22:30:58 DEBUG }, 22:30:58 DEBUG }, 22:30:58 DEBUG "value": [ 22:30:58 DEBUG "value": [ 22:30:58 DEBUG "value": [ 22:30:58 DEBUG "metric": { 22:30:58 DEBUG "value": [ 22:30:58 DEBUG "metric": { 22:30:58 DEBUG "value": [ 22:30:58 DEBUG "value": [ 22:30:58 DEBUG "value": [ 22:30:58 DEBUG }, 22:30:58 DEBUG }, 22:30:58 DEBUG "pod": "ds-idrepo-2" 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG "value": [ 22:30:58 DEBUG "value": [ 22:30:58 DEBUG "value": [ 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG "value": [ 22:30:58 DEBUG 22:30:58 DEBUG "value": [ 22:30:58 DEBUG "value": [ 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG "pod": "ds-idrepo-2" 22:30:58 DEBUG 22:30:58 DEBUG "pod": "ds-idrepo-2" 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG "value": [ 22:30:58 DEBUG "value": [ 22:30:58 DEBUG }, 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG }, 22:30:58 DEBUG 22:30:58 DEBUG }, 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG "value": [ 22:30:58 DEBUG [...] 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG "value": [ 22:30:58 DEBUG 22:30:58 DEBUG "value": [ 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG [...] 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG [...] 22:30:58 DEBUG [...] 22:30:58 DEBUG [...] 22:30:58 DEBUG [...] 22:30:58 DEBUG 22:30:58 DEBUG [...] 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG [...] 22:30:58 DEBUG [...] 22:30:58 DEBUG [...] 22:30:58 DEBUG 22:30:58 DEBUG [...] 22:30:58 DEBUG 22:30:58 DEBUG [...] 22:30:58 DEBUG [...] 22:30:58 DEBUG [...] 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG [...] 22:30:58 DEBUG [...] 22:30:58 DEBUG [...] 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG [...] 22:30:58 DEBUG 22:30:58 DEBUG [...] 22:30:58 DEBUG [...] 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG [...] 22:30:58 DEBUG [...] 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG [...] 22:30:58 DEBUG "instance": "10.142.0.43:9100" 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG [...] 22:30:58 DEBUG 22:30:58 DEBUG [...] 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG }, 22:30:58 DEBUG "instance": "10.142.0.43:9100" 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG "pod": "am-555f4db595-4lbss" 22:30:58 DEBUG "pod": "am-555f4db595-4lbss" 22:30:58 DEBUG "pod": "am-555f4db595-4lbss" 22:30:58 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:30:58 DEBUG 22:30:58 DEBUG "pod": "ds-idrepo-0" 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG "pod": "am-555f4db595-kzxsp" 22:30:58 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:30:58 DEBUG "pod": "am-555f4db595-kzxsp" 22:30:58 DEBUG 22:30:58 DEBUG "instance": "10.142.0.43:9100" 22:30:58 DEBUG 22:30:58 DEBUG "pod": "am-555f4db595-kzxsp" 22:30:58 DEBUG "pod": "am-555f4db595-kzxsp" 22:30:58 DEBUG "pod": "am-555f4db595-kzxsp" 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG 22:30:58 DEBUG "value": [ 22:30:58 DEBUG }, 22:30:58 DEBUG "instance": "10.142.0.43:9100" 22:30:58 DEBUG "instance": "10.142.0.43:9100" 22:30:58 DEBUG "pod": "am-555f4db595-4lbss" 22:30:58 DEBUG }, 22:30:58 DEBUG }, 22:30:58 DEBUG }, 22:30:58 DEBUG }, 22:30:58 DEBUG "pod": "am-555f4db595-kzxsp" 22:30:58 DEBUG }, 22:30:58 DEBUG "pod": "am-555f4db595-kzxsp" 22:30:58 DEBUG "pod": "ds-idrepo-0" 22:30:58 DEBUG }, 22:30:58 DEBUG }, 22:30:58 DEBUG }, 22:30:58 DEBUG 22:30:58 DEBUG }, 22:30:58 DEBUG 22:30:58 DEBUG }, 22:30:58 DEBUG }, 22:30:58 DEBUG }, 22:30:58 DEBUG "pod": "ds-idrepo-0" 22:30:58 DEBUG "pod": "ds-idrepo-0" 22:30:58 DEBUG 22:30:58 DEBUG 1713303058, 22:30:58 DEBUG "value": [ 22:30:58 DEBUG }, 22:30:58 DEBUG }, 22:30:58 DEBUG }, 22:30:58 DEBUG "value": [ 22:30:58 DEBUG "value": [ 22:30:58 DEBUG "value": [ 22:30:58 DEBUG "value": [ 22:30:58 DEBUG }, 22:30:58 DEBUG "value": [ 22:30:58 DEBUG }, 22:30:58 DEBUG }, 22:30:58 DEBUG "value": [ 22:30:58 DEBUG "value": [ 22:30:58 DEBUG "value": [ 22:30:58 DEBUG 22:30:58 DEBUG "value": [ 22:30:58 DEBUG 22:30:58 DEBUG "value": [ 22:30:58 DEBUG "value": [ 22:30:58 DEBUG "value": [ 22:30:58 DEBUG }, 22:30:58 DEBUG }, 22:30:58 DEBUG "pod": "ds-cts-2" 22:30:58 DEBUG "0.00040000000000001514" 22:30:58 DEBUG 1713303058, 22:30:58 DEBUG "value": [ 22:30:58 DEBUG "value": [ 22:30:58 DEBUG "value": [ 22:30:58 DEBUG 1713303058, 22:30:58 DEBUG 1713303058, 22:30:58 DEBUG 1713303058, 22:30:58 DEBUG 1713303058, 22:30:58 DEBUG "value": [ 22:30:58 DEBUG 1713303058, 22:30:58 DEBUG "value": [ 22:30:58 DEBUG "value": [ 22:30:58 DEBUG 1713303058, 22:30:58 DEBUG 1713303058, 22:30:58 DEBUG 1713303058, 22:30:58 DEBUG "pod": "ds-idrepo-0" 22:30:58 DEBUG 1713303058, 22:30:58 DEBUG "pod": "ds-idrepo-0" 22:30:58 DEBUG 1713303058, 22:30:58 DEBUG 1713303058, 22:30:58 DEBUG 1713303058, 22:30:58 DEBUG "value": [ 22:30:58 DEBUG "value": [ 22:30:58 DEBUG }, 22:30:58 DEBUG ] 22:30:58 DEBUG "45524334.166666664" 22:30:58 DEBUG 1713303058, 22:30:58 DEBUG 1713303058, 22:30:58 DEBUG 1713303058, 22:30:58 DEBUG "0" 22:30:58 DEBUG "252731044.8949833" 22:30:58 DEBUG "918.8749614424721" 22:30:58 DEBUG "6.32183745213949" 22:30:58 DEBUG 1713303058, 22:30:58 DEBUG "0" 22:30:58 DEBUG 1713303058, 22:30:58 DEBUG 1713303058, 22:30:58 DEBUG "0" 22:30:58 DEBUG "656547840" 22:30:58 DEBUG "1998.6333333333332" 22:30:58 DEBUG }, 22:30:58 DEBUG "45514997.3" 22:30:58 DEBUG }, 22:30:58 DEBUG "1998.6333333333332" 22:30:58 DEBUG "0.024" 22:30:58 DEBUG "0" 22:30:58 DEBUG 1713303058, 22:30:58 DEBUG 1713303058, 22:30:58 DEBUG "value": [ 22:30:58 DEBUG } 22:30:58 DEBUG ] 22:30:58 DEBUG "0" 22:30:58 DEBUG "19958699008" 22:30:58 DEBUG "9.290645607245997" 22:30:58 DEBUG ] 22:30:58 DEBUG ] 22:30:58 DEBUG ] 22:30:58 DEBUG ] 22:30:58 DEBUG "2300020.5" 22:30:58 DEBUG ] 22:30:58 DEBUG "1998.7666666666667" 22:30:58 DEBUG "0" 22:30:58 DEBUG ] 22:30:58 DEBUG ] 22:30:58 DEBUG ] 22:30:58 DEBUG "value": [ 22:30:58 DEBUG ] 22:30:58 DEBUG "value": [ 22:30:58 DEBUG ] 22:30:58 DEBUG ] 22:30:58 DEBUG ] 22:30:58 DEBUG "0" 22:30:58 DEBUG "0" 22:30:58 DEBUG 1713303058, 22:30:58 DEBUG ] 22:30:58 DEBUG } 22:30:58 DEBUG ] 22:30:58 DEBUG ] 22:30:58 DEBUG ] 22:30:58 DEBUG } 22:30:58 DEBUG } 22:30:58 DEBUG } 22:30:58 DEBUG } 22:30:58 DEBUG ] 22:30:58 DEBUG } 22:30:58 DEBUG ] 22:30:58 DEBUG ] 22:30:58 DEBUG } 22:30:58 DEBUG } 22:30:58 DEBUG } 22:30:58 DEBUG 1713303058, 22:30:58 DEBUG } 22:30:58 DEBUG 1713303058, 22:30:58 DEBUG } 22:30:58 DEBUG } 22:30:58 DEBUG } 22:30:58 DEBUG ] 22:30:58 DEBUG ] 22:30:58 DEBUG "0" 22:30:58 DEBUG } 22:30:58 DEBUG ] 22:30:58 DEBUG } 22:30:58 DEBUG } 22:30:58 DEBUG } 22:30:58 DEBUG ] 22:30:58 DEBUG ] 22:30:58 DEBUG ] 22:30:58 DEBUG ] 22:30:58 DEBUG } 22:30:58 DEBUG ] 22:30:58 DEBUG } 22:30:58 DEBUG } 22:30:58 DEBUG ] 22:30:58 DEBUG ] 22:30:58 DEBUG ] 22:30:58 DEBUG "0" 22:30:58 DEBUG ] 22:30:58 DEBUG "0" 22:30:58 DEBUG ] 22:30:58 DEBUG ] 22:30:58 DEBUG ] 22:30:58 DEBUG } 22:30:58 DEBUG } 22:30:58 DEBUG ] 22:30:58 DEBUG } 22:30:58 DEBUG } 22:30:58 DEBUG ] 22:30:58 DEBUG ] 22:30:58 DEBUG ] 22:30:58 DEBUG } 22:30:58 DEBUG } 22:30:58 DEBUG } 22:30:58 DEBUG } 22:30:58 DEBUG ] 22:30:58 DEBUG } 22:30:58 DEBUG ] 22:30:58 DEBUG ] 22:30:58 DEBUG } 22:30:58 DEBUG } 22:30:58 DEBUG } 22:30:58 DEBUG } 22:30:58 DEBUG } 22:30:58 DEBUG ] 22:30:58 DEBUG } 22:30:58 DEBUG } 22:30:58 DEBUG } 22:30:58 DEBUG ] 22:30:58 DEBUG ] 22:30:58 DEBUG } 22:30:58 DEBUG } 22:30:58 DEBUG } 22:30:58 DEBUG } 22:30:58 DEBUG } 22:30:58 DEBUG } 22:30:58 DEBUG } 22:30:58 DEBUG } 22:30:58 DEBUG } 22:30:58 DEBUG } 22:30:58 DEBUG } 22:30:58 DEBUG } 22:30:58 DEBUG } 22:30:58 DEBUG } 22:30:58 DEBUG } 22:30:58 DEBUG ] 22:30:58 DEBUG } 22:30:58 DEBUG } 22:30:58 DEBUG } 22:30:58 DEBUG } 22:30:58 DEBUG } 22:30:58 DEBUG } 22:30:58 DEBUG } 22:30:58 DEBUG ] 22:30:58 DEBUG } 22:30:58 DEBUG } 22:30:58 DEBUG } 22:30:58 DEBUG } 22:30:58 DEBUG } 22:30:58 DEBUG } 22:30:58 DEBUG } 22:30:58 DEBUG ] 22:30:58 DEBUG } 22:30:58 DEBUG } 22:30:58 DEBUG } 22:30:58 DEBUG ] 22:30:58 DEBUG } 22:30:58 DEBUG } 22:30:58 DEBUG } 22:30:58 DEBUG } 22:30:58 DEBUG } 22:30:58 INFO 22:30:58 INFO [loop_until]: kubectl --namespace=xlou top node 22:30:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:30:58 INFO [loop_until]: OK (rc = 0) 22:30:58 DEBUG --- stdout --- 22:30:58 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 63m 0% 5105Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7609m 25% 14784Mi 13% gke-xlou-cdm-default-pool-1b0769c8-nhlf 9476m 31% 10822Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 85m 0% 2646Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 4771m 15% 6495Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 8550m 28% 16659Mi 14% gke-xlou-cdm-default-pool-1b0769c8-xnmp 59m 0% 3158Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 9465m 31% 10376Mi 9% gke-xlou-cdm-ds-0f601751-hmm7 4643m 15% 5591Mi 5% gke-xlou-cdm-ds-0f601751-j488 7505m 25% 14122Mi 12% gke-xlou-cdm-ds-0f601751-mc5d 10652m 35% 3348Mi 2% gke-xlou-cdm-ds-0f601751-nj6q 9649m 32% 10413Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 4784m 16% 7353Mi 6% gke-xlou-cdm-frontend-a8771548-i82n 6437m 40% 5950Mi 10% 22:30:58 DEBUG --- stderr --- 22:30:58 DEBUG 22:31:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713303116" 22:31:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303116" 22:31:56 ERROR [http_cmd]: ERROR 22:31:56 INFO [http_cmd]: http status code OK 22:31:56 ERROR --- status code --- 22:31:56 DEBUG --- status code --- 22:31:56 ERROR http status code is 400 (expected [200, 401, 403, 404]) 22:31:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:31:56 ERROR --- http response --- 22:31:56 DEBUG --- http response --- 22:31:56 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 22:31:56 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 22:31:56 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 22:31:57 INFO 22:31:57 INFO [loop_until]: kubectl --namespace=xlou top pods 22:31:57 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:31:57 INFO [loop_until]: OK (rc = 0) 22:31:57 DEBUG --- stdout --- 22:31:57 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 9Mi am-555f4db595-4lbss 9131m 8594Mi am-555f4db595-fz8sn 9057m 8606Mi am-555f4db595-kzxsp 8909m 8558Mi ds-cts-0 8040m 12139Mi ds-cts-1 7351m 12230Mi ds-cts-2 7246m 12193Mi ds-idrepo-0 4843m 5032Mi ds-idrepo-1 4456m 2850Mi ds-idrepo-2 4542m 3160Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 4m 1207Mi idm-b85f9d978-rl76f 3m 1225Mi lodemon-bc756c647-cr7x6 2m 63Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-7c9b667974-9mr2s 6241m 537Mi 22:31:57 DEBUG --- stderr --- 22:31:57 DEBUG 22:31:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303118" 22:31:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303118" 22:31:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303118" 22:31:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303118" 22:31:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303118" 22:31:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713303118" 22:31:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713303118" 22:31:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303118" 22:31:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303118" 22:31:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713303118" 22:31:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303118" 22:31:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303118" 22:31:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303118" 22:31:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303118" 22:31:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303118" 22:31:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303118" 22:31:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303118" 22:31:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303118" 22:31:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303118" 22:31:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303118" 22:31:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303118" 22:31:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713303118" 22:31:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303118" 22:31:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303118" 22:31:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303118" 22:31:58 INFO [http_cmd]: http status code OK 22:31:58 INFO [http_cmd]: http status code OK 22:31:58 INFO [http_cmd]: http status code OK 22:31:58 INFO [http_cmd]: http status code OK 22:31:58 INFO [http_cmd]: http status code OK 22:31:58 INFO [http_cmd]: http status code OK 22:31:58 INFO [http_cmd]: http status code OK 22:31:58 INFO [http_cmd]: http status code OK 22:31:58 INFO [http_cmd]: http status code OK 22:31:58 INFO [http_cmd]: http status code OK 22:31:58 INFO [http_cmd]: http status code OK 22:31:58 INFO [http_cmd]: http status code OK 22:31:58 INFO [http_cmd]: http status code OK 22:31:58 INFO [http_cmd]: http status code OK 22:31:58 INFO [http_cmd]: http status code OK 22:31:58 INFO [http_cmd]: http status code OK 22:31:58 DEBUG --- status code --- 22:31:58 INFO [http_cmd]: http status code OK 22:31:58 INFO [http_cmd]: http status code OK 22:31:58 INFO [http_cmd]: http status code OK 22:31:58 DEBUG --- status code --- 22:31:58 DEBUG --- status code --- 22:31:58 DEBUG --- status code --- 22:31:58 DEBUG --- status code --- 22:31:58 DEBUG --- status code --- 22:31:58 INFO [http_cmd]: http status code OK 22:31:58 DEBUG --- status code --- 22:31:58 INFO [http_cmd]: http status code OK 22:31:58 INFO [http_cmd]: http status code OK 22:31:58 DEBUG --- status code --- 22:31:58 INFO [http_cmd]: http status code OK 22:31:58 DEBUG --- status code --- 22:31:58 DEBUG --- status code --- 22:31:58 DEBUG --- status code --- 22:31:58 DEBUG --- status code --- 22:31:58 DEBUG --- status code --- 22:31:58 DEBUG --- status code --- 22:31:58 DEBUG --- status code --- 22:31:58 DEBUG --- status code --- 22:31:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:31:58 DEBUG --- status code --- 22:31:58 DEBUG --- status code --- 22:31:58 DEBUG --- status code --- 22:31:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:31:58 INFO [http_cmd]: http status code OK 22:31:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:31:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:31:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:31:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:31:58 DEBUG --- status code --- 22:31:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:31:58 DEBUG --- status code --- 22:31:58 DEBUG --- status code --- 22:31:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:31:58 DEBUG --- status code --- 22:31:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:31:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:31:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:31:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:31:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:31:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:31:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:31:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:31:58 DEBUG --- http response --- 22:31:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:31:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:31:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:31:58 DEBUG --- http response --- 22:31:58 DEBUG --- status code --- 22:31:58 DEBUG --- http response --- 22:31:58 DEBUG --- http response --- 22:31:58 DEBUG --- http response --- 22:31:58 DEBUG --- http response --- 22:31:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:31:58 DEBUG --- http response --- 22:31:58 INFO [http_cmd]: http status code OK 22:31:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:31:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:31:58 DEBUG --- http response --- 22:31:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:31:58 DEBUG --- http response --- 22:31:58 DEBUG --- http response --- 22:31:58 DEBUG --- http response --- 22:31:58 DEBUG --- http response --- 22:31:58 DEBUG --- http response --- 22:31:58 DEBUG --- http response --- 22:31:58 DEBUG --- http response --- 22:31:58 DEBUG --- http response --- 22:31:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:31:58 DEBUG --- http response --- 22:31:58 DEBUG --- http response --- 22:31:58 DEBUG --- http response --- 22:31:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:31:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:31:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:31:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:31:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:31:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:31:58 DEBUG --- http response --- 22:31:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:31:58 DEBUG --- status code --- 22:31:58 DEBUG --- http response --- 22:31:58 DEBUG --- http response --- 22:31:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:31:58 DEBUG --- http response --- 22:31:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:31:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:31:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:31:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:31:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:31:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:31:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:31:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:31:58 DEBUG ----- output ----- 22:31:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:31:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:31:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:31:58 DEBUG ----- output ----- 22:31:58 DEBUG --- http response --- 22:31:58 DEBUG ----- output ----- 22:31:58 DEBUG ----- output ----- 22:31:58 DEBUG ----- output ----- 22:31:58 DEBUG ----- output ----- 22:31:58 DEBUG ----- output ----- 22:31:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:31:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:31:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:31:58 DEBUG ----- output ----- 22:31:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:31:58 DEBUG ----- output ----- 22:31:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:31:58 DEBUG ----- output ----- 22:31:58 DEBUG ----- output ----- 22:31:58 DEBUG ----- output ----- 22:31:58 DEBUG ----- output ----- 22:31:58 DEBUG ----- output ----- 22:31:58 DEBUG ----- output ----- 22:31:58 DEBUG ----- output ----- 22:31:58 DEBUG { 22:31:58 DEBUG ----- output ----- 22:31:58 DEBUG ----- output ----- 22:31:58 DEBUG ----- output ----- 22:31:58 DEBUG { 22:31:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:31:58 DEBUG { 22:31:58 DEBUG { 22:31:58 DEBUG { 22:31:58 DEBUG { 22:31:58 DEBUG { 22:31:58 DEBUG ----- output ----- 22:31:58 DEBUG --- http response --- 22:31:58 DEBUG ----- output ----- 22:31:58 DEBUG { 22:31:58 DEBUG ----- output ----- 22:31:58 DEBUG { 22:31:58 DEBUG ----- output ----- 22:31:58 DEBUG { 22:31:58 DEBUG { 22:31:58 DEBUG { 22:31:58 DEBUG { 22:31:58 DEBUG { 22:31:58 DEBUG { 22:31:58 DEBUG "status": "success", 22:31:58 DEBUG { 22:31:58 DEBUG { 22:31:58 DEBUG { 22:31:58 DEBUG { 22:31:58 DEBUG "status": "success", 22:31:58 DEBUG ----- output ----- 22:31:58 DEBUG "status": "success", 22:31:58 DEBUG "status": "success", 22:31:58 DEBUG "status": "success", 22:31:58 DEBUG "status": "success", 22:31:58 DEBUG "status": "success", 22:31:58 DEBUG { 22:31:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:31:58 DEBUG { 22:31:58 DEBUG "status": "success", 22:31:58 DEBUG { 22:31:58 DEBUG "status": "success", 22:31:58 DEBUG { 22:31:58 DEBUG "status": "success", 22:31:58 DEBUG "status": "success", 22:31:58 DEBUG "status": "success", 22:31:58 DEBUG "status": "success", 22:31:58 DEBUG "status": "success", 22:31:58 DEBUG "status": "success", 22:31:58 DEBUG "data": { 22:31:58 DEBUG "status": "success", 22:31:58 DEBUG "status": "success", 22:31:58 DEBUG "status": "success", 22:31:58 DEBUG "status": "success", 22:31:58 DEBUG "data": { 22:31:58 DEBUG { 22:31:58 DEBUG "data": { 22:31:58 DEBUG "data": { 22:31:58 DEBUG "data": { 22:31:58 DEBUG "data": { 22:31:58 DEBUG "data": { 22:31:58 DEBUG "status": "success", 22:31:58 DEBUG ----- output ----- 22:31:58 DEBUG "status": "success", 22:31:58 DEBUG "data": { 22:31:58 DEBUG "status": "success", 22:31:58 DEBUG "data": { 22:31:58 DEBUG "status": "success", 22:31:58 DEBUG "data": { 22:31:58 DEBUG "data": { 22:31:58 DEBUG "data": { 22:31:58 DEBUG "data": { 22:31:58 DEBUG "data": { 22:31:58 DEBUG "data": { 22:31:58 DEBUG "resultType": "vector", 22:31:58 DEBUG "data": { 22:31:58 DEBUG "data": { 22:31:58 DEBUG "data": { 22:31:58 DEBUG "data": { 22:31:58 DEBUG "resultType": "vector", 22:31:58 DEBUG "status": "success", 22:31:58 DEBUG "resultType": "vector", 22:31:58 DEBUG "resultType": "vector", 22:31:58 DEBUG "resultType": "vector", 22:31:58 DEBUG "resultType": "vector", 22:31:58 DEBUG "resultType": "vector", 22:31:58 DEBUG "data": { 22:31:58 DEBUG { 22:31:58 DEBUG "data": { 22:31:58 DEBUG "resultType": "vector", 22:31:58 DEBUG "data": { 22:31:58 DEBUG "resultType": "vector", 22:31:58 DEBUG "data": { 22:31:58 DEBUG "resultType": "vector", 22:31:58 DEBUG "resultType": "vector", 22:31:58 DEBUG "resultType": "vector", 22:31:58 DEBUG "resultType": "vector", 22:31:58 DEBUG "resultType": "vector", 22:31:58 DEBUG "resultType": "vector", 22:31:58 DEBUG "result": [ 22:31:58 DEBUG "resultType": "vector", 22:31:58 DEBUG "resultType": "vector", 22:31:58 DEBUG "resultType": "vector", 22:31:58 DEBUG "resultType": "vector", 22:31:58 DEBUG "result": [ 22:31:58 DEBUG "data": { 22:31:58 DEBUG "result": [ 22:31:58 DEBUG "result": [ 22:31:58 DEBUG "result": [ 22:31:58 DEBUG "result": [ 22:31:58 DEBUG "result": [ 22:31:58 DEBUG "resultType": "vector", 22:31:58 DEBUG "status": "success", 22:31:58 DEBUG "resultType": "vector", 22:31:58 DEBUG "result": [ 22:31:58 DEBUG "resultType": "vector", 22:31:58 DEBUG "result": [ 22:31:58 DEBUG "resultType": "vector", 22:31:58 DEBUG "result": [ 22:31:58 DEBUG "result": [ 22:31:58 DEBUG "result": [ 22:31:58 DEBUG "result": [ 22:31:58 DEBUG "result": [ 22:31:58 DEBUG "result": [ 22:31:58 DEBUG { 22:31:58 DEBUG "result": [ 22:31:58 DEBUG "result": [ 22:31:58 DEBUG "result": [ 22:31:58 DEBUG "result": [ 22:31:58 DEBUG { 22:31:58 DEBUG "resultType": "vector", 22:31:58 DEBUG { 22:31:58 DEBUG { 22:31:58 DEBUG { 22:31:58 DEBUG { 22:31:58 DEBUG { 22:31:58 DEBUG "result": [ 22:31:58 DEBUG "data": { 22:31:58 DEBUG "result": [ 22:31:58 DEBUG { 22:31:58 DEBUG "result": [ 22:31:58 DEBUG { 22:31:58 DEBUG "result": [ 22:31:58 DEBUG { 22:31:58 DEBUG { 22:31:58 DEBUG { 22:31:58 DEBUG { 22:31:58 DEBUG { 22:31:58 DEBUG { 22:31:58 DEBUG "metric": { 22:31:58 DEBUG { 22:31:58 DEBUG { 22:31:58 DEBUG { 22:31:58 DEBUG { 22:31:58 DEBUG "metric": { 22:31:58 DEBUG "result": [ 22:31:58 DEBUG "metric": { 22:31:58 DEBUG "metric": { 22:31:58 DEBUG "metric": { 22:31:58 DEBUG "metric": { 22:31:58 DEBUG "metric": { 22:31:58 DEBUG { 22:31:58 DEBUG "resultType": "vector", 22:31:58 DEBUG { 22:31:58 DEBUG "metric": { 22:31:58 DEBUG { 22:31:58 DEBUG "metric": { 22:31:58 DEBUG { 22:31:58 DEBUG "metric": { 22:31:58 DEBUG "metric": { 22:31:58 DEBUG "metric": { 22:31:58 DEBUG "metric": { 22:31:58 DEBUG "metric": { 22:31:58 DEBUG "metric": { 22:31:58 DEBUG "pod": "am-555f4db595-fz8sn" 22:31:58 DEBUG "metric": { 22:31:58 DEBUG "metric": { 22:31:58 DEBUG "metric": { 22:31:58 DEBUG "metric": { 22:31:58 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:31:58 DEBUG { 22:31:58 DEBUG "instance": "10.142.0.107:9100" 22:31:58 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:31:58 DEBUG "pod": "am-555f4db595-fz8sn" 22:31:58 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:31:58 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:31:58 DEBUG "metric": { 22:31:58 DEBUG "result": [ 22:31:58 DEBUG "metric": { 22:31:58 DEBUG "pod": "ds-idrepo-2" 22:31:58 DEBUG "metric": { 22:31:58 DEBUG "pod": "am-555f4db595-fz8sn" 22:31:58 DEBUG "metric": { 22:31:58 DEBUG "pod": "ds-idrepo-2" 22:31:58 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:31:58 DEBUG "instance": "10.142.0.107:9100" 22:31:58 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:31:58 DEBUG "pod": "am-555f4db595-fz8sn" 22:31:58 DEBUG "instance": "10.142.0.107:9100" 22:31:58 DEBUG }, 22:31:58 DEBUG "instance": "10.142.0.107:9100" 22:31:58 DEBUG "instance": "10.142.0.107:9100" 22:31:58 DEBUG "pod": "am-555f4db595-fz8sn" 22:31:58 DEBUG "pod": "am-555f4db595-fz8sn" 22:31:58 DEBUG }, 22:31:58 DEBUG "metric": { 22:31:58 DEBUG }, 22:31:58 DEBUG }, 22:31:58 DEBUG }, 22:31:58 DEBUG }, 22:31:58 DEBUG }, 22:31:58 DEBUG "pod": "ds-idrepo-2" 22:31:58 DEBUG { 22:31:58 DEBUG "pod": "am-555f4db595-fz8sn" 22:31:58 DEBUG }, 22:31:58 DEBUG "pod": "ds-idrepo-2" 22:31:58 DEBUG }, 22:31:58 DEBUG "pod": "ds-idrepo-2" 22:31:58 DEBUG }, 22:31:58 DEBUG }, 22:31:58 DEBUG }, 22:31:58 DEBUG }, 22:31:58 DEBUG }, 22:31:58 DEBUG }, 22:31:58 DEBUG "value": [ 22:31:58 DEBUG }, 22:31:58 DEBUG }, 22:31:58 DEBUG }, 22:31:58 DEBUG }, 22:31:58 DEBUG "value": [ 22:31:58 DEBUG "pod": "ds-idrepo-2" 22:31:58 DEBUG "value": [ 22:31:58 DEBUG "value": [ 22:31:58 DEBUG "value": [ 22:31:58 DEBUG "value": [ 22:31:58 DEBUG "value": [ 22:31:58 DEBUG }, 22:31:58 DEBUG "metric": { 22:31:58 DEBUG }, 22:31:58 DEBUG "value": [ 22:31:58 DEBUG }, 22:31:58 DEBUG "value": [ 22:31:58 DEBUG }, 22:31:58 DEBUG "value": [ 22:31:58 DEBUG "value": [ 22:31:58 DEBUG "value": [ 22:31:58 DEBUG "value": [ 22:31:58 DEBUG "value": [ 22:31:58 DEBUG "value": [ 22:31:58 DEBUG 22:31:58 DEBUG "value": [ 22:31:58 DEBUG "value": [ 22:31:58 DEBUG "value": [ 22:31:58 DEBUG "value": [ 22:31:58 DEBUG 22:31:58 DEBUG }, 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG "value": [ 22:31:58 DEBUG "pod": "ds-idrepo-2" 22:31:58 DEBUG "value": [ 22:31:58 DEBUG 22:31:58 DEBUG "value": [ 22:31:58 DEBUG 22:31:58 DEBUG "value": [ 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG "value": [ 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG }, 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG "value": [ 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG [...] 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG [...] 22:31:58 DEBUG 22:31:58 DEBUG [...] 22:31:58 DEBUG [...] 22:31:58 DEBUG [...] 22:31:58 DEBUG [...] 22:31:58 DEBUG [...] 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG [...] 22:31:58 DEBUG 22:31:58 DEBUG [...] 22:31:58 DEBUG 22:31:58 DEBUG [...] 22:31:58 DEBUG [...] 22:31:58 DEBUG [...] 22:31:58 DEBUG [...] 22:31:58 DEBUG [...] 22:31:58 DEBUG [...] 22:31:58 DEBUG 22:31:58 DEBUG [...] 22:31:58 DEBUG [...] 22:31:58 DEBUG [...] 22:31:58 DEBUG [...] 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG [...] 22:31:58 DEBUG 22:31:58 DEBUG [...] 22:31:58 DEBUG 22:31:58 DEBUG [...] 22:31:58 DEBUG 22:31:58 DEBUG [...] 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG [...] 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG "pod": "ds-idrepo-2" 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG [...] 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG "pod": "am-555f4db595-kzxsp" 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG "instance": "10.142.0.43:9100" 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG }, 22:31:58 DEBUG "instance": "10.142.0.43:9100" 22:31:58 DEBUG "pod": "ds-idrepo-2" 22:31:58 DEBUG "pod": "am-555f4db595-kzxsp" 22:31:58 DEBUG "pod": "ds-idrepo-2" 22:31:58 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG 22:31:58 DEBUG "pod": "am-555f4db595-kzxsp" 22:31:58 DEBUG 22:31:58 DEBUG "pod": "am-555f4db595-kzxsp" 22:31:58 DEBUG 22:31:58 DEBUG "pod": "ds-idrepo-0" 22:31:58 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:31:58 DEBUG "instance": "10.142.0.43:9100" 22:31:58 DEBUG "pod": "ds-idrepo-2" 22:31:58 DEBUG "pod": "am-555f4db595-kzxsp" 22:31:58 DEBUG "instance": "10.142.0.43:9100" 22:31:58 DEBUG }, 22:31:58 DEBUG "instance": "10.142.0.43:9100" 22:31:58 DEBUG 22:31:58 DEBUG }, 22:31:58 DEBUG "pod": "am-555f4db595-kzxsp" 22:31:58 DEBUG 22:31:58 DEBUG "value": [ 22:31:58 DEBUG }, 22:31:58 DEBUG }, 22:31:58 DEBUG }, 22:31:58 DEBUG }, 22:31:58 DEBUG }, 22:31:58 DEBUG "pod": "ds-idrepo-0" 22:31:58 DEBUG 22:31:58 DEBUG "pod": "ds-idrepo-0" 22:31:58 DEBUG }, 22:31:58 DEBUG "pod": "ds-idrepo-0" 22:31:58 DEBUG }, 22:31:58 DEBUG "value": [ 22:31:58 DEBUG }, 22:31:58 DEBUG }, 22:31:58 DEBUG }, 22:31:58 DEBUG }, 22:31:58 DEBUG }, 22:31:58 DEBUG }, 22:31:58 DEBUG "value": [ 22:31:58 DEBUG }, 22:31:58 DEBUG "pod": "am-555f4db595-kzxsp" 22:31:58 DEBUG "value": [ 22:31:58 DEBUG }, 22:31:58 DEBUG 22:31:58 DEBUG 1713303118, 22:31:58 DEBUG "value": [ 22:31:58 DEBUG "value": [ 22:31:58 DEBUG "value": [ 22:31:58 DEBUG "value": [ 22:31:58 DEBUG "value": [ 22:31:58 DEBUG }, 22:31:58 DEBUG 22:31:58 DEBUG }, 22:31:58 DEBUG "value": [ 22:31:58 DEBUG }, 22:31:58 DEBUG "pod": "ds-cts-2" 22:31:58 DEBUG 1713303118, 22:31:58 DEBUG "value": [ 22:31:58 DEBUG "value": [ 22:31:58 DEBUG "value": [ 22:31:58 DEBUG "value": [ 22:31:58 DEBUG "value": [ 22:31:58 DEBUG "value": [ 22:31:58 DEBUG 1713303118, 22:31:58 DEBUG "value": [ 22:31:58 DEBUG }, 22:31:58 DEBUG 1713303118, 22:31:58 DEBUG "value": [ 22:31:58 DEBUG "pod": "ds-idrepo-0" 22:31:58 DEBUG "1.7987099927211767" 22:31:58 DEBUG 1713303118, 22:31:58 DEBUG 1713303118, 22:31:58 DEBUG 1713303118, 22:31:58 DEBUG 1713303118, 22:31:58 DEBUG 1713303118, 22:31:58 DEBUG "value": [ 22:31:58 DEBUG "pod": "ds-idrepo-0" 22:31:58 DEBUG "value": [ 22:31:58 DEBUG 1713303118, 22:31:58 DEBUG "value": [ 22:31:58 DEBUG }, 22:31:58 DEBUG "1995.0666666666666" 22:31:58 DEBUG 1713303118, 22:31:58 DEBUG 1713303118, 22:31:58 DEBUG 1713303118, 22:31:58 DEBUG 1713303118, 22:31:58 DEBUG 1713303118, 22:31:58 DEBUG 1713303118, 22:31:58 DEBUG "1995.0666666666666" 22:31:58 DEBUG 1713303118, 22:31:58 DEBUG "value": [ 22:31:58 DEBUG "19959526400" 22:31:58 DEBUG 1713303118, 22:31:58 DEBUG }, 22:31:58 DEBUG ] 22:31:58 DEBUG "0.0007666666666665378" 22:31:58 DEBUG "0" 22:31:58 DEBUG "0" 22:31:58 DEBUG "111048056.8809257" 22:31:58 DEBUG "6.276810279841718" 22:31:58 DEBUG 1713303118, 22:31:58 DEBUG }, 22:31:58 DEBUG 1713303118, 22:31:58 DEBUG "0.024" 22:31:58 DEBUG 1713303118, 22:31:58 DEBUG "value": [ 22:31:58 DEBUG ] 22:31:58 DEBUG "0" 22:31:58 DEBUG "658984960" 22:31:58 DEBUG "45305265.53333333" 22:31:58 DEBUG "2283.6461366181406" 22:31:58 DEBUG "0" 22:31:58 DEBUG "0" 22:31:58 DEBUG ] 22:31:58 DEBUG "45314963.5" 22:31:58 DEBUG 1713303118, 22:31:58 DEBUG ] 22:31:58 DEBUG "1994.6333333333332" 22:31:58 DEBUG "value": [ 22:31:58 DEBUG } 22:31:58 DEBUG ] 22:31:58 DEBUG ] 22:31:58 DEBUG ] 22:31:58 DEBUG ] 22:31:58 DEBUG ] 22:31:58 DEBUG "0" 22:31:58 DEBUG "value": [ 22:31:58 DEBUG "0" 22:31:58 DEBUG ] 22:31:58 DEBUG "0" 22:31:58 DEBUG 1713303118, 22:31:58 DEBUG } 22:31:58 DEBUG ] 22:31:58 DEBUG ] 22:31:58 DEBUG ] 22:31:58 DEBUG ] 22:31:58 DEBUG ] 22:31:58 DEBUG ] 22:31:58 DEBUG } 22:31:58 DEBUG ] 22:31:58 DEBUG "2419721" 22:31:58 DEBUG } 22:31:58 DEBUG ] 22:31:58 DEBUG 1713303118, 22:31:58 DEBUG ] 22:31:58 DEBUG } 22:31:58 DEBUG } 22:31:58 DEBUG } 22:31:58 DEBUG } 22:31:58 DEBUG } 22:31:58 DEBUG ] 22:31:58 DEBUG 1713303118, 22:31:58 DEBUG ] 22:31:58 DEBUG } 22:31:58 DEBUG ] 22:31:58 DEBUG "0" 22:31:58 DEBUG ] 22:31:58 DEBUG } 22:31:58 DEBUG } 22:31:58 DEBUG } 22:31:58 DEBUG } 22:31:58 DEBUG } 22:31:58 DEBUG } 22:31:58 DEBUG ] 22:31:58 DEBUG } 22:31:58 DEBUG ] 22:31:58 DEBUG ] 22:31:58 DEBUG } 22:31:58 DEBUG "0" 22:31:58 DEBUG } 22:31:58 DEBUG ] 22:31:58 DEBUG ] 22:31:58 DEBUG ] 22:31:58 DEBUG ] 22:31:58 DEBUG ] 22:31:58 DEBUG } 22:31:58 DEBUG "0" 22:31:58 DEBUG } 22:31:58 DEBUG ] 22:31:58 DEBUG } 22:31:58 DEBUG ] 22:31:58 DEBUG } 22:31:58 DEBUG ] 22:31:58 DEBUG ] 22:31:58 DEBUG ] 22:31:58 DEBUG ] 22:31:58 DEBUG } 22:31:58 DEBUG ] 22:31:58 DEBUG } 22:31:58 DEBUG ] 22:31:58 DEBUG } 22:31:58 DEBUG } 22:31:58 DEBUG ] 22:31:58 DEBUG ] 22:31:58 DEBUG } 22:31:58 DEBUG } 22:31:58 DEBUG } 22:31:58 DEBUG } 22:31:58 DEBUG } 22:31:58 DEBUG } 22:31:58 DEBUG ] 22:31:58 DEBUG ] 22:31:58 DEBUG ] 22:31:58 DEBUG } 22:31:58 DEBUG ] 22:31:58 DEBUG } 22:31:58 DEBUG } 22:31:58 DEBUG } 22:31:58 DEBUG } 22:31:58 DEBUG } 22:31:58 DEBUG ] 22:31:58 DEBUG } 22:31:58 DEBUG } 22:31:58 DEBUG } 22:31:58 DEBUG } 22:31:58 DEBUG ] 22:31:58 DEBUG } 22:31:58 DEBUG } 22:31:58 DEBUG } 22:31:58 DEBUG } 22:31:58 DEBUG } 22:31:58 DEBUG } 22:31:58 DEBUG } 22:31:58 DEBUG } 22:31:58 DEBUG } 22:31:58 DEBUG } 22:31:58 DEBUG } 22:31:58 DEBUG } 22:31:58 DEBUG } 22:31:58 DEBUG ] 22:31:58 DEBUG } 22:31:58 DEBUG } 22:31:58 DEBUG } 22:31:58 DEBUG } 22:31:58 DEBUG } 22:31:58 DEBUG } 22:31:58 DEBUG } 22:31:58 DEBUG } 22:31:58 DEBUG ] 22:31:58 DEBUG } 22:31:58 DEBUG ] 22:31:58 DEBUG } 22:31:58 DEBUG } 22:31:58 DEBUG } 22:31:58 DEBUG } 22:31:58 DEBUG } 22:31:58 DEBUG } 22:31:58 DEBUG } 22:31:58 DEBUG } 22:31:58 DEBUG } 22:31:58 DEBUG } 22:31:58 INFO 22:31:58 INFO [loop_until]: kubectl --namespace=xlou top node 22:31:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:31:58 INFO [loop_until]: OK (rc = 0) 22:31:58 DEBUG --- stdout --- 22:31:58 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 61m 0% 5110Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7548m 25% 14799Mi 13% gke-xlou-cdm-default-pool-1b0769c8-nhlf 9290m 31% 10825Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 85m 0% 2650Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 4726m 15% 6509Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 8566m 28% 16635Mi 14% gke-xlou-cdm-default-pool-1b0769c8-xnmp 63m 0% 3149Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 9434m 31% 10381Mi 9% gke-xlou-cdm-ds-0f601751-hmm7 4682m 15% 5598Mi 5% gke-xlou-cdm-ds-0f601751-j488 7299m 24% 14124Mi 12% gke-xlou-cdm-ds-0f601751-mc5d 10605m 35% 3345Mi 2% gke-xlou-cdm-ds-0f601751-nj6q 9467m 31% 10423Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 4576m 15% 7355Mi 6% gke-xlou-cdm-frontend-a8771548-i82n 6424m 40% 5951Mi 10% 22:31:58 DEBUG --- stderr --- 22:31:58 DEBUG 22:32:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713303176" 22:32:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303176" 22:32:56 ERROR [http_cmd]: ERROR 22:32:56 ERROR --- status code --- 22:32:56 ERROR http status code is 400 (expected [200, 401, 403, 404]) 22:32:56 ERROR --- http response --- 22:32:56 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 22:32:56 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 22:32:56 INFO [http_cmd]: http status code OK 22:32:56 DEBUG --- status code --- 22:32:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:32:56 DEBUG --- http response --- 22:32:56 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 22:32:57 INFO 22:32:57 INFO [loop_until]: kubectl --namespace=xlou top pods 22:32:57 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:32:57 INFO [loop_until]: OK (rc = 0) 22:32:57 DEBUG --- stdout --- 22:32:57 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 9Mi am-555f4db595-4lbss 9003m 8595Mi am-555f4db595-fz8sn 9473m 8607Mi am-555f4db595-kzxsp 9252m 8559Mi ds-cts-0 7999m 12145Mi ds-cts-1 7527m 12239Mi ds-cts-2 7454m 12199Mi ds-idrepo-0 4788m 5032Mi ds-idrepo-1 4481m 2850Mi ds-idrepo-2 4407m 3160Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 4m 1207Mi idm-b85f9d978-rl76f 4m 1225Mi lodemon-bc756c647-cr7x6 2m 63Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-7c9b667974-9mr2s 6172m 538Mi 22:32:57 DEBUG --- stderr --- 22:32:57 DEBUG 22:32:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303178" 22:32:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303178" 22:32:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303178" 22:32:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303178" 22:32:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303178" 22:32:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303178" 22:32:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303178" 22:32:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713303178" 22:32:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713303178" 22:32:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303178" 22:32:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303178" 22:32:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303178" 22:32:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713303178" 22:32:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713303178" 22:32:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303178" 22:32:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303178" 22:32:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303178" 22:32:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303178" 22:32:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303178" 22:32:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303178" 22:32:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303178" 22:32:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303178" 22:32:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303178" 22:32:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303178" 22:32:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303178" 22:32:58 INFO [http_cmd]: http status code OK 22:32:58 INFO [http_cmd]: http status code OK 22:32:58 INFO [http_cmd]: http status code OK 22:32:58 INFO [http_cmd]: http status code OK 22:32:58 INFO [http_cmd]: http status code OK 22:32:58 INFO [http_cmd]: http status code OK 22:32:58 INFO [http_cmd]: http status code OK 22:32:58 INFO [http_cmd]: http status code OK 22:32:58 DEBUG --- status code --- 22:32:58 INFO [http_cmd]: http status code OK 22:32:58 INFO [http_cmd]: http status code OK 22:32:58 INFO [http_cmd]: http status code OK 22:32:58 INFO [http_cmd]: http status code OK 22:32:58 INFO [http_cmd]: http status code OK 22:32:58 DEBUG --- status code --- 22:32:58 INFO [http_cmd]: http status code OK 22:32:58 INFO [http_cmd]: http status code OK 22:32:58 INFO [http_cmd]: http status code OK 22:32:58 INFO [http_cmd]: http status code OK 22:32:58 INFO [http_cmd]: http status code OK 22:32:58 DEBUG --- status code --- 22:32:58 DEBUG --- status code --- 22:32:58 INFO [http_cmd]: http status code OK 22:32:58 DEBUG --- status code --- 22:32:58 DEBUG --- status code --- 22:32:58 DEBUG --- status code --- 22:32:58 DEBUG --- status code --- 22:32:58 DEBUG --- status code --- 22:32:58 INFO [http_cmd]: http status code OK 22:32:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:32:58 DEBUG --- status code --- 22:32:58 INFO [http_cmd]: http status code OK 22:32:58 DEBUG --- status code --- 22:32:58 DEBUG --- status code --- 22:32:58 DEBUG --- status code --- 22:32:58 INFO [http_cmd]: http status code OK 22:32:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:32:58 DEBUG --- status code --- 22:32:58 DEBUG --- status code --- 22:32:58 DEBUG --- status code --- 22:32:58 DEBUG --- status code --- 22:32:58 DEBUG --- status code --- 22:32:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:32:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:32:58 DEBUG --- status code --- 22:32:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:32:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:32:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:32:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:32:58 DEBUG --- http response --- 22:32:58 DEBUG --- status code --- 22:32:58 DEBUG --- http response --- 22:32:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:32:58 DEBUG --- status code --- 22:32:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:32:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:32:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:32:58 DEBUG --- status code --- 22:32:58 DEBUG --- http response --- 22:32:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:32:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:32:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:32:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:32:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:32:58 DEBUG --- http response --- 22:32:58 DEBUG --- http response --- 22:32:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:32:58 DEBUG --- http response --- 22:32:58 DEBUG --- http response --- 22:32:58 DEBUG --- http response --- 22:32:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:32:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:32:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:32:58 INFO [http_cmd]: http status code OK 22:32:58 INFO [http_cmd]: http status code OK 22:32:58 DEBUG --- http response --- 22:32:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 22:32:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:32:58 DEBUG --- http response --- 22:32:58 DEBUG --- http response --- 22:32:58 DEBUG --- http response --- 22:32:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:32:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:32:58 DEBUG --- http response --- 22:32:58 INFO [http_cmd]: http status code OK 22:32:58 DEBUG --- http response --- 22:32:58 DEBUG --- http response --- 22:32:58 DEBUG --- http response --- 22:32:58 DEBUG --- http response --- 22:32:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:32:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:32:58 DEBUG --- http response --- 22:32:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:32:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:32:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:32:58 DEBUG --- http response --- 22:32:58 DEBUG --- http response --- 22:32:58 DEBUG ----- output ----- 22:32:58 DEBUG --- status code --- 22:32:58 DEBUG --- status code --- 22:32:58 DEBUG ----- output ----- 22:32:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:32:58 DEBUG --- http response --- 22:32:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 22:32:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:32:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:32:58 DEBUG --- http response --- 22:32:58 DEBUG ----- output ----- 22:32:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:32:58 DEBUG --- status code --- 22:32:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:32:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:32:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:32:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:32:58 DEBUG ----- output ----- 22:32:58 DEBUG ----- output ----- 22:32:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:32:58 DEBUG ----- output ----- 22:32:58 DEBUG ----- output ----- 22:32:58 DEBUG ----- output ----- 22:32:58 DEBUG { 22:32:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:32:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 22:32:58 DEBUG ----- output ----- 22:32:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:32:58 DEBUG { 22:32:58 DEBUG ----- output ----- 22:32:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:32:58 DEBUG ----- output ----- 22:32:58 DEBUG ----- output ----- 22:32:58 DEBUG ----- output ----- 22:32:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:32:58 DEBUG { 22:32:58 DEBUG ----- output ----- 22:32:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:32:58 DEBUG ----- output ----- 22:32:58 DEBUG ----- output ----- 22:32:58 DEBUG ----- output ----- 22:32:58 DEBUG ----- output ----- 22:32:58 DEBUG { 22:32:58 DEBUG { 22:32:58 DEBUG ----- output ----- 22:32:58 DEBUG { 22:32:58 DEBUG { 22:32:58 DEBUG { 22:32:58 DEBUG "status": "success", 22:32:58 DEBUG ----- output ----- 22:32:58 DEBUG { 22:32:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:32:58 DEBUG --- http response --- 22:32:58 DEBUG "status": "success", 22:32:58 DEBUG { 22:32:58 DEBUG ----- output ----- 22:32:58 DEBUG { 22:32:58 DEBUG { 22:32:58 DEBUG { 22:32:58 DEBUG ----- output ----- 22:32:58 DEBUG "status": "success", 22:32:58 DEBUG { 22:32:58 DEBUG --- http response --- 22:32:58 DEBUG { 22:32:58 DEBUG { 22:32:58 DEBUG { 22:32:58 DEBUG { 22:32:58 DEBUG "status": "success", 22:32:58 DEBUG "status": "success", 22:32:58 DEBUG { 22:32:58 DEBUG "status": "success", 22:32:58 DEBUG "status": "success", 22:32:58 DEBUG "status": "success", 22:32:58 DEBUG "data": { 22:32:58 DEBUG { 22:32:58 DEBUG "status": "success", 22:32:58 DEBUG --- http response --- 22:32:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 22:32:58 DEBUG "data": { 22:32:58 DEBUG "status": "success", 22:32:58 DEBUG { 22:32:58 DEBUG "status": "success", 22:32:58 DEBUG "status": "success", 22:32:58 DEBUG "status": "success", 22:32:58 DEBUG { 22:32:58 DEBUG "data": { 22:32:58 DEBUG "status": "success", 22:32:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:32:58 DEBUG "status": "success", 22:32:58 DEBUG "status": "success", 22:32:58 DEBUG "status": "success", 22:32:58 DEBUG "status": "success", 22:32:58 DEBUG "data": { 22:32:58 DEBUG "data": { 22:32:58 DEBUG "status": "success", 22:32:58 DEBUG "data": { 22:32:58 DEBUG "data": { 22:32:58 DEBUG "data": { 22:32:58 DEBUG "resultType": "vector", 22:32:58 DEBUG "status": "success", 22:32:58 DEBUG "data": { 22:32:58 DEBUG ----- output ----- 22:32:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:32:58 DEBUG "resultType": "vector", 22:32:58 DEBUG "data": { 22:32:58 DEBUG "status": "success", 22:32:58 DEBUG "data": { 22:32:58 DEBUG "data": { 22:32:58 DEBUG "data": { 22:32:58 DEBUG "status": "success", 22:32:58 DEBUG "resultType": "vector", 22:32:58 DEBUG "data": { 22:32:58 DEBUG ----- output ----- 22:32:58 DEBUG "data": { 22:32:58 DEBUG "data": { 22:32:58 DEBUG "data": { 22:32:58 DEBUG "data": { 22:32:58 DEBUG "resultType": "vector", 22:32:58 DEBUG "resultType": "vector", 22:32:58 DEBUG "result": [ 22:32:58 DEBUG "resultType": "vector", 22:32:58 DEBUG "resultType": "vector", 22:32:58 DEBUG "resultType": "vector", 22:32:58 DEBUG "result": [ 22:32:58 DEBUG "data": { 22:32:58 DEBUG "resultType": "vector", 22:32:58 DEBUG { 22:32:58 DEBUG ----- output ----- 22:32:58 DEBUG "result": [ 22:32:58 DEBUG "resultType": "vector", 22:32:58 DEBUG "data": { 22:32:58 DEBUG "resultType": "vector", 22:32:58 DEBUG "resultType": "vector", 22:32:58 DEBUG "resultType": "vector", 22:32:58 DEBUG "data": { 22:32:58 DEBUG "result": [ 22:32:58 DEBUG "resultType": "vector", 22:32:58 DEBUG { 22:32:58 DEBUG "resultType": "vector", 22:32:58 DEBUG "resultType": "vector", 22:32:58 DEBUG "resultType": "vector", 22:32:58 DEBUG "resultType": "vector", 22:32:58 DEBUG "result": [ 22:32:58 DEBUG "data": { 22:32:58 DEBUG { 22:32:58 DEBUG "result": [ 22:32:58 DEBUG "result": [ 22:32:58 DEBUG "result": [ 22:32:58 DEBUG { 22:32:58 DEBUG "resultType": "vector", 22:32:58 DEBUG "result": [ 22:32:58 DEBUG "status": "success", 22:32:58 DEBUG { 22:32:58 DEBUG { 22:32:58 DEBUG "result": [ 22:32:58 DEBUG "resultType": "vector", 22:32:58 DEBUG "result": [ 22:32:58 DEBUG "result": [ 22:32:58 DEBUG "result": [ 22:32:58 DEBUG "resultType": "vector", 22:32:58 DEBUG { 22:32:58 DEBUG "result": [ 22:32:58 DEBUG "status": "success", 22:32:58 DEBUG "result": [ 22:32:58 DEBUG "result": [ 22:32:58 DEBUG "result": [ 22:32:58 DEBUG "result": [ 22:32:58 DEBUG { 22:32:58 DEBUG "resultType": "vector", 22:32:58 DEBUG "metric": { 22:32:58 DEBUG { 22:32:58 DEBUG { 22:32:58 DEBUG { 22:32:58 DEBUG "metric": { 22:32:58 DEBUG "result": [ 22:32:58 DEBUG { 22:32:58 DEBUG "data": { 22:32:58 DEBUG "status": "success", 22:32:58 DEBUG "metric": { 22:32:58 DEBUG { 22:32:58 DEBUG "result": [ 22:32:58 DEBUG { 22:32:58 DEBUG { 22:32:58 DEBUG { 22:32:58 DEBUG "result": [ 22:32:58 DEBUG "metric": { 22:32:58 DEBUG { 22:32:58 DEBUG "data": { 22:32:58 DEBUG { 22:32:58 DEBUG { 22:32:58 DEBUG { 22:32:58 DEBUG { 22:32:58 DEBUG "metric": { 22:32:58 DEBUG "result": [ 22:32:58 DEBUG "pod": "ds-idrepo-2" 22:32:58 DEBUG "metric": { 22:32:58 DEBUG "metric": { 22:32:58 DEBUG "metric": { 22:32:58 DEBUG "pod": "ds-idrepo-2" 22:32:58 DEBUG { 22:32:58 DEBUG "metric": { 22:32:58 DEBUG "resultType": "vector", 22:32:58 DEBUG "data": { 22:32:58 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:32:58 DEBUG "metric": { 22:32:58 DEBUG { 22:32:58 DEBUG "metric": { 22:32:58 DEBUG "metric": { 22:32:58 DEBUG "metric": { 22:32:58 DEBUG { 22:32:58 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:32:58 DEBUG "metric": { 22:32:58 DEBUG "resultType": "vector", 22:32:58 DEBUG "metric": { 22:32:58 DEBUG "metric": { 22:32:58 DEBUG "metric": { 22:32:58 DEBUG "metric": { 22:32:58 DEBUG "instance": "10.142.0.107:9100" 22:32:58 DEBUG { 22:32:58 DEBUG }, 22:32:58 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:32:58 DEBUG "pod": "am-555f4db595-fz8sn" 22:32:58 DEBUG "pod": "am-555f4db595-fz8sn" 22:32:58 DEBUG }, 22:32:58 DEBUG "metric": { 22:32:58 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:32:58 DEBUG "result": [ 22:32:58 DEBUG "resultType": "vector", 22:32:58 DEBUG }, 22:32:58 DEBUG "instance": "10.142.0.107:9100" 22:32:58 DEBUG "metric": { 22:32:58 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:32:58 DEBUG "pod": "ds-idrepo-2" 22:32:58 DEBUG "pod": "am-555f4db595-fz8sn" 22:32:58 DEBUG "metric": { 22:32:58 DEBUG }, 22:32:58 DEBUG "pod": "ds-idrepo-2" 22:32:58 DEBUG "result": [ 22:32:58 DEBUG "pod": "ds-idrepo-2" 22:32:58 DEBUG "pod": "am-555f4db595-fz8sn" 22:32:58 DEBUG "pod": "ds-idrepo-2" 22:32:58 DEBUG "pod": "am-555f4db595-fz8sn" 22:32:58 DEBUG }, 22:32:58 DEBUG "metric": { 22:32:58 DEBUG "value": [ 22:32:58 DEBUG }, 22:32:58 DEBUG }, 22:32:58 DEBUG }, 22:32:58 DEBUG "value": [ 22:32:58 DEBUG "instance": "10.142.0.107:9100" 22:32:58 DEBUG }, 22:32:58 DEBUG { 22:32:58 DEBUG "result": [ 22:32:58 DEBUG "value": [ 22:32:58 DEBUG }, 22:32:58 DEBUG "pod": "am-555f4db595-fz8sn" 22:32:58 DEBUG }, 22:32:58 DEBUG }, 22:32:58 DEBUG }, 22:32:58 DEBUG "instance": "10.142.0.107:9100" 22:32:58 DEBUG "value": [ 22:32:58 DEBUG }, 22:32:58 DEBUG { 22:32:58 DEBUG }, 22:32:58 DEBUG }, 22:32:58 DEBUG }, 22:32:58 DEBUG }, 22:32:58 DEBUG "value": [ 22:32:58 DEBUG "pod": "ds-idrepo-2" 22:32:58 DEBUG 22:32:58 DEBUG "value": [ 22:32:58 DEBUG "value": [ 22:32:58 DEBUG "value": [ 22:32:58 DEBUG 22:32:58 DEBUG }, 22:32:58 DEBUG "value": [ 22:32:58 DEBUG "metric": { 22:32:58 DEBUG { 22:32:58 DEBUG 22:32:58 DEBUG "value": [ 22:32:58 DEBUG }, 22:32:58 DEBUG "value": [ 22:32:58 DEBUG "value": [ 22:32:58 DEBUG "value": [ 22:32:58 DEBUG }, 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG "metric": { 22:32:58 DEBUG "value": [ 22:32:58 DEBUG "value": [ 22:32:58 DEBUG "value": [ 22:32:58 DEBUG "value": [ 22:32:58 DEBUG 22:32:58 DEBUG }, 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG "value": [ 22:32:58 DEBUG 22:32:58 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:32:58 DEBUG "metric": { 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG "value": [ 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG "value": [ 22:32:58 DEBUG 22:32:58 DEBUG "value": [ 22:32:58 DEBUG "pod": "am-555f4db595-fz8sn" 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG "value": [ 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG }, 22:32:58 DEBUG "instance": "10.142.0.107:9100" 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG [...] 22:32:58 DEBUG 22:32:58 DEBUG }, 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG [...] 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG [...] 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG "value": [ 22:32:58 DEBUG }, 22:32:58 DEBUG [...] 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG "value": [ 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG [...] 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG [...] 22:32:58 DEBUG [...] 22:32:58 DEBUG [...] 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG [...] 22:32:58 DEBUG 22:32:58 DEBUG "value": [ 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG [...] 22:32:58 DEBUG [...] 22:32:58 DEBUG [...] 22:32:58 DEBUG [...] 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG [...] 22:32:58 DEBUG [...] 22:32:58 DEBUG [...] 22:32:58 DEBUG [...] 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG [...] 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG [...] 22:32:58 DEBUG 22:32:58 DEBUG [...] 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG [...] 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG [...] 22:32:58 DEBUG "pod": "ds-idrepo-2" 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG "pod": "ds-idrepo-0" 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG "pod": "ds-idrepo-0" 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG [...] 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG }, 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG }, 22:32:58 DEBUG 22:32:58 DEBUG [...] 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG "instance": "10.142.0.43:9100" 22:32:58 DEBUG 22:32:58 DEBUG }, 22:32:58 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:32:58 DEBUG "pod": "am-555f4db595-kzxsp" 22:32:58 DEBUG }, 22:32:58 DEBUG }, 22:32:58 DEBUG 22:32:58 DEBUG "pod": "ds-idrepo-2" 22:32:58 DEBUG 22:32:58 DEBUG [...] 22:32:58 DEBUG 22:32:58 DEBUG "value": [ 22:32:58 DEBUG "instance": "10.142.0.43:9100" 22:32:58 DEBUG "pod": "ds-idrepo-2" 22:32:58 DEBUG "pod": "ds-idrepo-0" 22:32:58 DEBUG "pod": "am-555f4db595-kzxsp" 22:32:58 DEBUG 22:32:58 DEBUG "value": [ 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG "pod": "ds-cts-2" 22:32:58 DEBUG "pod": "am-555f4db595-kzxsp" 22:32:58 DEBUG "pod": "ds-idrepo-0" 22:32:58 DEBUG "pod": "am-555f4db595-kzxsp" 22:32:58 DEBUG }, 22:32:58 DEBUG 22:32:58 DEBUG "value": [ 22:32:58 DEBUG }, 22:32:58 DEBUG "pod": "am-555f4db595-kzxsp" 22:32:58 DEBUG "value": [ 22:32:58 DEBUG "value": [ 22:32:58 DEBUG "instance": "10.142.0.43:9100" 22:32:58 DEBUG }, 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG 1713303178, 22:32:58 DEBUG }, 22:32:58 DEBUG }, 22:32:58 DEBUG }, 22:32:58 DEBUG }, 22:32:58 DEBUG "instance": "10.142.0.43:9100" 22:32:58 DEBUG 1713303178, 22:32:58 DEBUG "pod": "ds-idrepo-0" 22:32:58 DEBUG 22:32:58 DEBUG }, 22:32:58 DEBUG }, 22:32:58 DEBUG }, 22:32:58 DEBUG }, 22:32:58 DEBUG "value": [ 22:32:58 DEBUG "pod": "ds-idrepo-0" 22:32:58 DEBUG 1713303178, 22:32:58 DEBUG "value": [ 22:32:58 DEBUG }, 22:32:58 DEBUG 1713303178, 22:32:58 INFO 22:32:58 DEBUG 1713303178, 22:32:58 DEBUG }, 22:32:58 DEBUG "value": [ 22:32:58 DEBUG 22:32:58 DEBUG 22:32:58 DEBUG "pod": "am-555f4db595-kzxsp" 22:32:58 DEBUG "2528.248224554219" 22:32:58 DEBUG "value": [ 22:32:58 DEBUG "value": [ 22:32:58 DEBUG "value": [ 22:32:58 DEBUG "value": [ 22:32:58 DEBUG }, 22:32:58 DEBUG "661676032" 22:32:58 DEBUG }, 22:32:58 DEBUG 22:32:58 DEBUG "value": [ 22:32:58 DEBUG "value": [ 22:32:58 DEBUG "value": [ 22:32:58 DEBUG "value": [ 22:32:58 DEBUG 1713303178, 22:32:58 DEBUG }, 22:32:58 DEBUG "0" 22:32:58 DEBUG 1713303178, 22:32:58 DEBUG "value": [ 22:32:58 DEBUG "0.025" 22:32:58 INFO [loop_until]: kubectl --namespace=xlou top node 22:32:58 DEBUG "0" 22:32:58 DEBUG "value": [ 22:32:58 DEBUG 1713303178, 22:32:58 DEBUG "pod": "ds-idrepo-2" 22:32:58 DEBUG 22:32:58 DEBUG "instance": "10.142.0.43:9100" 22:32:58 DEBUG ] 22:32:58 DEBUG 1713303178, 22:32:58 DEBUG 1713303178, 22:32:58 DEBUG 1713303178, 22:32:58 DEBUG 1713303178, 22:32:58 DEBUG "value": [ 22:32:58 DEBUG ] 22:32:58 DEBUG "value": [ 22:32:58 DEBUG "pod": "am-555f4db595-kzxsp" 22:32:58 DEBUG 1713303178, 22:32:58 DEBUG 1713303178, 22:32:58 DEBUG 1713303178, 22:32:58 DEBUG 1713303178, 22:32:58 DEBUG "0.0005000000000004927" 22:32:58 DEBUG "value": [ 22:32:58 DEBUG ] 22:32:58 DEBUG "6.40544447414008" 22:32:58 DEBUG 1713303178, 22:32:58 DEBUG ] 22:32:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:32:58 DEBUG ] 22:32:58 DEBUG 1713303178, 22:32:58 DEBUG "2.3579903726917504" 22:32:58 DEBUG }, 22:32:58 DEBUG }, 22:32:58 DEBUG }, 22:32:58 DEBUG } 22:32:58 DEBUG "45510572.06666666" 22:32:58 DEBUG "0" 22:32:58 DEBUG "0" 22:32:58 DEBUG "0" 22:32:58 DEBUG 1713303178, 22:32:58 DEBUG } 22:32:58 DEBUG 1713303178, 22:32:58 DEBUG }, 22:32:58 DEBUG "0" 22:32:58 DEBUG "0" 22:32:58 DEBUG "0" 22:32:58 DEBUG "1998.5666666666666" 22:32:58 DEBUG ] 22:32:58 DEBUG 1713303178, 22:32:58 DEBUG } 22:32:58 DEBUG ] 22:32:58 DEBUG "1999" 22:32:58 DEBUG } 22:32:58 DEBUG } 22:32:58 DEBUG "19960333312" 22:32:58 DEBUG ] 22:32:58 DEBUG "value": [ 22:32:58 DEBUG "value": [ 22:32:58 DEBUG "value": [ 22:32:58 DEBUG ] 22:32:58 DEBUG ] 22:32:58 DEBUG ] 22:32:58 DEBUG ] 22:32:58 DEBUG ] 22:32:58 DEBUG "45502581.13333333" 22:32:58 DEBUG ] 22:32:58 DEBUG "0" 22:32:58 DEBUG "value": [ 22:32:58 DEBUG ] 22:32:58 DEBUG ] 22:32:58 DEBUG ] 22:32:58 DEBUG ] 22:32:58 DEBUG } 22:32:58 DEBUG "0" 22:32:58 DEBUG ] 22:32:58 DEBUG } 22:32:58 DEBUG ] 22:32:58 DEBUG ] 22:32:58 DEBUG ] 22:32:58 DEBUG ] 22:32:58 DEBUG } 22:32:58 DEBUG 1713303178, 22:32:58 DEBUG 1713303178, 22:32:58 DEBUG 1713303178, 22:32:58 DEBUG } 22:32:58 DEBUG } 22:32:58 DEBUG } 22:32:58 DEBUG } 22:32:58 DEBUG } 22:32:58 DEBUG ] 22:32:58 DEBUG } 22:32:58 DEBUG ] 22:32:58 DEBUG 1713303178, 22:32:58 DEBUG } 22:32:58 DEBUG } 22:32:58 DEBUG } 22:32:58 DEBUG } 22:32:58 DEBUG ] 22:32:58 DEBUG ] 22:32:58 DEBUG } 22:32:58 DEBUG ] 22:32:58 DEBUG } 22:32:58 DEBUG } 22:32:58 DEBUG } 22:32:58 DEBUG } 22:32:58 DEBUG ] 22:32:58 DEBUG "112212340.1887484" 22:32:58 DEBUG "1998.7" 22:32:58 DEBUG "0.0003333333333333262" 22:32:58 DEBUG } 22:32:58 DEBUG ] 22:32:58 DEBUG ] 22:32:58 DEBUG ] 22:32:58 DEBUG ] 22:32:58 DEBUG } 22:32:58 DEBUG } 22:32:58 DEBUG } 22:32:58 DEBUG "2539726.5" 22:32:58 DEBUG ] 22:32:58 DEBUG ] 22:32:58 DEBUG } 22:32:58 DEBUG ] 22:32:58 DEBUG } 22:32:58 DEBUG } 22:32:58 DEBUG } 22:32:58 DEBUG } 22:32:58 DEBUG ] 22:32:58 DEBUG } 22:32:58 DEBUG } 22:32:58 DEBUG ] 22:32:58 DEBUG } 22:32:58 DEBUG ] 22:32:58 DEBUG ] 22:32:58 DEBUG ] 22:32:58 DEBUG } 22:32:58 DEBUG } 22:32:58 DEBUG } 22:32:58 DEBUG } 22:32:58 DEBUG ] 22:32:58 DEBUG ] 22:32:58 DEBUG ] 22:32:58 DEBUG } 22:32:58 DEBUG ] 22:32:58 DEBUG } 22:32:58 DEBUG } 22:32:58 DEBUG } 22:32:58 DEBUG ] 22:32:58 DEBUG } 22:32:58 DEBUG } 22:32:58 DEBUG } 22:32:58 DEBUG } 22:32:58 DEBUG } 22:32:58 DEBUG } 22:32:58 DEBUG } 22:32:58 DEBUG } 22:32:58 DEBUG } 22:32:58 DEBUG } 22:32:58 DEBUG } 22:32:58 DEBUG } 22:32:58 DEBUG } 22:32:58 DEBUG } 22:32:58 DEBUG } 22:32:58 DEBUG } 22:32:58 DEBUG } 22:32:58 DEBUG } 22:32:58 DEBUG } 22:32:58 DEBUG } 22:32:58 DEBUG ] 22:32:58 DEBUG ] 22:32:58 DEBUG ] 22:32:58 DEBUG } 22:32:58 DEBUG } 22:32:58 DEBUG ] 22:32:58 DEBUG } 22:32:58 DEBUG } 22:32:58 DEBUG } 22:32:58 DEBUG } 22:32:58 DEBUG } 22:32:58 DEBUG } 22:32:58 DEBUG } 22:32:58 DEBUG } 22:32:58 DEBUG } 22:32:58 DEBUG } 22:32:58 INFO [loop_until]: OK (rc = 0) 22:32:58 DEBUG --- stdout --- 22:32:58 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 63m 0% 5103Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7523m 25% 14801Mi 13% gke-xlou-cdm-default-pool-1b0769c8-nhlf 9440m 31% 10825Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 83m 0% 2655Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 4749m 15% 6497Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 8629m 28% 16669Mi 14% gke-xlou-cdm-default-pool-1b0769c8-xnmp 60m 0% 3147Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 9281m 31% 10383Mi 9% gke-xlou-cdm-ds-0f601751-hmm7 4678m 15% 5597Mi 5% gke-xlou-cdm-ds-0f601751-j488 7274m 24% 14129Mi 12% gke-xlou-cdm-ds-0f601751-mc5d 10666m 35% 3365Mi 3% gke-xlou-cdm-ds-0f601751-nj6q 9534m 31% 10422Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 4767m 15% 7353Mi 6% gke-xlou-cdm-frontend-a8771548-i82n 6438m 40% 5953Mi 10% 22:32:58 DEBUG --- stderr --- 22:32:58 DEBUG 22:33:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713303236" 22:33:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303236" 22:33:56 ERROR [http_cmd]: ERROR 22:33:56 INFO [http_cmd]: http status code OK 22:33:56 ERROR --- status code --- 22:33:56 DEBUG --- status code --- 22:33:56 ERROR http status code is 400 (expected [200, 401, 403, 404]) 22:33:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:33:56 ERROR --- http response --- 22:33:56 DEBUG --- http response --- 22:33:56 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 22:33:56 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 22:33:56 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 22:33:57 INFO 22:33:57 INFO [loop_until]: kubectl --namespace=xlou top pods 22:33:57 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:33:57 INFO [loop_until]: OK (rc = 0) 22:33:57 DEBUG --- stdout --- 22:33:57 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 9Mi am-555f4db595-4lbss 9240m 8597Mi am-555f4db595-fz8sn 9470m 8609Mi am-555f4db595-kzxsp 9271m 8561Mi ds-cts-0 8251m 12152Mi ds-cts-1 7568m 12243Mi ds-cts-2 7264m 12205Mi ds-idrepo-0 4855m 5032Mi ds-idrepo-1 4441m 2850Mi ds-idrepo-2 4383m 3160Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 4m 1207Mi idm-b85f9d978-rl76f 4m 1225Mi lodemon-bc756c647-cr7x6 2m 63Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-7c9b667974-9mr2s 6103m 540Mi 22:33:57 DEBUG --- stderr --- 22:33:57 DEBUG 22:33:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303238" 22:33:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713303238" 22:33:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303238" 22:33:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713303238" 22:33:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303238" 22:33:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303238" 22:33:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303238" 22:33:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303238" 22:33:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303238" 22:33:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303238" 22:33:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303238" 22:33:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303238" 22:33:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713303238" 22:33:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713303238" 22:33:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303238" 22:33:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303238" 22:33:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303238" 22:33:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303238" 22:33:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303238" 22:33:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303238" 22:33:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303238" 22:33:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303238" 22:33:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303238" 22:33:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303238" 22:33:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303238" 22:33:58 INFO [http_cmd]: http status code OK 22:33:58 INFO [http_cmd]: http status code OK 22:33:58 INFO [http_cmd]: http status code OK 22:33:58 INFO [http_cmd]: http status code OK 22:33:58 INFO [http_cmd]: http status code OK 22:33:58 INFO [http_cmd]: http status code OK 22:33:58 INFO [http_cmd]: http status code OK 22:33:58 INFO [http_cmd]: http status code OK 22:33:58 INFO [http_cmd]: http status code OK 22:33:58 INFO [http_cmd]: http status code OK 22:33:58 INFO [http_cmd]: http status code OK 22:33:58 INFO [http_cmd]: http status code OK 22:33:58 INFO [http_cmd]: http status code OK 22:33:58 INFO [http_cmd]: http status code OK 22:33:58 INFO [http_cmd]: http status code OK 22:33:58 INFO [http_cmd]: http status code OK 22:33:58 INFO [http_cmd]: http status code OK 22:33:58 DEBUG --- status code --- 22:33:58 INFO [http_cmd]: http status code OK 22:33:58 DEBUG --- status code --- 22:33:58 INFO [http_cmd]: http status code OK 22:33:58 DEBUG --- status code --- 22:33:58 DEBUG --- status code --- 22:33:58 DEBUG --- status code --- 22:33:58 DEBUG --- status code --- 22:33:58 DEBUG --- status code --- 22:33:58 DEBUG --- status code --- 22:33:58 INFO [http_cmd]: http status code OK 22:33:58 DEBUG --- status code --- 22:33:58 INFO [http_cmd]: http status code OK 22:33:58 DEBUG --- status code --- 22:33:58 DEBUG --- status code --- 22:33:58 DEBUG --- status code --- 22:33:58 DEBUG --- status code --- 22:33:58 DEBUG --- status code --- 22:33:58 DEBUG --- status code --- 22:33:58 DEBUG --- status code --- 22:33:58 DEBUG --- status code --- 22:33:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:33:58 DEBUG --- status code --- 22:33:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:33:58 DEBUG --- status code --- 22:33:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:33:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:33:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:33:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:33:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:33:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:33:58 INFO [http_cmd]: http status code OK 22:33:58 DEBUG --- status code --- 22:33:58 INFO [http_cmd]: http status code OK 22:33:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:33:58 DEBUG --- status code --- 22:33:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:33:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:33:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:33:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:33:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:33:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:33:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:33:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:33:58 DEBUG --- http response --- 22:33:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:33:58 DEBUG --- http response --- 22:33:58 INFO [http_cmd]: http status code OK 22:33:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:33:58 DEBUG --- http response --- 22:33:58 DEBUG --- http response --- 22:33:58 DEBUG --- http response --- 22:33:58 DEBUG --- http response --- 22:33:58 DEBUG --- http response --- 22:33:58 DEBUG --- http response --- 22:33:58 DEBUG --- status code --- 22:33:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:33:58 DEBUG --- status code --- 22:33:58 DEBUG --- http response --- 22:33:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:33:58 DEBUG --- http response --- 22:33:58 DEBUG --- http response --- 22:33:58 DEBUG --- http response --- 22:33:58 DEBUG --- http response --- 22:33:58 DEBUG --- http response --- 22:33:58 DEBUG --- http response --- 22:33:58 DEBUG --- http response --- 22:33:58 DEBUG --- http response --- 22:33:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:33:58 DEBUG --- http response --- 22:33:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:33:58 DEBUG --- status code --- 22:33:58 DEBUG --- http response --- 22:33:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:33:58 INFO [http_cmd]: http status code OK 22:33:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:33:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:33:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:33:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:33:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:33:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:33:58 DEBUG --- http response --- 22:33:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:33:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:33:58 DEBUG --- http response --- 22:33:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:33:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:33:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:33:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:33:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:33:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:33:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:33:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:33:58 DEBUG ----- output ----- 22:33:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:33:58 DEBUG ----- output ----- 22:33:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:33:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:33:58 DEBUG ----- output ----- 22:33:58 DEBUG --- status code --- 22:33:58 DEBUG ----- output ----- 22:33:58 DEBUG ----- output ----- 22:33:58 DEBUG ----- output ----- 22:33:58 DEBUG ----- output ----- 22:33:58 DEBUG ----- output ----- 22:33:58 DEBUG --- http response --- 22:33:58 DEBUG --- http response --- 22:33:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:33:58 DEBUG ----- output ----- 22:33:58 DEBUG ----- output ----- 22:33:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:33:58 DEBUG ----- output ----- 22:33:58 DEBUG ----- output ----- 22:33:58 DEBUG ----- output ----- 22:33:58 DEBUG ----- output ----- 22:33:58 DEBUG ----- output ----- 22:33:58 DEBUG ----- output ----- 22:33:58 DEBUG ----- output ----- 22:33:58 DEBUG { 22:33:58 DEBUG ----- output ----- 22:33:58 DEBUG { 22:33:58 DEBUG --- http response --- 22:33:58 DEBUG ----- output ----- 22:33:58 DEBUG { 22:33:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:33:58 DEBUG --- http response --- 22:33:58 DEBUG { 22:33:58 DEBUG { 22:33:58 DEBUG { 22:33:58 DEBUG { 22:33:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:33:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:33:58 DEBUG ----- output ----- 22:33:58 DEBUG { 22:33:58 DEBUG { 22:33:58 DEBUG ----- output ----- 22:33:58 DEBUG { 22:33:58 DEBUG { 22:33:58 DEBUG { 22:33:58 DEBUG { 22:33:58 DEBUG { 22:33:58 DEBUG { 22:33:58 DEBUG { 22:33:58 DEBUG "status": "success", 22:33:58 DEBUG { 22:33:58 DEBUG "status": "success", 22:33:58 DEBUG { 22:33:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:33:58 DEBUG "status": "success", 22:33:58 DEBUG { 22:33:58 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:33:58 DEBUG "status": "success", 22:33:58 DEBUG "status": "success", 22:33:58 DEBUG "status": "success", 22:33:58 DEBUG "status": "success", 22:33:58 DEBUG ----- output ----- 22:33:58 DEBUG ----- output ----- 22:33:58 DEBUG { 22:33:58 DEBUG "status": "success", 22:33:58 DEBUG "status": "success", 22:33:58 DEBUG { 22:33:58 DEBUG "status": "success", 22:33:58 DEBUG "status": "success", 22:33:58 DEBUG "status": "success", 22:33:58 DEBUG "data": { 22:33:58 DEBUG "status": "success", 22:33:58 DEBUG "status": "success", 22:33:58 DEBUG "status": "success", 22:33:58 DEBUG "data": { 22:33:58 DEBUG "status": "success", 22:33:58 DEBUG "data": { 22:33:58 DEBUG "status": "success", 22:33:58 DEBUG ----- output ----- 22:33:58 DEBUG "data": { 22:33:58 DEBUG "status": "success", 22:33:58 DEBUG ----- output ----- 22:33:58 DEBUG "data": { 22:33:58 DEBUG "data": { 22:33:58 DEBUG "data": { 22:33:58 DEBUG "data": { 22:33:58 DEBUG { 22:33:58 DEBUG { 22:33:58 DEBUG "status": "success", 22:33:58 DEBUG "data": { 22:33:58 DEBUG "data": { 22:33:58 DEBUG "status": "success", 22:33:58 DEBUG "data": { 22:33:58 DEBUG "data": { 22:33:58 DEBUG "resultType": "vector", 22:33:58 DEBUG "status": "success", 22:33:58 DEBUG "data": { 22:33:58 DEBUG "data": { 22:33:58 DEBUG "data": { 22:33:58 DEBUG "resultType": "vector", 22:33:58 DEBUG "data": { 22:33:58 DEBUG "resultType": "vector", 22:33:58 DEBUG "data": { 22:33:58 DEBUG { 22:33:58 DEBUG "resultType": "vector", 22:33:58 DEBUG "data": { 22:33:58 DEBUG { 22:33:58 DEBUG "resultType": "vector", 22:33:58 DEBUG "resultType": "vector", 22:33:58 DEBUG "resultType": "vector", 22:33:58 DEBUG "resultType": "vector", 22:33:58 DEBUG "status": "success", 22:33:58 DEBUG "status": "success", 22:33:58 DEBUG "data": { 22:33:58 DEBUG "resultType": "vector", 22:33:58 DEBUG "resultType": "vector", 22:33:58 DEBUG "data": { 22:33:58 DEBUG "resultType": "vector", 22:33:58 DEBUG "resultType": "vector", 22:33:58 DEBUG "result": [ 22:33:58 DEBUG "data": { 22:33:58 DEBUG "resultType": "vector", 22:33:58 DEBUG "resultType": "vector", 22:33:58 DEBUG "resultType": "vector", 22:33:58 DEBUG "result": [ 22:33:58 DEBUG "resultType": "vector", 22:33:58 DEBUG "result": [ 22:33:58 DEBUG "resultType": "vector", 22:33:58 DEBUG "status": "success", 22:33:58 DEBUG "result": [ 22:33:58 DEBUG "resultType": "vector", 22:33:58 DEBUG "status": "success", 22:33:58 DEBUG "result": [ 22:33:58 DEBUG "result": [ 22:33:58 DEBUG "result": [ 22:33:58 DEBUG "result": [ 22:33:58 DEBUG "data": { 22:33:58 DEBUG "data": { 22:33:58 DEBUG "resultType": "vector", 22:33:58 DEBUG "result": [ 22:33:58 DEBUG "result": [ 22:33:58 DEBUG "resultType": "vector", 22:33:58 DEBUG "result": [ 22:33:58 DEBUG "result": [ 22:33:58 DEBUG { 22:33:58 DEBUG "resultType": "vector", 22:33:58 DEBUG "result": [ 22:33:58 DEBUG "result": [ 22:33:58 DEBUG "result": [ 22:33:58 DEBUG { 22:33:58 DEBUG "result": [ 22:33:58 DEBUG { 22:33:58 DEBUG "result": [ 22:33:58 DEBUG "data": { 22:33:58 DEBUG { 22:33:58 DEBUG "result": [ 22:33:58 DEBUG "data": { 22:33:58 DEBUG { 22:33:58 DEBUG { 22:33:58 DEBUG { 22:33:58 DEBUG { 22:33:58 DEBUG "resultType": "vector", 22:33:58 DEBUG "resultType": "vector", 22:33:58 DEBUG "result": [ 22:33:58 DEBUG { 22:33:58 DEBUG { 22:33:58 DEBUG "result": [ 22:33:58 DEBUG { 22:33:58 DEBUG { 22:33:58 DEBUG "metric": { 22:33:58 DEBUG "result": [ 22:33:58 DEBUG { 22:33:58 DEBUG { 22:33:58 DEBUG { 22:33:58 DEBUG "metric": { 22:33:58 DEBUG { 22:33:58 DEBUG "metric": { 22:33:58 DEBUG { 22:33:58 DEBUG "resultType": "vector", 22:33:58 DEBUG "metric": { 22:33:58 DEBUG { 22:33:58 DEBUG "resultType": "vector", 22:33:58 DEBUG "metric": { 22:33:58 DEBUG "metric": { 22:33:58 DEBUG "metric": { 22:33:58 DEBUG "metric": { 22:33:58 DEBUG "result": [ 22:33:58 DEBUG "result": [ 22:33:58 DEBUG { 22:33:58 DEBUG "metric": { 22:33:58 DEBUG "metric": { 22:33:58 DEBUG { 22:33:58 DEBUG "metric": { 22:33:58 DEBUG "metric": { 22:33:58 DEBUG "pod": "ds-idrepo-2" 22:33:58 DEBUG { 22:33:58 DEBUG "metric": { 22:33:58 DEBUG "metric": { 22:33:58 DEBUG "metric": { 22:33:58 DEBUG "pod": "am-555f4db595-fz8sn" 22:33:58 DEBUG "metric": { 22:33:58 DEBUG "pod": "ds-idrepo-2" 22:33:58 DEBUG "metric": { 22:33:58 DEBUG "result": [ 22:33:58 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:33:58 DEBUG "metric": { 22:33:58 DEBUG "result": [ 22:33:58 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:33:58 DEBUG "instance": "10.142.0.107:9100" 22:33:58 DEBUG "pod": "ds-idrepo-2" 22:33:58 DEBUG "instance": "10.142.0.107:9100" 22:33:58 DEBUG { 22:33:58 DEBUG { 22:33:58 DEBUG "metric": { 22:33:58 DEBUG "pod": "am-555f4db595-fz8sn" 22:33:58 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:33:58 DEBUG "metric": { 22:33:58 DEBUG "pod": "ds-idrepo-2" 22:33:58 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:33:58 DEBUG }, 22:33:58 DEBUG "metric": { 22:33:58 DEBUG "pod": "am-555f4db595-fz8sn" 22:33:58 DEBUG "pod": "am-555f4db595-fz8sn" 22:33:58 DEBUG "instance": "10.142.0.107:9100" 22:33:58 DEBUG }, 22:33:58 DEBUG "instance": "10.142.0.107:9100" 22:33:58 DEBUG }, 22:33:58 DEBUG "pod": "am-555f4db595-fz8sn" 22:33:58 DEBUG { 22:33:58 DEBUG }, 22:33:58 DEBUG "pod": "am-555f4db595-fz8sn" 22:33:58 DEBUG { 22:33:58 DEBUG }, 22:33:58 DEBUG }, 22:33:58 DEBUG }, 22:33:58 DEBUG }, 22:33:58 DEBUG "metric": { 22:33:58 DEBUG "metric": { 22:33:58 DEBUG "pod": "am-555f4db595-fz8sn" 22:33:58 DEBUG }, 22:33:58 DEBUG }, 22:33:58 DEBUG "pod": "ds-idrepo-2" 22:33:58 DEBUG }, 22:33:58 DEBUG }, 22:33:58 DEBUG "value": [ 22:33:58 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:33:58 DEBUG }, 22:33:58 DEBUG }, 22:33:58 DEBUG }, 22:33:58 DEBUG "value": [ 22:33:58 DEBUG }, 22:33:58 DEBUG "value": [ 22:33:58 DEBUG }, 22:33:58 DEBUG "metric": { 22:33:58 DEBUG "value": [ 22:33:58 DEBUG }, 22:33:58 DEBUG "metric": { 22:33:58 DEBUG "value": [ 22:33:58 DEBUG "value": [ 22:33:58 DEBUG "value": [ 22:33:58 DEBUG "value": [ 22:33:58 DEBUG "pod": "ds-idrepo-2" 22:33:58 DEBUG "pod": "ds-idrepo-2" 22:33:58 DEBUG }, 22:33:58 DEBUG "value": [ 22:33:58 DEBUG "value": [ 22:33:58 DEBUG }, 22:33:58 DEBUG "value": [ 22:33:58 DEBUG "value": [ 22:33:58 DEBUG 22:33:58 DEBUG }, 22:33:58 DEBUG "value": [ 22:33:58 DEBUG "value": [ 22:33:58 DEBUG "value": [ 22:33:58 DEBUG 22:33:58 DEBUG "value": [ 22:33:58 DEBUG 22:33:58 DEBUG "value": [ 22:33:58 DEBUG "instance": "10.142.0.107:9100" 22:33:58 DEBUG 22:33:58 DEBUG "value": [ 22:33:58 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG }, 22:33:58 DEBUG }, 22:33:58 DEBUG "value": [ 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG "value": [ 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG "value": [ 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG }, 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG }, 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG "value": [ 22:33:58 DEBUG "value": [ 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG "value": [ 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG "value": [ 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG [...] 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG [...] 22:33:58 DEBUG 22:33:58 DEBUG [...] 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG [...] 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG [...] 22:33:58 DEBUG [...] 22:33:58 DEBUG [...] 22:33:58 DEBUG [...] 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG [...] 22:33:58 DEBUG [...] 22:33:58 DEBUG 22:33:58 DEBUG [...] 22:33:58 DEBUG [...] 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG [...] 22:33:58 DEBUG [...] 22:33:58 DEBUG [...] 22:33:58 DEBUG 22:33:58 DEBUG [...] 22:33:58 DEBUG 22:33:58 DEBUG [...] 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG [...] 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG [...] 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG [...] 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG [...] 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG [...] 22:33:58 DEBUG [...] 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG [...] 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG [...] 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG "pod": "ds-idrepo-0" 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG "pod": "am-555f4db595-kzxsp" 22:33:58 DEBUG 22:33:58 DEBUG "pod": "ds-idrepo-0" 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG "pod": "overseer-0-7c9b667974-9mr2s" 22:33:58 DEBUG "instance": "10.142.0.43:9100" 22:33:58 DEBUG "pod": "ds-idrepo-0" 22:33:58 DEBUG "instance": "10.142.0.43:9100" 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG "pod": "am-555f4db595-kzxsp" 22:33:58 DEBUG "pod": "overseer-0-7c9b667974-9mr2s" 22:33:58 DEBUG 22:33:58 DEBUG "pod": "ds-cts-2" 22:33:58 DEBUG "pod": "overseer-0-7c9b667974-9mr2s" 22:33:58 DEBUG }, 22:33:58 DEBUG 22:33:58 DEBUG "pod": "am-555f4db595-kzxsp" 22:33:58 DEBUG "pod": "am-555f4db595-kzxsp" 22:33:58 DEBUG "instance": "10.142.0.43:9100" 22:33:58 DEBUG }, 22:33:58 DEBUG "instance": "10.142.0.43:9100" 22:33:58 DEBUG }, 22:33:58 DEBUG "pod": "am-555f4db595-kzxsp" 22:33:58 DEBUG 22:33:58 DEBUG }, 22:33:58 DEBUG "pod": "am-555f4db595-kzxsp" 22:33:58 DEBUG 22:33:58 DEBUG }, 22:33:58 DEBUG }, 22:33:58 DEBUG }, 22:33:58 DEBUG }, 22:33:58 DEBUG 22:33:58 DEBUG 22:33:58 DEBUG "pod": "am-555f4db595-kzxsp" 22:33:58 DEBUG }, 22:33:58 DEBUG }, 22:33:58 DEBUG "pod": "ds-idrepo-0" 22:33:58 DEBUG }, 22:33:58 DEBUG }, 22:33:58 DEBUG "value": [ 22:33:58 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:33:58 DEBUG }, 22:33:58 DEBUG }, 22:33:58 DEBUG }, 22:33:58 DEBUG "value": [ 22:33:58 DEBUG }, 22:33:58 DEBUG "value": [ 22:33:58 DEBUG }, 22:33:58 DEBUG 22:33:58 DEBUG "value": [ 22:33:58 DEBUG }, 22:33:58 DEBUG 22:33:58 DEBUG "value": [ 22:33:58 DEBUG "value": [ 22:33:58 DEBUG "value": [ 22:33:58 DEBUG "value": [ 22:33:58 DEBUG "pod": "ds-idrepo-0" 22:33:58 DEBUG "pod": "ds-idrepo-0" 22:33:58 DEBUG }, 22:33:58 DEBUG "value": [ 22:33:58 DEBUG "value": [ 22:33:58 DEBUG }, 22:33:58 DEBUG "value": [ 22:33:58 DEBUG "value": [ 22:33:58 DEBUG 1713303238, 22:33:58 DEBUG }, 22:33:58 DEBUG "value": [ 22:33:58 DEBUG "value": [ 22:33:58 DEBUG "value": [ 22:33:58 DEBUG 1713303238, 22:33:58 DEBUG "value": [ 22:33:58 DEBUG 1713303238, 22:33:58 DEBUG "value": [ 22:33:58 DEBUG "instance": "10.142.0.43:9100" 22:33:58 DEBUG 1713303238, 22:33:58 DEBUG "value": [ 22:33:58 DEBUG "pod": "overseer-0-7c9b667974-9mr2s" 22:33:58 DEBUG 1713303238, 22:33:58 DEBUG 1713303238, 22:33:58 DEBUG 1713303238, 22:33:58 DEBUG 1713303238, 22:33:58 DEBUG }, 22:33:58 DEBUG }, 22:33:58 DEBUG "value": [ 22:33:58 DEBUG 1713303238, 22:33:58 DEBUG 1713303238, 22:33:58 DEBUG "value": [ 22:33:58 DEBUG 1713303238, 22:33:58 DEBUG 1713303238, 22:33:58 DEBUG "0" 22:33:58 DEBUG "value": [ 22:33:58 DEBUG 1713303238, 22:33:58 DEBUG "2004.4666666666667" 22:33:58 DEBUG 1713303238, 22:33:58 DEBUG "0.025" 22:33:58 DEBUG 1713303238, 22:33:58 DEBUG "0" 22:33:58 DEBUG 1713303238, 22:33:58 DEBUG }, 22:33:58 DEBUG "664715264" 22:33:58 DEBUG 1713303238, 22:33:58 DEBUG }, 22:33:58 DEBUG "0" 22:33:58 DEBUG "45386641.699999996" 22:33:58 DEBUG "0" 22:33:58 DEBUG "0.0006666666666670077" 22:33:58 DEBUG "value": [ 22:33:58 DEBUG "value": [ 22:33:58 DEBUG 1713303238, 22:33:58 DEBUG "0" 22:33:58 DEBUG "0" 22:33:58 DEBUG 1713303238, 22:33:58 DEBUG "0" 22:33:58 DEBUG "6.226017233327275" 22:33:58 DEBUG ] 22:33:58 DEBUG 1713303238, 22:33:58 DEBUG ] 22:33:58 DEBUG 1713303238, 22:33:58 DEBUG "19961308160" 22:33:58 DEBUG ] 22:33:58 DEBUG "45377012.03333333" 22:33:58 DEBUG ] 22:33:58 DEBUG "2659491" 22:33:58 DEBUG "value": [ 22:33:58 DEBUG ] 22:33:58 DEBUG "0" 22:33:58 DEBUG "value": [ 22:33:58 DEBUG ] 22:33:58 DEBUG ] 22:33:58 DEBUG ] 22:33:58 DEBUG ] 22:33:58 DEBUG 1713303238, 22:33:58 DEBUG 1713303238, 22:33:58 DEBUG "2004.4666666666667" 22:33:58 DEBUG ] 22:33:58 DEBUG ] 22:33:58 DEBUG "0" 22:33:58 DEBUG ] 22:33:58 DEBUG ] 22:33:58 DEBUG } 22:33:58 DEBUG "6.255335589164456" 22:33:58 DEBUG } 22:33:58 DEBUG "2004.4666666666667" 22:33:58 DEBUG ] 22:33:58 DEBUG } 22:33:58 DEBUG ] 22:33:58 DEBUG } 22:33:58 DEBUG ] 22:33:58 DEBUG 1713303238, 22:33:58 DEBUG } 22:33:58 DEBUG ] 22:33:58 DEBUG 1713303238, 22:33:58 DEBUG } 22:33:58 DEBUG } 22:33:58 DEBUG } 22:33:58 DEBUG } 22:33:58 DEBUG "0" 22:33:58 DEBUG "0" 22:33:58 DEBUG ] 22:33:58 DEBUG } 22:33:58 DEBUG } 22:33:58 DEBUG ] 22:33:58 DEBUG } 22:33:58 DEBUG } 22:33:58 DEBUG ] 22:33:58 DEBUG ] 22:33:58 DEBUG ] 22:33:58 DEBUG ] 22:33:58 DEBUG } 22:33:58 DEBUG ] 22:33:58 DEBUG } 22:33:58 DEBUG ] 22:33:58 DEBUG } 22:33:58 DEBUG "0" 22:33:58 DEBUG ] 22:33:58 DEBUG } 22:33:58 DEBUG "45444.88905056384" 22:33:58 DEBUG ] 22:33:58 DEBUG ] 22:33:58 DEBUG ] 22:33:58 DEBUG ] 22:33:58 DEBUG ] 22:33:58 DEBUG ] 22:33:58 DEBUG } 22:33:58 DEBUG ] 22:33:58 DEBUG ] 22:33:58 DEBUG } 22:33:58 DEBUG ] 22:33:58 DEBUG ] 22:33:58 DEBUG } 22:33:58 DEBUG } 22:33:58 DEBUG } 22:33:58 DEBUG } 22:33:58 DEBUG ] 22:33:58 DEBUG } 22:33:58 DEBUG ] 22:33:58 DEBUG } 22:33:58 DEBUG ] 22:33:58 DEBUG ] 22:33:58 DEBUG } 22:33:58 DEBUG ] 22:33:58 DEBUG ] 22:33:58 DEBUG } 22:33:58 DEBUG } 22:33:58 DEBUG } 22:33:58 DEBUG } 22:33:58 DEBUG } 22:33:58 DEBUG } 22:33:58 DEBUG ] 22:33:58 DEBUG } 22:33:58 DEBUG } 22:33:58 DEBUG ] 22:33:58 DEBUG } 22:33:58 DEBUG } 22:33:58 DEBUG } 22:33:58 DEBUG ] 22:33:58 DEBUG } 22:33:58 DEBUG ] 22:33:58 DEBUG } 22:33:58 DEBUG } 22:33:58 DEBUG } 22:33:58 DEBUG } 22:33:58 DEBUG } 22:33:58 DEBUG } 22:33:58 DEBUG } 22:33:58 DEBUG } 22:33:58 DEBUG } 22:33:58 DEBUG } 22:33:58 DEBUG } 22:33:58 DEBUG } 22:33:58 DEBUG } 22:33:58 DEBUG ] 22:33:58 DEBUG ] 22:33:58 DEBUG } 22:33:58 DEBUG } 22:33:58 DEBUG } 22:33:58 DEBUG } 22:33:58 DEBUG } 22:33:58 DEBUG } 22:33:58 DEBUG } 22:33:58 DEBUG } 22:33:58 DEBUG } 22:33:58 DEBUG } 22:33:58 DEBUG } 22:33:58 DEBUG ] 22:33:58 DEBUG } 22:33:58 DEBUG ] 22:33:58 DEBUG } 22:33:58 DEBUG } 22:33:58 DEBUG } 22:33:58 DEBUG } 22:33:58 DEBUG } 22:33:58 DEBUG } 22:33:58 DEBUG } 22:33:58 DEBUG } 22:33:58 DEBUG } 22:33:58 DEBUG } 22:33:58 DEBUG } 22:33:58 DEBUG } 22:33:58 INFO 22:33:58 INFO [loop_until]: kubectl --namespace=xlou top node 22:33:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:33:59 INFO [loop_until]: OK (rc = 0) 22:33:59 DEBUG --- stdout --- 22:33:59 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 65m 0% 5100Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7538m 25% 14798Mi 13% gke-xlou-cdm-default-pool-1b0769c8-nhlf 9526m 31% 10823Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 88m 0% 2638Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 4759m 15% 6511Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 8883m 29% 16646Mi 14% gke-xlou-cdm-default-pool-1b0769c8-xnmp 61m 0% 3144Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 9484m 31% 10375Mi 9% gke-xlou-cdm-ds-0f601751-hmm7 4609m 15% 5598Mi 5% gke-xlou-cdm-ds-0f601751-j488 7201m 24% 14143Mi 12% gke-xlou-cdm-ds-0f601751-mc5d 10611m 35% 3341Mi 2% gke-xlou-cdm-ds-0f601751-nj6q 9695m 32% 10422Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 4808m 16% 7357Mi 6% gke-xlou-cdm-frontend-a8771548-i82n 6464m 40% 5953Mi 10% 22:33:59 DEBUG --- stderr --- 22:33:59 DEBUG 22:34:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713303296" 22:34:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303296" 22:34:56 INFO [http_cmd]: http status code OK 22:34:56 DEBUG --- status code --- 22:34:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:34:56 DEBUG --- http response --- 22:34:56 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 22:34:56 ERROR [http_cmd]: ERROR 22:34:56 ERROR --- status code --- 22:34:56 ERROR http status code is 400 (expected [200, 401, 403, 404]) 22:34:56 ERROR --- http response --- 22:34:56 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 22:34:56 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 22:34:57 INFO 22:34:57 INFO [loop_until]: kubectl --namespace=xlou top pods 22:34:57 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:34:57 INFO [loop_until]: OK (rc = 0) 22:34:57 DEBUG --- stdout --- 22:34:57 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 9Mi am-555f4db595-4lbss 9223m 8597Mi am-555f4db595-fz8sn 9055m 8609Mi am-555f4db595-kzxsp 9271m 8563Mi ds-cts-0 8387m 12160Mi ds-cts-1 7379m 12249Mi ds-cts-2 7222m 12212Mi ds-idrepo-0 4779m 5032Mi ds-idrepo-1 4446m 2875Mi ds-idrepo-2 4685m 3161Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 4m 1208Mi idm-b85f9d978-rl76f 4m 1225Mi lodemon-bc756c647-cr7x6 2m 63Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-7c9b667974-9mr2s 6110m 541Mi 22:34:57 DEBUG --- stderr --- 22:34:57 DEBUG 22:34:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303298" 22:34:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303298" 22:34:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713303298" 22:34:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303298" 22:34:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303298" 22:34:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303298" 22:34:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303298" 22:34:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303298" 22:34:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303298" 22:34:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303298" 22:34:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303298" 22:34:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303298" 22:34:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303298" 22:34:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713303298" 22:34:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713303298" 22:34:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303298" 22:34:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303298" 22:34:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303298" 22:34:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303298" 22:34:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303298" 22:34:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303298" 22:34:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303298" 22:34:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303298" 22:34:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303298" 22:34:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713303298" 22:34:58 INFO [http_cmd]: http status code OK 22:34:58 INFO [http_cmd]: http status code OK 22:34:58 INFO [http_cmd]: http status code OK 22:34:58 INFO [http_cmd]: http status code OK 22:34:58 INFO [http_cmd]: http status code OK 22:34:58 INFO [http_cmd]: http status code OK 22:34:58 INFO [http_cmd]: http status code OK 22:34:58 INFO [http_cmd]: http status code OK 22:34:58 INFO [http_cmd]: http status code OK 22:34:58 INFO [http_cmd]: http status code OK 22:34:58 INFO [http_cmd]: http status code OK 22:34:59 INFO [http_cmd]: http status code OK 22:34:59 INFO [http_cmd]: http status code OK 22:34:59 INFO [http_cmd]: http status code OK 22:34:59 INFO [http_cmd]: http status code OK 22:34:59 INFO [http_cmd]: http status code OK 22:34:59 INFO [http_cmd]: http status code OK 22:34:59 DEBUG --- status code --- 22:34:59 INFO [http_cmd]: http status code OK 22:34:59 DEBUG --- status code --- 22:34:59 INFO [http_cmd]: http status code OK 22:34:59 DEBUG --- status code --- 22:34:59 INFO [http_cmd]: http status code OK 22:34:59 DEBUG --- status code --- 22:34:59 DEBUG --- status code --- 22:34:59 DEBUG --- status code --- 22:34:59 DEBUG --- status code --- 22:34:59 DEBUG --- status code --- 22:34:59 DEBUG --- status code --- 22:34:59 DEBUG --- status code --- 22:34:59 DEBUG --- status code --- 22:34:59 DEBUG --- status code --- 22:34:59 DEBUG --- status code --- 22:34:59 DEBUG --- status code --- 22:34:59 DEBUG --- status code --- 22:34:59 DEBUG --- status code --- 22:34:59 DEBUG --- status code --- 22:34:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:34:59 DEBUG --- status code --- 22:34:59 INFO [http_cmd]: http status code OK 22:34:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:34:59 DEBUG --- status code --- 22:34:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:34:59 DEBUG --- status code --- 22:34:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:34:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:34:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:34:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:34:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:34:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:34:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:34:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:34:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:34:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:34:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:34:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:34:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:34:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:34:59 DEBUG --- http response --- 22:34:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:34:59 DEBUG --- status code --- 22:34:59 DEBUG --- http response --- 22:34:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:34:59 DEBUG --- http response --- 22:34:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:34:59 DEBUG --- http response --- 22:34:59 DEBUG --- http response --- 22:34:59 DEBUG --- http response --- 22:34:59 DEBUG --- http response --- 22:34:59 DEBUG --- http response --- 22:34:59 DEBUG --- http response --- 22:34:59 DEBUG --- http response --- 22:34:59 DEBUG --- http response --- 22:34:59 DEBUG --- http response --- 22:34:59 DEBUG --- http response --- 22:34:59 DEBUG --- http response --- 22:34:59 DEBUG --- http response --- 22:34:59 DEBUG --- http response --- 22:34:59 DEBUG --- http response --- 22:34:59 DEBUG --- http response --- 22:34:59 INFO [http_cmd]: http status code OK 22:34:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:34:59 INFO [http_cmd]: http status code OK 22:34:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:34:59 DEBUG --- http response --- 22:34:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:34:59 DEBUG --- http response --- 22:34:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 22:34:59 INFO [http_cmd]: http status code OK 22:34:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:34:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:34:59 INFO [http_cmd]: http status code OK 22:34:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:34:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:34:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:34:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:34:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:34:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:34:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 22:34:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:34:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:34:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:34:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:34:59 DEBUG --- status code --- 22:34:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:34:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:34:59 DEBUG ----- output ----- 22:34:59 DEBUG --- status code --- 22:34:59 DEBUG --- http response --- 22:34:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:34:59 DEBUG ----- output ----- 22:34:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:34:59 DEBUG ----- output ----- 22:34:59 DEBUG --- status code --- 22:34:59 DEBUG ----- output ----- 22:34:59 DEBUG ----- output ----- 22:34:59 DEBUG --- status code --- 22:34:59 DEBUG ----- output ----- 22:34:59 DEBUG ----- output ----- 22:34:59 DEBUG ----- output ----- 22:34:59 DEBUG ----- output ----- 22:34:59 DEBUG ----- output ----- 22:34:59 DEBUG ----- output ----- 22:34:59 DEBUG ----- output ----- 22:34:59 DEBUG ----- output ----- 22:34:59 DEBUG ----- output ----- 22:34:59 DEBUG ----- output ----- 22:34:59 DEBUG ----- output ----- 22:34:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:34:59 DEBUG ----- output ----- 22:34:59 DEBUG ----- output ----- 22:34:59 DEBUG { 22:34:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:34:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:34:59 DEBUG ----- output ----- 22:34:59 DEBUG { 22:34:59 DEBUG ----- output ----- 22:34:59 DEBUG { 22:34:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:34:59 DEBUG { 22:34:59 DEBUG { 22:34:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:34:59 DEBUG { 22:34:59 DEBUG { 22:34:59 DEBUG { 22:34:59 DEBUG { 22:34:59 DEBUG { 22:34:59 DEBUG { 22:34:59 DEBUG { 22:34:59 DEBUG { 22:34:59 DEBUG { 22:34:59 DEBUG { 22:34:59 DEBUG { 22:34:59 DEBUG --- http response --- 22:34:59 DEBUG { 22:34:59 DEBUG { 22:34:59 DEBUG "status": "success", 22:34:59 DEBUG --- http response --- 22:34:59 DEBUG ----- output ----- 22:34:59 DEBUG { 22:34:59 DEBUG "status": "success", 22:34:59 DEBUG { 22:34:59 DEBUG "status": "success", 22:34:59 DEBUG --- http response --- 22:34:59 DEBUG "status": "success", 22:34:59 DEBUG "status": "success", 22:34:59 DEBUG "data": { 22:34:59 DEBUG "status": "success", 22:34:59 DEBUG "status": "success", 22:34:59 DEBUG "status": "success", 22:34:59 DEBUG "status": "success", 22:34:59 DEBUG "status": "success", 22:34:59 DEBUG "status": "success", 22:34:59 DEBUG "status": "success", 22:34:59 DEBUG "status": "success", 22:34:59 DEBUG "status": "success", 22:34:59 DEBUG "status": "success", 22:34:59 DEBUG "status": "success", 22:34:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:34:59 DEBUG "status": "success", 22:34:59 DEBUG "status": "success", 22:34:59 DEBUG "data": { 22:34:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:34:59 DEBUG { 22:34:59 DEBUG "status": "success", 22:34:59 DEBUG "data": { 22:34:59 DEBUG "status": "success", 22:34:59 DEBUG "data": { 22:34:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:34:59 DEBUG "data": { 22:34:59 DEBUG --- http response --- 22:34:59 DEBUG "resultType": "vector", 22:34:59 DEBUG "data": { 22:34:59 DEBUG "data": { 22:34:59 DEBUG "data": { 22:34:59 DEBUG "data": { 22:34:59 DEBUG "data": { 22:34:59 DEBUG "data": { 22:34:59 DEBUG "data": { 22:34:59 DEBUG "data": { 22:34:59 DEBUG "data": { 22:34:59 DEBUG "data": { 22:34:59 DEBUG "data": { 22:34:59 DEBUG ----- output ----- 22:34:59 DEBUG "data": { 22:34:59 DEBUG "data": { 22:34:59 DEBUG "resultType": "vector", 22:34:59 DEBUG ----- output ----- 22:34:59 DEBUG "status": "success", 22:34:59 DEBUG "data": { 22:34:59 DEBUG "resultType": "vector", 22:34:59 DEBUG "data": { 22:34:59 DEBUG "resultType": "vector", 22:34:59 DEBUG ----- output ----- 22:34:59 DEBUG "resultType": "vector", 22:34:59 DEBUG "result": [ 22:34:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:34:59 DEBUG "resultType": "vector", 22:34:59 DEBUG "resultType": "vector", 22:34:59 DEBUG "resultType": "vector", 22:34:59 DEBUG "resultType": "vector", 22:34:59 DEBUG "resultType": "vector", 22:34:59 DEBUG "resultType": "vector", 22:34:59 DEBUG "resultType": "vector", 22:34:59 DEBUG "resultType": "vector", 22:34:59 DEBUG "resultType": "vector", 22:34:59 DEBUG "resultType": "vector", 22:34:59 DEBUG "resultType": "vector", 22:34:59 DEBUG { 22:34:59 DEBUG "resultType": "vector", 22:34:59 DEBUG "resultType": "vector", 22:34:59 DEBUG "result": [ 22:34:59 DEBUG { 22:34:59 DEBUG "data": { 22:34:59 DEBUG "resultType": "vector", 22:34:59 DEBUG "result": [ 22:34:59 DEBUG "resultType": "vector", 22:34:59 DEBUG "result": [ 22:34:59 DEBUG { 22:34:59 DEBUG "result": [ 22:34:59 DEBUG { 22:34:59 DEBUG ----- output ----- 22:34:59 DEBUG "result": [ 22:34:59 DEBUG "result": [ 22:34:59 DEBUG "result": [ 22:34:59 DEBUG "result": [ 22:34:59 DEBUG "result": [ 22:34:59 DEBUG "result": [ 22:34:59 DEBUG "result": [ 22:34:59 DEBUG "result": [ 22:34:59 DEBUG "result": [ 22:34:59 DEBUG "result": [ 22:34:59 DEBUG "result": [ 22:34:59 DEBUG "status": "success", 22:34:59 DEBUG "result": [ 22:34:59 DEBUG "result": [ 22:34:59 DEBUG { 22:34:59 DEBUG "status": "success", 22:34:59 DEBUG "resultType": "vector", 22:34:59 DEBUG "result": [ 22:34:59 DEBUG { 22:34:59 DEBUG "result": [ 22:34:59 DEBUG { 22:34:59 DEBUG "status": "success", 22:34:59 DEBUG { 22:34:59 DEBUG "metric": { 22:34:59 DEBUG { 22:34:59 DEBUG { 22:34:59 DEBUG { 22:34:59 DEBUG { 22:34:59 DEBUG { 22:34:59 DEBUG { 22:34:59 DEBUG { 22:34:59 DEBUG { 22:34:59 DEBUG { 22:34:59 DEBUG { 22:34:59 DEBUG { 22:34:59 DEBUG { 22:34:59 DEBUG "data": { 22:34:59 DEBUG { 22:34:59 DEBUG { 22:34:59 DEBUG "metric": { 22:34:59 DEBUG "data": { 22:34:59 DEBUG "result": [ 22:34:59 DEBUG { 22:34:59 DEBUG "metric": { 22:34:59 DEBUG { 22:34:59 DEBUG "metric": { 22:34:59 DEBUG "data": { 22:34:59 DEBUG "metric": { 22:34:59 DEBUG "pod": "am-555f4db595-fz8sn" 22:34:59 DEBUG }, 22:34:59 DEBUG "metric": { 22:34:59 DEBUG "metric": { 22:34:59 DEBUG "metric": { 22:34:59 DEBUG "metric": { 22:34:59 DEBUG "metric": { 22:34:59 DEBUG "metric": { 22:34:59 DEBUG "metric": { 22:34:59 DEBUG "metric": { 22:34:59 DEBUG "metric": { 22:34:59 DEBUG "metric": { 22:34:59 DEBUG "metric": { 22:34:59 DEBUG "resultType": "vector", 22:34:59 DEBUG "metric": { 22:34:59 DEBUG "metric": { 22:34:59 DEBUG "pod": "am-555f4db595-fz8sn" 22:34:59 DEBUG "resultType": "vector", 22:34:59 DEBUG { 22:34:59 DEBUG "metric": { 22:34:59 DEBUG "pod": "ds-idrepo-2" 22:34:59 DEBUG "metric": { 22:34:59 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:34:59 DEBUG "resultType": "vector", 22:34:59 DEBUG "pod": "am-555f4db595-fz8sn" 22:34:59 DEBUG "status": "success", 22:34:59 DEBUG "value": [ 22:34:59 DEBUG "instance": "10.142.0.107:9100" 22:34:59 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:34:59 DEBUG "instance": "10.142.0.107:9100" 22:34:59 DEBUG "instance": "10.142.0.107:9100" 22:34:59 DEBUG "pod": "ds-idrepo-2" 22:34:59 DEBUG "pod": "ds-idrepo-2" 22:34:59 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:34:59 DEBUG "pod": "am-555f4db595-fz8sn" 22:34:59 DEBUG "instance": "10.142.0.107:9100" 22:34:59 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:34:59 DEBUG "pod": "ds-idrepo-2" 22:34:59 DEBUG "result": [ 22:34:59 DEBUG "pod": "am-555f4db595-fz8sn" 22:34:59 DEBUG "pod": "am-555f4db595-fz8sn" 22:34:59 DEBUG }, 22:34:59 DEBUG "result": [ 22:34:59 DEBUG "metric": { 22:34:59 DEBUG "pod": "ds-idrepo-2" 22:34:59 DEBUG }, 22:34:59 DEBUG "pod": "am-555f4db595-fz8sn" 22:34:59 DEBUG }, 22:34:59 DEBUG "result": [ 22:34:59 DEBUG }, 22:34:59 DEBUG "data": { 22:34:59 DEBUG 22:34:59 DEBUG }, 22:34:59 DEBUG }, 22:34:59 DEBUG }, 22:34:59 DEBUG }, 22:34:59 DEBUG }, 22:34:59 DEBUG }, 22:34:59 DEBUG }, 22:34:59 DEBUG }, 22:34:59 DEBUG }, 22:34:59 DEBUG }, 22:34:59 DEBUG }, 22:34:59 DEBUG { 22:34:59 DEBUG }, 22:34:59 DEBUG }, 22:34:59 DEBUG "value": [ 22:34:59 DEBUG { 22:34:59 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:34:59 DEBUG }, 22:34:59 DEBUG "value": [ 22:34:59 DEBUG }, 22:34:59 DEBUG "value": [ 22:34:59 DEBUG { 22:34:59 DEBUG "value": [ 22:34:59 DEBUG "resultType": "vector", 22:34:59 DEBUG 22:34:59 DEBUG "value": [ 22:34:59 DEBUG "value": [ 22:34:59 DEBUG "value": [ 22:34:59 DEBUG "value": [ 22:34:59 DEBUG "value": [ 22:34:59 DEBUG "value": [ 22:34:59 DEBUG "value": [ 22:34:59 DEBUG "value": [ 22:34:59 DEBUG "value": [ 22:34:59 DEBUG "value": [ 22:34:59 DEBUG "value": [ 22:34:59 DEBUG "metric": { 22:34:59 DEBUG "value": [ 22:34:59 DEBUG "value": [ 22:34:59 DEBUG 22:34:59 DEBUG "metric": { 22:34:59 DEBUG }, 22:34:59 DEBUG "value": [ 22:34:59 DEBUG 22:34:59 DEBUG "value": [ 22:34:59 DEBUG 22:34:59 DEBUG "metric": { 22:34:59 DEBUG 22:34:59 DEBUG "result": [ 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG "pod": "ds-idrepo-2" 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG "pod": "ds-idrepo-2" 22:34:59 DEBUG "value": [ 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:34:59 DEBUG 22:34:59 DEBUG { 22:34:59 DEBUG [...] 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG }, 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG }, 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG }, 22:34:59 DEBUG 22:34:59 DEBUG "metric": { 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG [...] 22:34:59 DEBUG 22:34:59 DEBUG "value": [ 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG [...] 22:34:59 DEBUG "value": [ 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG [...] 22:34:59 DEBUG 22:34:59 DEBUG [...] 22:34:59 DEBUG "value": [ 22:34:59 DEBUG [...] 22:34:59 DEBUG "instance": "10.142.0.107:9100" 22:34:59 DEBUG 22:34:59 DEBUG [...] 22:34:59 DEBUG [...] 22:34:59 DEBUG [...] 22:34:59 DEBUG [...] 22:34:59 DEBUG [...] 22:34:59 DEBUG [...] 22:34:59 DEBUG [...] 22:34:59 DEBUG [...] 22:34:59 DEBUG [...] 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG [...] 22:34:59 DEBUG [...] 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG [...] 22:34:59 DEBUG 22:34:59 DEBUG [...] 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG }, 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG [...] 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG [...] 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG "value": [ 22:34:59 DEBUG "pod": "am-555f4db595-kzxsp" 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG "pod": "end-user-ui-58878d7668-zg4h2" 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG }, 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG "pod": "am-555f4db595-kzxsp" 22:34:59 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:34:59 DEBUG 22:34:59 DEBUG [...] 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG "pod": "am-555f4db595-kzxsp" 22:34:59 DEBUG [...] 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG "pod": "ds-idrepo-0" 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG }, 22:34:59 DEBUG "pod": "am-555f4db595-kzxsp" 22:34:59 DEBUG 22:34:59 DEBUG "value": [ 22:34:59 DEBUG "instance": "10.142.0.43:9100" 22:34:59 INFO 22:34:59 DEBUG "pod": "end-user-ui-58878d7668-zg4h2" 22:34:59 DEBUG "instance": "10.142.0.43:9100" 22:34:59 DEBUG "instance": "10.142.0.43:9100" 22:34:59 DEBUG "pod": "ds-idrepo-0" 22:34:59 DEBUG "pod": "ds-idrepo-0" 22:34:59 DEBUG "pod": "end-user-ui-58878d7668-zg4h2" 22:34:59 DEBUG }, 22:34:59 DEBUG 22:34:59 DEBUG }, 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG "pod": "ds-cts-2" 22:34:59 DEBUG "pod": "am-555f4db595-kzxsp" 22:34:59 DEBUG }, 22:34:59 DEBUG 22:34:59 DEBUG 22:34:59 DEBUG "pod": "ds-idrepo-0" 22:34:59 DEBUG }, 22:34:59 DEBUG "pod": "am-555f4db595-kzxsp" 22:34:59 DEBUG [...] 22:34:59 DEBUG "value": [ 22:34:59 DEBUG }, 22:34:59 DEBUG 22:34:59 DEBUG 1713303298, 22:34:59 DEBUG }, 22:34:59 INFO [loop_until]: kubectl --namespace=xlou top node 22:34:59 DEBUG }, 22:34:59 DEBUG }, 22:34:59 DEBUG }, 22:34:59 DEBUG }, 22:34:59 DEBUG }, 22:34:59 DEBUG }, 22:34:59 DEBUG "value": [ 22:34:59 DEBUG "instance": "10.142.0.43:9100" 22:34:59 DEBUG "value": [ 22:34:59 DEBUG "pod": "am-555f4db595-kzxsp" 22:34:59 DEBUG 22:34:59 DEBUG }, 22:34:59 DEBUG }, 22:34:59 DEBUG "value": [ 22:34:59 DEBUG 22:34:59 DEBUG "pod": "end-user-ui-58878d7668-zg4h2" 22:34:59 DEBUG }, 22:34:59 DEBUG "value": [ 22:34:59 DEBUG }, 22:34:59 DEBUG 22:34:59 DEBUG 1713303298, 22:34:59 DEBUG "value": [ 22:34:59 DEBUG [...] 22:34:59 DEBUG "0" 22:34:59 DEBUG "value": [ 22:34:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:34:59 DEBUG "value": [ 22:34:59 DEBUG "value": [ 22:34:59 DEBUG "value": [ 22:34:59 DEBUG "value": [ 22:34:59 DEBUG "value": [ 22:34:59 DEBUG "value": [ 22:34:59 DEBUG 1713303298, 22:34:59 DEBUG }, 22:34:59 DEBUG 1713303298, 22:34:59 DEBUG }, 22:34:59 DEBUG 22:34:59 DEBUG "value": [ 22:34:59 DEBUG "value": [ 22:34:59 DEBUG 1713303298, 22:34:59 DEBUG 22:34:59 DEBUG }, 22:34:59 DEBUG "value": [ 22:34:59 DEBUG 1713303298, 22:34:59 DEBUG "value": [ 22:34:59 DEBUG 22:34:59 DEBUG "0" 22:34:59 DEBUG 1713303298, 22:34:59 DEBUG 22:34:59 DEBUG ] 22:34:59 DEBUG 1713303298, 22:34:59 DEBUG 1713303298, 22:34:59 DEBUG 1713303298, 22:34:59 DEBUG 1713303298, 22:34:59 DEBUG 1713303298, 22:34:59 DEBUG 1713303298, 22:34:59 DEBUG "0" 22:34:59 DEBUG "2779583" 22:34:59 DEBUG "value": [ 22:34:59 DEBUG "665751552" 22:34:59 DEBUG "value": [ 22:34:59 DEBUG "pod": "ds-idrepo-0" 22:34:59 DEBUG 1713303298, 22:34:59 DEBUG 1713303298, 22:34:59 DEBUG "1997" 22:34:59 DEBUG "pod": "ds-idrepo-0" 22:34:59 DEBUG "value": [ 22:34:59 DEBUG 1713303298, 22:34:59 DEBUG "0" 22:34:59 DEBUG 1713303298, 22:34:59 DEBUG 22:34:59 DEBUG ] 22:34:59 DEBUG "0.025" 22:34:59 DEBUG 22:34:59 DEBUG } 22:34:59 DEBUG "0.00029999999999953767" 22:34:59 DEBUG "0.000022817361180501138" 22:34:59 DEBUG "45356086.36666666" 22:34:59 DEBUG "19962072064" 22:34:59 DEBUG "0" 22:34:59 DEBUG 1713303298, 22:34:59 DEBUG ] 22:34:59 DEBUG ] 22:34:59 DEBUG 1713303298, 22:34:59 DEBUG ] 22:34:59 DEBUG 1713303298, 22:34:59 DEBUG }, 22:34:59 DEBUG "0" 22:34:59 DEBUG "0" 22:34:59 DEBUG ] 22:34:59 DEBUG }, 22:34:59 DEBUG 1713303298, 22:34:59 DEBUG "0" 22:34:59 DEBUG ] 22:34:59 DEBUG "1999.0666666666666" 22:34:59 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:34:59 DEBUG } 22:34:59 DEBUG ] 22:34:59 DEBUG 22:34:59 DEBUG ] 22:34:59 DEBUG ] 22:34:59 DEBUG ] 22:34:59 DEBUG ] 22:34:59 DEBUG ] 22:34:59 DEBUG ] 22:34:59 DEBUG "0" 22:34:59 DEBUG } 22:34:59 DEBUG } 22:34:59 DEBUG "45346307.233333334" 22:34:59 DEBUG } 22:34:59 DEBUG "1997.1666666666667" 22:34:59 DEBUG "value": [ 22:34:59 DEBUG 1713303298, 22:34:59 DEBUG ] 22:34:59 DEBUG } 22:34:59 DEBUG "value": [ 22:34:59 DEBUG "0" 22:34:59 DEBUG ] 22:34:59 DEBUG } 22:34:59 DEBUG ] 22:34:59 DEBUG }, 22:34:59 DEBUG ] 22:34:59 DEBUG } 22:34:59 DEBUG "instance": "10.142.0.43:9100" 22:34:59 DEBUG } 22:34:59 DEBUG } 22:34:59 DEBUG } 22:34:59 DEBUG } 22:34:59 DEBUG } 22:34:59 DEBUG } 22:34:59 DEBUG ] 22:34:59 DEBUG ] 22:34:59 DEBUG ] 22:34:59 DEBUG ] 22:34:59 DEBUG ] 22:34:59 DEBUG ] 22:34:59 DEBUG "0" 22:34:59 DEBUG } 22:34:59 DEBUG ] 22:34:59 DEBUG 1713303298, 22:34:59 DEBUG ] 22:34:59 DEBUG } 22:34:59 DEBUG ] 22:34:59 DEBUG } 22:34:59 DEBUG "value": [ 22:34:59 DEBUG } 22:34:59 DEBUG ] 22:34:59 DEBUG }, 22:34:59 DEBUG } 22:34:59 DEBUG ] 22:34:59 DEBUG ] 22:34:59 DEBUG ] 22:34:59 DEBUG ] 22:34:59 DEBUG ] 22:34:59 DEBUG } 22:34:59 DEBUG } 22:34:59 DEBUG } 22:34:59 DEBUG } 22:34:59 DEBUG } 22:34:59 DEBUG } 22:34:59 DEBUG ] 22:34:59 DEBUG ] 22:34:59 DEBUG ] 22:34:59 DEBUG } 22:34:59 DEBUG "0" 22:34:59 DEBUG } 22:34:59 DEBUG ] 22:34:59 DEBUG } 22:34:59 DEBUG ] 22:34:59 DEBUG 1713303298, 22:34:59 DEBUG } 22:34:59 DEBUG } 22:34:59 DEBUG "value": [ 22:34:59 DEBUG } 22:34:59 DEBUG } 22:34:59 DEBUG } 22:34:59 DEBUG } 22:34:59 DEBUG } 22:34:59 DEBUG ] 22:34:59 DEBUG } 22:34:59 DEBUG } 22:34:59 DEBUG ] 22:34:59 DEBUG } 22:34:59 DEBUG ] 22:34:59 DEBUG } 22:34:59 DEBUG } 22:34:59 DEBUG } 22:34:59 DEBUG } 22:34:59 DEBUG ] 22:34:59 DEBUG ] 22:34:59 DEBUG } 22:34:59 DEBUG } 22:34:59 DEBUG } 22:34:59 DEBUG "6.310195781815096" 22:34:59 DEBUG ] 22:34:59 DEBUG 1713303298, 22:34:59 DEBUG } 22:34:59 DEBUG } 22:34:59 DEBUG } 22:34:59 DEBUG } 22:34:59 DEBUG } 22:34:59 DEBUG } 22:34:59 DEBUG } 22:34:59 DEBUG } 22:34:59 DEBUG ] 22:34:59 DEBUG ] 22:34:59 DEBUG } 22:34:59 DEBUG } 22:34:59 DEBUG } 22:34:59 DEBUG } 22:34:59 DEBUG } 22:34:59 DEBUG } 22:34:59 DEBUG } 22:34:59 DEBUG "0" 22:34:59 DEBUG } 22:34:59 DEBUG } 22:34:59 DEBUG } 22:34:59 DEBUG } 22:34:59 DEBUG } 22:34:59 DEBUG ] 22:34:59 DEBUG } 22:34:59 DEBUG ] 22:34:59 DEBUG ] 22:34:59 DEBUG } 22:34:59 DEBUG } 22:34:59 DEBUG } 22:34:59 DEBUG } 22:34:59 DEBUG } 22:34:59 DEBUG } 22:34:59 DEBUG } 22:34:59 DEBUG ] 22:34:59 DEBUG } 22:34:59 DEBUG } 22:34:59 INFO [loop_until]: OK (rc = 0) 22:34:59 DEBUG --- stdout --- 22:34:59 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 62m 0% 5096Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7532m 25% 14811Mi 13% gke-xlou-cdm-default-pool-1b0769c8-nhlf 9315m 31% 10842Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 80m 0% 2627Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 4736m 15% 6508Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 8561m 28% 16664Mi 14% gke-xlou-cdm-default-pool-1b0769c8-xnmp 58m 0% 3139Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 9467m 31% 10378Mi 9% gke-xlou-cdm-ds-0f601751-hmm7 4625m 15% 5610Mi 5% gke-xlou-cdm-ds-0f601751-j488 7459m 24% 14148Mi 12% gke-xlou-cdm-ds-0f601751-mc5d 10573m 35% 3345Mi 2% gke-xlou-cdm-ds-0f601751-nj6q 9710m 32% 10402Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 4779m 16% 7356Mi 6% gke-xlou-cdm-frontend-a8771548-i82n 6449m 40% 5953Mi 10% 22:34:59 DEBUG --- stderr --- 22:34:59 DEBUG 22:35:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303356" 22:35:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713303356" 22:35:56 ERROR [http_cmd]: ERROR 22:35:56 ERROR --- status code --- 22:35:56 ERROR http status code is 400 (expected [200, 401, 403, 404]) 22:35:56 ERROR --- http response --- 22:35:56 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 22:35:56 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 22:35:56 INFO [http_cmd]: http status code OK 22:35:56 DEBUG --- status code --- 22:35:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:35:56 DEBUG --- http response --- 22:35:56 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 22:35:57 INFO 22:35:57 INFO [loop_until]: kubectl --namespace=xlou top pods 22:35:57 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:35:58 INFO [loop_until]: OK (rc = 0) 22:35:58 DEBUG --- stdout --- 22:35:58 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 9Mi am-555f4db595-4lbss 9300m 8598Mi am-555f4db595-fz8sn 9392m 8610Mi am-555f4db595-kzxsp 9129m 8562Mi ds-cts-0 8511m 12166Mi ds-cts-1 7483m 12257Mi ds-cts-2 7324m 12218Mi ds-idrepo-0 4908m 5022Mi ds-idrepo-1 4355m 2874Mi ds-idrepo-2 4432m 3164Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 4m 1207Mi idm-b85f9d978-rl76f 4m 1225Mi lodemon-bc756c647-cr7x6 2m 63Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-7c9b667974-9mr2s 6463m 543Mi 22:35:58 DEBUG --- stderr --- 22:35:58 DEBUG 22:35:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303359" 22:35:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303359" 22:35:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303359" 22:35:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303359" 22:35:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713303359" 22:35:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303359" 22:35:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303359" 22:35:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303359" 22:35:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303359" 22:35:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303359" 22:35:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303359" 22:35:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303359" 22:35:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303359" 22:35:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303359" 22:35:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303359" 22:35:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303359" 22:35:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303359" 22:35:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303359" 22:35:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713303359" 22:35:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303359" 22:35:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303359" 22:35:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713303359" 22:35:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713303359" 22:35:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303359" 22:35:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303359" 22:35:59 INFO [http_cmd]: http status code OK 22:35:59 INFO [http_cmd]: http status code OK 22:35:59 INFO [http_cmd]: http status code OK 22:35:59 INFO [http_cmd]: http status code OK 22:35:59 INFO [http_cmd]: http status code OK 22:35:59 INFO [http_cmd]: http status code OK 22:35:59 INFO [http_cmd]: http status code OK 22:35:59 INFO [http_cmd]: http status code OK 22:35:59 INFO [http_cmd]: http status code OK 22:35:59 INFO [http_cmd]: http status code OK 22:35:59 INFO [http_cmd]: http status code OK 22:35:59 INFO [http_cmd]: http status code OK 22:35:59 INFO [http_cmd]: http status code OK 22:35:59 INFO [http_cmd]: http status code OK 22:35:59 INFO [http_cmd]: http status code OK 22:35:59 INFO [http_cmd]: http status code OK 22:35:59 INFO [http_cmd]: http status code OK 22:35:59 DEBUG --- status code --- 22:35:59 DEBUG --- status code --- 22:35:59 INFO [http_cmd]: http status code OK 22:35:59 INFO [http_cmd]: http status code OK 22:35:59 DEBUG --- status code --- 22:35:59 INFO [http_cmd]: http status code OK 22:35:59 DEBUG --- status code --- 22:35:59 DEBUG --- status code --- 22:35:59 DEBUG --- status code --- 22:35:59 DEBUG --- status code --- 22:35:59 DEBUG --- status code --- 22:35:59 DEBUG --- status code --- 22:35:59 DEBUG --- status code --- 22:35:59 DEBUG --- status code --- 22:35:59 INFO [http_cmd]: http status code OK 22:35:59 DEBUG --- status code --- 22:35:59 DEBUG --- status code --- 22:35:59 DEBUG --- status code --- 22:35:59 DEBUG --- status code --- 22:35:59 DEBUG --- status code --- 22:35:59 DEBUG --- status code --- 22:35:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:35:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:35:59 DEBUG --- status code --- 22:35:59 DEBUG --- status code --- 22:35:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:35:59 INFO 22:35:59 DEBUG --- status code --- 22:35:59 INFO [http_cmd]: http status code OK 22:35:59 INFO [http_cmd]: http status code OK 22:35:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:35:59 INFO [http_cmd]: http status code OK 22:35:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:35:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:35:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:35:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:35:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:35:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:35:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:35:59 DEBUG --- status code --- 22:35:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:35:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:35:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:35:59 DEBUG --- http response --- 22:35:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:35:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:35:59 DEBUG --- http response --- 22:35:59 DEBUG --- http response --- 22:35:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:35:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:35:59 DEBUG --- http response --- 22:35:59 INFO [http_cmd]: http status code OK 22:35:59 INFO [loop_until]: kubectl --namespace=xlou top node 22:35:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:35:59 DEBUG --- status code --- 22:35:59 DEBUG --- status code --- 22:35:59 DEBUG --- http response --- 22:35:59 DEBUG --- status code --- 22:35:59 DEBUG --- http response --- 22:35:59 DEBUG --- http response --- 22:35:59 DEBUG --- http response --- 22:35:59 DEBUG --- http response --- 22:35:59 DEBUG --- http response --- 22:35:59 DEBUG --- http response --- 22:35:59 DEBUG --- http response --- 22:35:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:35:59 DEBUG --- http response --- 22:35:59 DEBUG --- http response --- 22:35:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:35:59 DEBUG --- http response --- 22:35:59 DEBUG --- http response --- 22:35:59 DEBUG --- http response --- 22:35:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:35:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 22:35:59 DEBUG --- http response --- 22:35:59 DEBUG --- http response --- 22:35:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:35:59 DEBUG --- status code --- 22:35:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:35:59 DEBUG --- http response --- 22:35:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:35:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:35:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:35:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:35:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:35:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:35:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 22:35:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:35:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:35:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:35:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:35:59 DEBUG --- http response --- 22:35:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 22:35:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:35:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:35:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:35:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:35:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:35:59 DEBUG ----- output ----- 22:35:59 DEBUG ----- output ----- 22:35:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 22:35:59 DEBUG ----- output ----- 22:35:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:35:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:35:59 DEBUG --- http response --- 22:35:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:35:59 DEBUG --- http response --- 22:35:59 DEBUG --- http response --- 22:35:59 DEBUG ----- output ----- 22:35:59 DEBUG ----- output ----- 22:35:59 DEBUG ----- output ----- 22:35:59 DEBUG ----- output ----- 22:35:59 DEBUG ----- output ----- 22:35:59 DEBUG ----- output ----- 22:35:59 DEBUG ----- output ----- 22:35:59 DEBUG ----- output ----- 22:35:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:35:59 DEBUG ----- output ----- 22:35:59 DEBUG ----- output ----- 22:35:59 DEBUG ----- output ----- 22:35:59 DEBUG ----- output ----- 22:35:59 DEBUG ----- output ----- 22:35:59 DEBUG ----- output ----- 22:35:59 DEBUG { 22:35:59 DEBUG { 22:35:59 DEBUG ----- output ----- 22:35:59 DEBUG { 22:35:59 DEBUG ----- output ----- 22:35:59 DEBUG --- http response --- 22:35:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:35:59 DEBUG ----- output ----- 22:35:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:35:59 DEBUG { 22:35:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:35:59 DEBUG { 22:35:59 DEBUG { 22:35:59 DEBUG { 22:35:59 DEBUG { 22:35:59 DEBUG { 22:35:59 DEBUG { 22:35:59 DEBUG { 22:35:59 DEBUG ----- output ----- 22:35:59 DEBUG { 22:35:59 DEBUG { 22:35:59 DEBUG { 22:35:59 DEBUG { 22:35:59 DEBUG { 22:35:59 DEBUG { 22:35:59 DEBUG "status": "success", 22:35:59 DEBUG "status": "success", 22:35:59 DEBUG { 22:35:59 DEBUG "status": "success", 22:35:59 DEBUG { 22:35:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:35:59 DEBUG ----- output ----- 22:35:59 DEBUG { 22:35:59 DEBUG ----- output ----- 22:35:59 DEBUG "status": "success", 22:35:59 DEBUG ----- output ----- 22:35:59 DEBUG "status": "success", 22:35:59 DEBUG "status": "success", 22:35:59 DEBUG "status": "success", 22:35:59 DEBUG "status": "success", 22:35:59 DEBUG "status": "success", 22:35:59 DEBUG "status": "success", 22:35:59 DEBUG "status": "success", 22:35:59 DEBUG { 22:35:59 DEBUG "status": "success", 22:35:59 DEBUG "status": "success", 22:35:59 DEBUG "status": "success", 22:35:59 DEBUG "status": "success", 22:35:59 DEBUG "status": "success", 22:35:59 DEBUG "status": "success", 22:35:59 DEBUG "data": { 22:35:59 DEBUG "data": { 22:35:59 DEBUG "status": "success", 22:35:59 DEBUG "data": { 22:35:59 DEBUG "status": "success", 22:35:59 DEBUG ----- output ----- 22:35:59 DEBUG { 22:35:59 DEBUG "status": "success", 22:35:59 DEBUG { 22:35:59 DEBUG "data": { 22:35:59 DEBUG { 22:35:59 DEBUG "data": { 22:35:59 DEBUG "data": { 22:35:59 DEBUG "resultType": "vector", 22:35:59 DEBUG "data": { 22:35:59 DEBUG "data": { 22:35:59 DEBUG "data": { 22:35:59 DEBUG "data": { 22:35:59 DEBUG "status": "success", 22:35:59 DEBUG "data": { 22:35:59 DEBUG "data": { 22:35:59 DEBUG "data": { 22:35:59 DEBUG "data": { 22:35:59 DEBUG "data": { 22:35:59 DEBUG "data": { 22:35:59 DEBUG "resultType": "vector", 22:35:59 DEBUG "resultType": "vector", 22:35:59 DEBUG "data": { 22:35:59 DEBUG "resultType": "vector", 22:35:59 DEBUG "data": { 22:35:59 DEBUG { 22:35:59 DEBUG "status": "success", 22:35:59 DEBUG "data": { 22:35:59 DEBUG "status": "success", 22:35:59 DEBUG "resultType": "vector", 22:35:59 DEBUG "status": "success", 22:35:59 DEBUG "resultType": "vector", 22:35:59 DEBUG "result": [ 22:35:59 DEBUG "resultType": "vector", 22:35:59 DEBUG "resultType": "vector", 22:35:59 DEBUG "resultType": "vector", 22:35:59 DEBUG "resultType": "vector", 22:35:59 DEBUG "data": { 22:35:59 DEBUG "resultType": "vector", 22:35:59 DEBUG "resultType": "vector", 22:35:59 DEBUG "resultType": "vector", 22:35:59 DEBUG "resultType": "vector", 22:35:59 DEBUG "resultType": "vector", 22:35:59 DEBUG "resultType": "vector", 22:35:59 DEBUG "result": [ 22:35:59 DEBUG "result": [ 22:35:59 DEBUG "resultType": "vector", 22:35:59 DEBUG "result": [ 22:35:59 DEBUG "resultType": "vector", 22:35:59 DEBUG "data": { 22:35:59 DEBUG "status": "success", 22:35:59 DEBUG "data": { 22:35:59 DEBUG "resultType": "vector", 22:35:59 DEBUG "data": { 22:35:59 DEBUG "result": [ 22:35:59 DEBUG "data": { 22:35:59 DEBUG "result": [ 22:35:59 DEBUG { 22:35:59 DEBUG "result": [ 22:35:59 DEBUG "result": [ 22:35:59 DEBUG "result": [ 22:35:59 DEBUG "result": [ 22:35:59 DEBUG "resultType": "vector", 22:35:59 DEBUG "result": [ 22:35:59 DEBUG "result": [ 22:35:59 DEBUG "result": [ 22:35:59 DEBUG "result": [ 22:35:59 DEBUG "result": [ 22:35:59 DEBUG "result": [ 22:35:59 DEBUG { 22:35:59 DEBUG { 22:35:59 DEBUG "metric": { 22:35:59 DEBUG { 22:35:59 DEBUG "result": [ 22:35:59 DEBUG "resultType": "vector", 22:35:59 DEBUG "data": { 22:35:59 DEBUG "resultType": "vector", 22:35:59 DEBUG "result": [ 22:35:59 DEBUG "resultType": "vector", 22:35:59 DEBUG { 22:35:59 DEBUG "resultType": "vector", 22:35:59 DEBUG { 22:35:59 DEBUG "metric": { 22:35:59 DEBUG { 22:35:59 DEBUG { 22:35:59 DEBUG { 22:35:59 DEBUG { 22:35:59 DEBUG "result": [ 22:35:59 DEBUG { 22:35:59 DEBUG { 22:35:59 DEBUG { 22:35:59 DEBUG { 22:35:59 DEBUG { 22:35:59 DEBUG { 22:35:59 DEBUG "metric": { 22:35:59 DEBUG "result": [ 22:35:59 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:35:59 DEBUG "metric": { 22:35:59 DEBUG { 22:35:59 DEBUG "result": [ 22:35:59 DEBUG "resultType": "vector", 22:35:59 DEBUG "result": [ 22:35:59 DEBUG { 22:35:59 DEBUG "result": [ 22:35:59 DEBUG "metric": { 22:35:59 DEBUG "result": [ 22:35:59 DEBUG "metric": { 22:35:59 DEBUG "pod": "am-555f4db595-fz8sn" 22:35:59 DEBUG "metric": { 22:35:59 DEBUG "metric": { 22:35:59 DEBUG "metric": { 22:35:59 DEBUG "metric": { 22:35:59 DEBUG { 22:35:59 DEBUG "metric": { 22:35:59 DEBUG "metric": { 22:35:59 DEBUG "metric": { 22:35:59 DEBUG "metric": { 22:35:59 DEBUG "metric": { 22:35:59 DEBUG "metric": { 22:35:59 DEBUG "pod": "am-555f4db595-fz8sn" 22:35:59 DEBUG { 22:35:59 DEBUG }, 22:35:59 DEBUG "pod": "ds-idrepo-2" 22:35:59 DEBUG "metric": { 22:35:59 DEBUG { 22:35:59 DEBUG "result": [ 22:35:59 DEBUG { 22:35:59 DEBUG "metric": { 22:35:59 DEBUG { 22:35:59 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:35:59 DEBUG { 22:35:59 DEBUG "pod": "ds-idrepo-2" 22:35:59 DEBUG }, 22:35:59 DEBUG "pod": "am-555f4db595-fz8sn" 22:35:59 DEBUG "pod": "am-555f4db595-fz8sn" 22:35:59 DEBUG "instance": "10.142.0.107:9100" 22:35:59 DEBUG "instance": "10.142.0.107:9100" 22:35:59 DEBUG "metric": { 22:35:59 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:35:59 DEBUG "instance": "10.142.0.107:9100" 22:35:59 DEBUG "pod": "am-555f4db595-fz8sn" 22:35:59 DEBUG "pod": "am-555f4db595-fz8sn" 22:35:59 DEBUG "instance": "10.142.0.107:9100" 22:35:59 DEBUG "pod": "ds-idrepo-2" 22:35:59 DEBUG }, 22:35:59 DEBUG "metric": { 22:35:59 DEBUG "value": [ 22:35:59 DEBUG }, 22:35:59 DEBUG "pod": "ds-idrepo-2" 22:35:59 DEBUG "metric": { 22:35:59 DEBUG { 22:35:59 DEBUG "metric": { 22:35:59 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:35:59 DEBUG "metric": { 22:35:59 DEBUG }, 22:35:59 DEBUG "metric": { 22:35:59 DEBUG }, 22:35:59 DEBUG "value": [ 22:35:59 DEBUG }, 22:35:59 DEBUG }, 22:35:59 DEBUG }, 22:35:59 DEBUG }, 22:35:59 DEBUG "pod": "ds-idrepo-2" 22:35:59 DEBUG }, 22:35:59 DEBUG }, 22:35:59 DEBUG }, 22:35:59 DEBUG }, 22:35:59 DEBUG }, 22:35:59 DEBUG }, 22:35:59 DEBUG "value": [ 22:35:59 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:35:59 DEBUG 22:35:59 DEBUG "value": [ 22:35:59 DEBUG }, 22:35:59 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:35:59 DEBUG "metric": { 22:35:59 DEBUG }, 22:35:59 DEBUG "pod": "am-555f4db595-fz8sn" 22:35:59 DEBUG "pod": "ds-idrepo-2" 22:35:59 DEBUG "value": [ 22:35:59 DEBUG "pod": "ds-idrepo-2" 22:35:59 DEBUG "value": [ 22:35:59 DEBUG 22:35:59 DEBUG "value": [ 22:35:59 DEBUG "value": [ 22:35:59 DEBUG "value": [ 22:35:59 DEBUG "value": [ 22:35:59 DEBUG }, 22:35:59 DEBUG "value": [ 22:35:59 DEBUG "value": [ 22:35:59 DEBUG "value": [ 22:35:59 DEBUG "value": [ 22:35:59 DEBUG "value": [ 22:35:59 DEBUG "value": [ 22:35:59 DEBUG 22:35:59 DEBUG }, 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG "value": [ 22:35:59 DEBUG }, 22:35:59 DEBUG "instance": "10.142.0.107:9100" 22:35:59 DEBUG "value": [ 22:35:59 DEBUG }, 22:35:59 DEBUG }, 22:35:59 DEBUG 22:35:59 DEBUG }, 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG "value": [ 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG "value": [ 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG "value": [ 22:35:59 DEBUG }, 22:35:59 DEBUG 22:35:59 DEBUG "value": [ 22:35:59 DEBUG "value": [ 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG [...] 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG "value": [ 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG "value": [ 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG [...] 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG [...] 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG [...] 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG [...] 22:35:59 DEBUG [...] 22:35:59 DEBUG 22:35:59 DEBUG [...] 22:35:59 DEBUG [...] 22:35:59 DEBUG [...] 22:35:59 DEBUG [...] 22:35:59 DEBUG 22:35:59 DEBUG [...] 22:35:59 DEBUG [...] 22:35:59 DEBUG [...] 22:35:59 DEBUG [...] 22:35:59 DEBUG [...] 22:35:59 DEBUG [...] 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG [...] 22:35:59 DEBUG 22:35:59 DEBUG [...] 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG [...] 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG [...] 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG [...] 22:35:59 DEBUG 22:35:59 DEBUG [...] 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG [...] 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG [...] 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG "pod": "am-555f4db595-kzxsp" 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG "instance": "10.142.0.43:9100" 22:35:59 DEBUG "pod": "am-555f4db595-kzxsp" 22:35:59 DEBUG "pod": "lodemon-bc756c647-cr7x6" 22:35:59 DEBUG 22:35:59 DEBUG "pod": "ds-idrepo-0" 22:35:59 DEBUG [...] 22:35:59 DEBUG "pod": "ds-idrepo-0" 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:35:59 DEBUG "pod": "ds-idrepo-0" 22:35:59 DEBUG }, 22:35:59 DEBUG "pod": "am-555f4db595-kzxsp" 22:35:59 DEBUG "pod": "am-555f4db595-kzxsp" 22:35:59 DEBUG "instance": "10.142.0.43:9100" 22:35:59 DEBUG "instance": "10.142.0.43:9100" 22:35:59 DEBUG 22:35:59 DEBUG "pod": "lodemon-bc756c647-cr7x6" 22:35:59 DEBUG "instance": "10.142.0.43:9100" 22:35:59 DEBUG "pod": "am-555f4db595-kzxsp" 22:35:59 DEBUG "pod": "am-555f4db595-kzxsp" 22:35:59 DEBUG 22:35:59 DEBUG }, 22:35:59 DEBUG }, 22:35:59 DEBUG }, 22:35:59 DEBUG "pod": "lodemon-bc756c647-cr7x6" 22:35:59 DEBUG }, 22:35:59 DEBUG 22:35:59 DEBUG }, 22:35:59 DEBUG 22:35:59 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:35:59 DEBUG 22:35:59 DEBUG 22:35:59 DEBUG "pod": "ds-cts-2" 22:35:59 DEBUG }, 22:35:59 DEBUG }, 22:35:59 DEBUG "value": [ 22:35:59 DEBUG }, 22:35:59 DEBUG }, 22:35:59 DEBUG }, 22:35:59 DEBUG }, 22:35:59 DEBUG "pod": "ds-idrepo-0" 22:35:59 DEBUG }, 22:35:59 DEBUG }, 22:35:59 DEBUG }, 22:35:59 DEBUG }, 22:35:59 DEBUG "pod": "ds-idrepo-0" 22:35:59 DEBUG "value": [ 22:35:59 DEBUG "value": [ 22:35:59 DEBUG "value": [ 22:35:59 DEBUG }, 22:35:59 DEBUG "value": [ 22:35:59 DEBUG 22:35:59 DEBUG "value": [ 22:35:59 DEBUG "pod": "lodemon-bc756c647-cr7x6" 22:35:59 DEBUG }, 22:35:59 DEBUG "pod": "am-555f4db595-kzxsp" 22:35:59 DEBUG 22:35:59 DEBUG }, 22:35:59 DEBUG "value": [ 22:35:59 DEBUG "value": [ 22:35:59 DEBUG 1713303359, 22:35:59 DEBUG "value": [ 22:35:59 DEBUG "value": [ 22:35:59 DEBUG "value": [ 22:35:59 DEBUG "value": [ 22:35:59 DEBUG }, 22:35:59 DEBUG "value": [ 22:35:59 DEBUG "value": [ 22:35:59 DEBUG "value": [ 22:35:59 DEBUG "value": [ 22:35:59 DEBUG }, 22:35:59 DEBUG 1713303359, 22:35:59 DEBUG 1713303359, 22:35:59 DEBUG 1713303359, 22:35:59 DEBUG "value": [ 22:35:59 DEBUG 1713303359, 22:35:59 DEBUG 22:35:59 DEBUG 1713303359, 22:35:59 DEBUG }, 22:35:59 DEBUG "value": [ 22:35:59 DEBUG 1713303359, 22:35:59 DEBUG "pod": "ds-idrepo-0" 22:35:59 DEBUG "value": [ 22:35:59 DEBUG 1713303359, 22:35:59 DEBUG 1713303359, 22:35:59 DEBUG "2899289.5" 22:35:59 DEBUG 1713303359, 22:35:59 DEBUG 1713303359, 22:35:59 DEBUG 1713303359, 22:35:59 DEBUG 1713303359, 22:35:59 DEBUG "value": [ 22:35:59 DEBUG 1713303359, 22:35:59 DEBUG 1713303359, 22:35:59 DEBUG 1713303359, 22:35:59 DEBUG 1713303359, 22:35:59 DEBUG "value": [ 22:35:59 DEBUG "45371881.56666666" 22:35:59 DEBUG "0" 22:35:59 DEBUG "4261.949665084217" 22:35:59 DEBUG 1713303359, 22:35:59 DEBUG "0" 22:35:59 DEBUG "instance": "10.142.0.43:9100" 22:35:59 DEBUG "0" 22:35:59 DEBUG "value": [ 22:35:59 DEBUG }, 22:35:59 DEBUG "6.2914037497499065" 22:35:59 DEBUG }, 22:35:59 DEBUG 1713303359, 22:35:59 DEBUG "667910144" 22:35:59 DEBUG "0" 22:35:59 DEBUG ] 22:35:59 DEBUG "1991.4666666666667" 22:35:59 DEBUG ] 22:35:59 DEBUG "45381243.03333333" 22:35:59 DEBUG "19962745856" 22:35:59 DEBUG 1713303359, 22:35:59 DEBUG "0" 22:35:59 DEBUG "0.0006000000000000227" 22:35:59 DEBUG "1991.5" 22:35:59 DEBUG "0" 22:35:59 DEBUG 1713303359, 22:35:59 DEBUG ] 22:35:59 DEBUG ] 22:35:59 DEBUG ] 22:35:59 DEBUG "0" 22:35:59 DEBUG ] 22:35:59 DEBUG }, 22:35:59 DEBUG ] 22:35:59 DEBUG 1713303359, 22:35:59 DEBUG "value": [ 22:35:59 DEBUG ] 22:35:59 DEBUG "value": [ 22:35:59 DEBUG "0" 22:35:59 DEBUG ] 22:35:59 DEBUG ] 22:35:59 DEBUG } 22:35:59 DEBUG "1991.3" 22:35:59 DEBUG } 22:35:59 DEBUG ] 22:35:59 DEBUG ] 22:35:59 DEBUG "0" 22:35:59 DEBUG ] 22:35:59 DEBUG ] 22:35:59 DEBUG ] 22:35:59 DEBUG ] 22:35:59 DEBUG "0" 22:35:59 DEBUG } 22:35:59 DEBUG } 22:35:59 DEBUG } 22:35:59 DEBUG ] 22:35:59 DEBUG } 22:35:59 DEBUG "value": [ 22:35:59 DEBUG } 22:35:59 DEBUG "0.001301651817649821" 22:35:59 DEBUG 1713303359, 22:35:59 DEBUG } 22:35:59 DEBUG 1713303359, 22:35:59 DEBUG ] 22:35:59 DEBUG } 22:35:59 DEBUG } 22:35:59 DEBUG ] 22:35:59 DEBUG ] 22:35:59 DEBUG ] 22:35:59 DEBUG } 22:35:59 DEBUG } 22:35:59 DEBUG ] 22:35:59 DEBUG } 22:35:59 DEBUG } 22:35:59 DEBUG } 22:35:59 DEBUG } 22:35:59 DEBUG ] 22:35:59 DEBUG ] 22:35:59 DEBUG ] 22:35:59 DEBUG ] 22:35:59 DEBUG } 22:35:59 DEBUG ] 22:35:59 DEBUG 1713303359, 22:35:59 DEBUG ] 22:35:59 DEBUG ] 22:35:59 DEBUG "0.025" 22:35:59 DEBUG ] 22:35:59 DEBUG "0" 22:35:59 DEBUG } 22:35:59 DEBUG ] 22:35:59 DEBUG ] 22:35:59 DEBUG } 22:35:59 DEBUG } 22:35:59 DEBUG } 22:35:59 DEBUG ] 22:35:59 DEBUG ] 22:35:59 DEBUG } 22:35:59 DEBUG ] 22:35:59 DEBUG ] 22:35:59 DEBUG ] 22:35:59 DEBUG ] 22:35:59 DEBUG } 22:35:59 DEBUG } 22:35:59 DEBUG } 22:35:59 DEBUG } 22:35:59 DEBUG ] 22:35:59 DEBUG } 22:35:59 DEBUG "0" 22:35:59 DEBUG } 22:35:59 DEBUG } 22:35:59 DEBUG ] 22:35:59 DEBUG } 22:35:59 DEBUG ] 22:35:59 DEBUG ] 22:35:59 DEBUG } 22:35:59 DEBUG } 22:35:59 DEBUG } 22:35:59 DEBUG ] 22:35:59 DEBUG } 22:35:59 DEBUG } 22:35:59 DEBUG } 22:35:59 DEBUG ] 22:35:59 DEBUG } 22:35:59 DEBUG } 22:35:59 DEBUG } 22:35:59 DEBUG } 22:35:59 DEBUG ] 22:35:59 DEBUG } 22:35:59 DEBUG } 22:35:59 DEBUG } 22:35:59 DEBUG } 22:35:59 DEBUG } 22:35:59 DEBUG ] 22:35:59 DEBUG } 22:35:59 DEBUG ] 22:35:59 DEBUG } 22:35:59 DEBUG } 22:35:59 DEBUG } 22:35:59 DEBUG } 22:35:59 DEBUG } 22:35:59 DEBUG } 22:35:59 DEBUG } 22:35:59 DEBUG } 22:35:59 DEBUG } 22:35:59 DEBUG } 22:35:59 DEBUG } 22:35:59 DEBUG } 22:35:59 DEBUG } 22:35:59 DEBUG } 22:35:59 DEBUG } 22:35:59 DEBUG } 22:35:59 DEBUG } 22:35:59 DEBUG } 22:35:59 DEBUG ] 22:35:59 DEBUG ] 22:35:59 DEBUG } 22:35:59 DEBUG } 22:35:59 DEBUG } 22:35:59 DEBUG } 22:35:59 DEBUG ] 22:35:59 DEBUG } 22:35:59 DEBUG } 22:35:59 DEBUG } 22:35:59 DEBUG } 22:35:59 DEBUG } 22:35:59 DEBUG } 22:35:59 DEBUG } 22:35:59 INFO [loop_until]: OK (rc = 0) 22:35:59 DEBUG --- stdout --- 22:35:59 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 65m 0% 5093Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7506m 25% 14810Mi 13% gke-xlou-cdm-default-pool-1b0769c8-nhlf 9424m 31% 10835Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 82m 0% 2636Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 4759m 15% 6498Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 8792m 29% 16672Mi 14% gke-xlou-cdm-default-pool-1b0769c8-xnmp 60m 0% 3145Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 9517m 31% 10376Mi 9% gke-xlou-cdm-ds-0f601751-hmm7 4664m 15% 5624Mi 5% gke-xlou-cdm-ds-0f601751-j488 7489m 25% 14150Mi 12% gke-xlou-cdm-ds-0f601751-mc5d 10598m 35% 3347Mi 2% gke-xlou-cdm-ds-0f601751-nj6q 9172m 30% 10436Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 4757m 15% 7350Mi 6% gke-xlou-cdm-frontend-a8771548-i82n 6438m 40% 5954Mi 10% 22:35:59 DEBUG --- stderr --- 22:35:59 DEBUG 22:36:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713303416" 22:36:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303416" 22:36:56 ERROR [http_cmd]: ERROR 22:36:56 ERROR --- status code --- 22:36:56 ERROR http status code is 400 (expected [200, 401, 403, 404]) 22:36:56 ERROR --- http response --- 22:36:56 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 22:36:56 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 22:36:56 INFO [http_cmd]: http status code OK 22:36:56 DEBUG --- status code --- 22:36:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:36:56 DEBUG --- http response --- 22:36:56 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 22:36:58 INFO 22:36:58 INFO [loop_until]: kubectl --namespace=xlou top pods 22:36:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:36:58 INFO [loop_until]: OK (rc = 0) 22:36:58 DEBUG --- stdout --- 22:36:58 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 9Mi am-555f4db595-4lbss 9425m 8599Mi am-555f4db595-fz8sn 9364m 8610Mi am-555f4db595-kzxsp 9434m 8565Mi ds-cts-0 8126m 12172Mi ds-cts-1 7543m 12262Mi ds-cts-2 7660m 12224Mi ds-idrepo-0 4765m 5022Mi ds-idrepo-1 4542m 2875Mi ds-idrepo-2 4430m 3167Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 4m 1207Mi idm-b85f9d978-rl76f 4m 1226Mi lodemon-bc756c647-cr7x6 2m 63Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-7c9b667974-9mr2s 6220m 544Mi 22:36:58 DEBUG --- stderr --- 22:36:58 DEBUG 22:36:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303419" 22:36:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303419" 22:36:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303419" 22:36:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303419" 22:36:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303419" 22:36:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303419" 22:36:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713303419" 22:36:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303419" 22:36:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303419" 22:36:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713303419" 22:36:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303419" 22:36:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303419" 22:36:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303419" 22:36:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303419" 22:36:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303419" 22:36:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303419" 22:36:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303419" 22:36:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303419" 22:36:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713303419" 22:36:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303419" 22:36:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303419" 22:36:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303419" 22:36:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303419" 22:36:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713303419" 22:36:59 INFO 22:36:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303419" 22:36:59 INFO [http_cmd]: http status code OK 22:36:59 INFO [http_cmd]: http status code OK 22:36:59 INFO [http_cmd]: http status code OK 22:36:59 INFO [http_cmd]: http status code OK 22:36:59 INFO [http_cmd]: http status code OK 22:36:59 INFO [http_cmd]: http status code OK 22:36:59 INFO [http_cmd]: http status code OK 22:36:59 INFO [http_cmd]: http status code OK 22:36:59 INFO [http_cmd]: http status code OK 22:36:59 INFO [http_cmd]: http status code OK 22:36:59 INFO [http_cmd]: http status code OK 22:36:59 INFO [http_cmd]: http status code OK 22:36:59 INFO [http_cmd]: http status code OK 22:36:59 INFO [http_cmd]: http status code OK 22:36:59 INFO [http_cmd]: http status code OK 22:36:59 INFO [http_cmd]: http status code OK 22:36:59 INFO [http_cmd]: http status code OK 22:36:59 INFO [http_cmd]: http status code OK 22:36:59 INFO [http_cmd]: http status code OK 22:36:59 INFO [http_cmd]: http status code OK 22:36:59 INFO [loop_until]: kubectl --namespace=xlou top node 22:36:59 DEBUG --- status code --- 22:36:59 DEBUG --- status code --- 22:36:59 DEBUG --- status code --- 22:36:59 DEBUG --- status code --- 22:36:59 DEBUG --- status code --- 22:36:59 DEBUG --- status code --- 22:36:59 DEBUG --- status code --- 22:36:59 DEBUG --- status code --- 22:36:59 DEBUG --- status code --- 22:36:59 DEBUG --- status code --- 22:36:59 DEBUG --- status code --- 22:36:59 DEBUG --- status code --- 22:36:59 DEBUG --- status code --- 22:36:59 DEBUG --- status code --- 22:36:59 INFO [http_cmd]: http status code OK 22:36:59 INFO [http_cmd]: http status code OK 22:36:59 DEBUG --- status code --- 22:36:59 DEBUG --- status code --- 22:36:59 DEBUG --- status code --- 22:36:59 DEBUG --- status code --- 22:36:59 DEBUG --- status code --- 22:36:59 DEBUG --- status code --- 22:36:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:36:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:36:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:36:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:36:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:36:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:36:59 INFO [http_cmd]: http status code OK 22:36:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:36:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:36:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:36:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:36:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:36:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:36:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:36:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:36:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:36:59 DEBUG --- status code --- 22:36:59 DEBUG --- status code --- 22:36:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:36:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:36:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:36:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:36:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:36:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:36:59 INFO [http_cmd]: http status code OK 22:36:59 DEBUG --- status code --- 22:36:59 DEBUG --- http response --- 22:36:59 DEBUG --- http response --- 22:36:59 INFO [http_cmd]: http status code OK 22:36:59 DEBUG --- http response --- 22:36:59 DEBUG --- http response --- 22:36:59 DEBUG --- status code --- 22:36:59 DEBUG --- http response --- 22:36:59 DEBUG --- http response --- 22:36:59 DEBUG --- http response --- 22:36:59 DEBUG --- http response --- 22:36:59 DEBUG --- http response --- 22:36:59 DEBUG --- http response --- 22:36:59 DEBUG --- http response --- 22:36:59 DEBUG --- http response --- 22:36:59 DEBUG --- http response --- 22:36:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:36:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:36:59 DEBUG --- http response --- 22:36:59 DEBUG --- http response --- 22:36:59 DEBUG --- http response --- 22:36:59 DEBUG --- http response --- 22:36:59 DEBUG --- http response --- 22:36:59 DEBUG --- http response --- 22:36:59 DEBUG --- http response --- 22:36:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:36:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:36:59 DEBUG --- status code --- 22:36:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:36:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:36:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:36:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:36:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:36:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:36:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:36:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:36:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:36:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:36:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:36:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:36:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:36:59 DEBUG --- http response --- 22:36:59 DEBUG --- http response --- 22:36:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 22:36:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:36:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:36:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:36:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:36:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:36:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:36:59 DEBUG --- http response --- 22:36:59 DEBUG ----- output ----- 22:36:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:36:59 DEBUG ----- output ----- 22:36:59 DEBUG ----- output ----- 22:36:59 DEBUG ----- output ----- 22:36:59 DEBUG --- http response --- 22:36:59 DEBUG ----- output ----- 22:36:59 DEBUG ----- output ----- 22:36:59 DEBUG ----- output ----- 22:36:59 DEBUG ----- output ----- 22:36:59 DEBUG ----- output ----- 22:36:59 DEBUG ----- output ----- 22:36:59 DEBUG ----- output ----- 22:36:59 DEBUG ----- output ----- 22:36:59 DEBUG ----- output ----- 22:36:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:36:59 DEBUG ----- output ----- 22:36:59 DEBUG { 22:36:59 DEBUG "status": "success", 22:36:59 DEBUG ----- output ----- 22:36:59 DEBUG ----- output ----- 22:36:59 DEBUG ----- output ----- 22:36:59 DEBUG ----- output ----- 22:36:59 DEBUG ----- output ----- 22:36:59 DEBUG { 22:36:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:36:59 DEBUG --- http response --- 22:36:59 DEBUG { 22:36:59 DEBUG { 22:36:59 DEBUG { 22:36:59 DEBUG { 22:36:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:36:59 DEBUG { 22:36:59 DEBUG { 22:36:59 DEBUG { 22:36:59 DEBUG { 22:36:59 DEBUG { 22:36:59 DEBUG "status": "success", 22:36:59 DEBUG { 22:36:59 DEBUG { 22:36:59 DEBUG ----- output ----- 22:36:59 DEBUG "data": { 22:36:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 22:36:59 DEBUG { 22:36:59 DEBUG { 22:36:59 DEBUG { 22:36:59 DEBUG { 22:36:59 DEBUG { 22:36:59 DEBUG "status": "success", 22:36:59 DEBUG ----- output ----- 22:36:59 DEBUG ----- output ----- 22:36:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:36:59 DEBUG "status": "success", 22:36:59 DEBUG "status": "success", 22:36:59 DEBUG "status": "success", 22:36:59 DEBUG "status": "success", 22:36:59 DEBUG ----- output ----- 22:36:59 DEBUG "status": "success", 22:36:59 DEBUG "status": "success", 22:36:59 DEBUG "status": "success", 22:36:59 DEBUG "status": "success", 22:36:59 DEBUG { 22:36:59 DEBUG "data": { 22:36:59 DEBUG "status": "success", 22:36:59 DEBUG "status": "success", 22:36:59 DEBUG { 22:36:59 DEBUG "resultType": "vector", 22:36:59 DEBUG ----- output ----- 22:36:59 DEBUG "status": "success", 22:36:59 DEBUG "status": "success", 22:36:59 DEBUG "status": "success", 22:36:59 DEBUG "status": "success", 22:36:59 DEBUG "status": "success", 22:36:59 DEBUG "data": { 22:36:59 DEBUG { 22:36:59 DEBUG { 22:36:59 DEBUG ----- output ----- 22:36:59 DEBUG "data": { 22:36:59 DEBUG "data": { 22:36:59 DEBUG "resultType": "vector", 22:36:59 DEBUG "data": { 22:36:59 DEBUG { 22:36:59 DEBUG "data": { 22:36:59 DEBUG "data": { 22:36:59 DEBUG "resultType": "vector", 22:36:59 DEBUG "data": { 22:36:59 DEBUG "status": "success", 22:36:59 DEBUG "resultType": "vector", 22:36:59 DEBUG "data": { 22:36:59 DEBUG "data": { 22:36:59 DEBUG "status": "success", 22:36:59 DEBUG "result": [ 22:36:59 DEBUG { 22:36:59 DEBUG "data": { 22:36:59 DEBUG "data": { 22:36:59 DEBUG "data": { 22:36:59 DEBUG "data": { 22:36:59 DEBUG "data": { 22:36:59 DEBUG "resultType": "vector", 22:36:59 DEBUG "status": "success", 22:36:59 DEBUG "status": "success", 22:36:59 DEBUG { 22:36:59 DEBUG "resultType": "vector", 22:36:59 DEBUG "result": [ 22:36:59 DEBUG "resultType": "vector", 22:36:59 DEBUG "data": { 22:36:59 DEBUG "status": "success", 22:36:59 DEBUG "resultType": "vector", 22:36:59 DEBUG "data": { 22:36:59 DEBUG "result": [ 22:36:59 DEBUG "resultType": "vector", 22:36:59 DEBUG "data": { 22:36:59 DEBUG "result": [ 22:36:59 DEBUG "resultType": "vector", 22:36:59 DEBUG "resultType": "vector", 22:36:59 DEBUG "data": { 22:36:59 DEBUG { 22:36:59 DEBUG "status": "success", 22:36:59 DEBUG "resultType": "vector", 22:36:59 DEBUG "resultType": "vector", 22:36:59 DEBUG "resultType": "vector", 22:36:59 DEBUG "resultType": "vector", 22:36:59 DEBUG "resultType": "vector", 22:36:59 DEBUG "result": [ 22:36:59 DEBUG "data": { 22:36:59 DEBUG "data": { 22:36:59 DEBUG "status": "success", 22:36:59 DEBUG "result": [ 22:36:59 DEBUG { 22:36:59 DEBUG "result": [ 22:36:59 DEBUG "resultType": "vector", 22:36:59 DEBUG "data": { 22:36:59 DEBUG "result": [ 22:36:59 DEBUG "resultType": "vector", 22:36:59 DEBUG { 22:36:59 DEBUG "result": [ 22:36:59 DEBUG "resultType": "vector", 22:36:59 DEBUG { 22:36:59 DEBUG "result": [ 22:36:59 DEBUG "result": [ 22:36:59 DEBUG "resultType": "vector", 22:36:59 DEBUG "metric": { 22:36:59 DEBUG "data": { 22:36:59 DEBUG "result": [ 22:36:59 DEBUG "result": [ 22:36:59 DEBUG "result": [ 22:36:59 DEBUG "result": [ 22:36:59 DEBUG "result": [ 22:36:59 DEBUG { 22:36:59 DEBUG "resultType": "vector", 22:36:59 DEBUG "resultType": "vector", 22:36:59 DEBUG "data": { 22:36:59 DEBUG { 22:36:59 DEBUG "metric": { 22:36:59 DEBUG { 22:36:59 DEBUG "result": [ 22:36:59 DEBUG "resultType": "vector", 22:36:59 DEBUG { 22:36:59 DEBUG "result": [ 22:36:59 DEBUG "metric": { 22:36:59 DEBUG { 22:36:59 DEBUG "result": [ 22:36:59 DEBUG "metric": { 22:36:59 DEBUG { 22:36:59 DEBUG { 22:36:59 DEBUG "result": [ 22:36:59 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:36:59 DEBUG "resultType": "vector", 22:36:59 DEBUG { 22:36:59 DEBUG { 22:36:59 DEBUG { 22:36:59 DEBUG { 22:36:59 DEBUG { 22:36:59 DEBUG "metric": { 22:36:59 DEBUG "result": [ 22:36:59 DEBUG "result": [ 22:36:59 DEBUG "resultType": "vector", 22:36:59 DEBUG "metric": { 22:36:59 DEBUG "pod": "am-555f4db595-fz8sn" 22:36:59 DEBUG "metric": { 22:36:59 DEBUG { 22:36:59 DEBUG "result": [ 22:36:59 DEBUG "metric": { 22:36:59 DEBUG "pod": "am-555f4db595-fz8sn" 22:36:59 DEBUG "pod": "am-555f4db595-fz8sn" 22:36:59 DEBUG "metric": { 22:36:59 DEBUG { 22:36:59 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:36:59 DEBUG "metric": { 22:36:59 DEBUG "metric": { 22:36:59 DEBUG { 22:36:59 DEBUG "metric": { 22:36:59 DEBUG "result": [ 22:36:59 DEBUG "metric": { 22:36:59 DEBUG "metric": { 22:36:59 DEBUG "metric": { 22:36:59 DEBUG "metric": { 22:36:59 DEBUG "metric": { 22:36:59 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:36:59 DEBUG { 22:36:59 DEBUG { 22:36:59 DEBUG "result": [ 22:36:59 DEBUG "instance": "10.142.0.107:9100" 22:36:59 DEBUG }, 22:36:59 DEBUG "pod": "ds-idrepo-2" 22:36:59 DEBUG "metric": { 22:36:59 DEBUG { 22:36:59 DEBUG }, 22:36:59 DEBUG }, 22:36:59 DEBUG { 22:36:59 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:36:59 DEBUG "metric": { 22:36:59 DEBUG }, 22:36:59 DEBUG "instance": "10.142.0.107:9100" 22:36:59 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:36:59 DEBUG }, 22:36:59 DEBUG "pod": "ds-idrepo-2" 22:36:59 DEBUG { 22:36:59 DEBUG "instance": "10.142.0.107:9100" 22:36:59 DEBUG "pod": "am-555f4db595-fz8sn" 22:36:59 DEBUG "pod": "ds-idrepo-2" 22:36:59 DEBUG "pod": "ds-idrepo-2" 22:36:59 DEBUG "pod": "am-555f4db595-fz8sn" 22:36:59 DEBUG }, 22:36:59 DEBUG "metric": { 22:36:59 DEBUG "metric": { 22:36:59 DEBUG { 22:36:59 DEBUG }, 22:36:59 DEBUG "value": [ 22:36:59 DEBUG }, 22:36:59 DEBUG "pod": "ds-idrepo-2" 22:36:59 DEBUG "metric": { 22:36:59 DEBUG "value": [ 22:36:59 DEBUG "value": [ 22:36:59 DEBUG "metric": { 22:36:59 DEBUG }, 22:36:59 DEBUG "pod": "am-555f4db595-fz8sn" 22:36:59 DEBUG "value": [ 22:36:59 DEBUG }, 22:36:59 DEBUG }, 22:36:59 DEBUG "value": [ 22:36:59 DEBUG }, 22:36:59 DEBUG "metric": { 22:36:59 DEBUG }, 22:36:59 DEBUG }, 22:36:59 DEBUG }, 22:36:59 DEBUG }, 22:36:59 DEBUG }, 22:36:59 DEBUG "value": [ 22:36:59 DEBUG "pod": "am-555f4db595-fz8sn" 22:36:59 DEBUG "instance": "10.142.0.107:9100" 22:36:59 DEBUG "metric": { 22:36:59 DEBUG "value": [ 22:36:59 DEBUG 22:36:59 DEBUG "value": [ 22:36:59 DEBUG }, 22:36:59 DEBUG "pod": "ds-idrepo-2" 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG "pod": "ds-idrepo-2" 22:36:59 DEBUG "value": [ 22:36:59 DEBUG }, 22:36:59 DEBUG 22:36:59 DEBUG "value": [ 22:36:59 DEBUG "value": [ 22:36:59 DEBUG 22:36:59 DEBUG "value": [ 22:36:59 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:36:59 DEBUG "value": [ 22:36:59 DEBUG "value": [ 22:36:59 DEBUG "value": [ 22:36:59 DEBUG "value": [ 22:36:59 DEBUG "value": [ 22:36:59 DEBUG 22:36:59 DEBUG }, 22:36:59 DEBUG }, 22:36:59 DEBUG "instance": "10.142.0.107:9100" 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG "value": [ 22:36:59 DEBUG }, 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG }, 22:36:59 DEBUG 22:36:59 DEBUG "value": [ 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG }, 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG "value": [ 22:36:59 DEBUG "value": [ 22:36:59 DEBUG }, 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG "value": [ 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG "value": [ 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG "value": [ 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG "value": [ 22:36:59 DEBUG 22:36:59 DEBUG [...] 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG [...] 22:36:59 DEBUG [...] 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG [...] 22:36:59 DEBUG 22:36:59 DEBUG [...] 22:36:59 DEBUG [...] 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG [...] 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG [...] 22:36:59 DEBUG 22:36:59 DEBUG [...] 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG [...] 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG [...] 22:36:59 DEBUG 22:36:59 DEBUG [...] 22:36:59 DEBUG [...] 22:36:59 DEBUG [...] 22:36:59 DEBUG [...] 22:36:59 DEBUG [...] 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG [...] 22:36:59 DEBUG 22:36:59 DEBUG [...] 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG [...] 22:36:59 DEBUG 22:36:59 DEBUG [...] 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG [...] 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG [...] 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG [...] 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG [...] 22:36:59 DEBUG "pod": "overseer-0-7c9b667974-9mr2s" 22:36:59 DEBUG 22:36:59 DEBUG [...] 22:36:59 DEBUG 22:36:59 DEBUG "pod": "am-555f4db595-kzxsp" 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG "pod": "am-555f4db595-kzxsp" 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:36:59 DEBUG 22:36:59 DEBUG "instance": "10.142.0.43:9100" 22:36:59 DEBUG "pod": "overseer-0-7c9b667974-9mr2s" 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG }, 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG "instance": "10.142.0.43:9100" 22:36:59 DEBUG }, 22:36:59 DEBUG "pod": "ds-idrepo-0" 22:36:59 DEBUG 22:36:59 DEBUG "pod": "am-555f4db595-kzxsp" 22:36:59 DEBUG 22:36:59 DEBUG }, 22:36:59 DEBUG 22:36:59 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:36:59 DEBUG 22:36:59 DEBUG }, 22:36:59 DEBUG 22:36:59 DEBUG }, 22:36:59 DEBUG }, 22:36:59 DEBUG "pod": "ds-idrepo-0" 22:36:59 DEBUG 22:36:59 DEBUG "instance": "10.142.0.43:9100" 22:36:59 DEBUG "pod": "am-555f4db595-kzxsp" 22:36:59 DEBUG "pod": "ds-idrepo-0" 22:36:59 DEBUG "pod": "ds-cts-2" 22:36:59 DEBUG "pod": "am-555f4db595-kzxsp" 22:36:59 DEBUG 22:36:59 DEBUG "value": [ 22:36:59 DEBUG 22:36:59 DEBUG 22:36:59 DEBUG }, 22:36:59 DEBUG "value": [ 22:36:59 DEBUG }, 22:36:59 DEBUG "pod": "ds-idrepo-0" 22:36:59 DEBUG }, 22:36:59 DEBUG "pod": "ds-idrepo-0" 22:36:59 DEBUG "value": [ 22:36:59 DEBUG 22:36:59 DEBUG }, 22:36:59 DEBUG "pod": "am-555f4db595-kzxsp" 22:36:59 DEBUG "value": [ 22:36:59 DEBUG "pod": "overseer-0-7c9b667974-9mr2s" 22:36:59 DEBUG "value": [ 22:36:59 DEBUG "value": [ 22:36:59 DEBUG }, 22:36:59 DEBUG 22:36:59 DEBUG }, 22:36:59 DEBUG }, 22:36:59 DEBUG }, 22:36:59 DEBUG }, 22:36:59 DEBUG }, 22:36:59 DEBUG 22:36:59 DEBUG 1713303419, 22:36:59 DEBUG "pod": "am-555f4db595-kzxsp" 22:36:59 DEBUG 22:36:59 DEBUG "value": [ 22:36:59 DEBUG 1713303419, 22:36:59 DEBUG "value": [ 22:36:59 DEBUG }, 22:36:59 DEBUG "value": [ 22:36:59 DEBUG }, 22:36:59 DEBUG 1713303419, 22:36:59 DEBUG "pod": "ds-idrepo-0" 22:36:59 DEBUG "value": [ 22:36:59 DEBUG }, 22:36:59 DEBUG 1713303419, 22:36:59 DEBUG }, 22:36:59 DEBUG 1713303419, 22:36:59 DEBUG 1713303419, 22:36:59 DEBUG "value": [ 22:36:59 DEBUG "pod": "overseer-0-7c9b667974-9mr2s" 22:36:59 DEBUG "value": [ 22:36:59 DEBUG "value": [ 22:36:59 DEBUG "value": [ 22:36:59 DEBUG "value": [ 22:36:59 DEBUG "value": [ 22:36:59 DEBUG "instance": "10.142.0.43:9100" 22:36:59 DEBUG "0" 22:36:59 DEBUG }, 22:36:59 DEBUG "value": [ 22:36:59 DEBUG 1713303419, 22:36:59 DEBUG "3019026.5" 22:36:59 DEBUG 1713303419, 22:36:59 DEBUG "value": [ 22:36:59 DEBUG 1713303419, 22:36:59 DEBUG "value": [ 22:36:59 DEBUG "0" 22:36:59 DEBUG }, 22:36:59 DEBUG 1713303419, 22:36:59 DEBUG "value": [ 22:36:59 DEBUG "6.296402255478154" 22:36:59 DEBUG "value": [ 22:36:59 DEBUG "0.0005333333333330377" 22:36:59 DEBUG "22512.799669558033" 22:36:59 DEBUG 1713303419, 22:36:59 DEBUG }, 22:36:59 DEBUG 1713303419, 22:36:59 DEBUG 1713303419, 22:36:59 DEBUG 1713303419, 22:36:59 DEBUG 1713303419, 22:36:59 DEBUG 1713303419, 22:36:59 DEBUG }, 22:36:59 DEBUG 1713303419, 22:36:59 DEBUG "instance": "10.142.0.43:9100" 22:36:59 DEBUG ] 22:36:59 DEBUG "45271649.06666666" 22:36:59 DEBUG ] 22:36:59 DEBUG "0" 22:36:59 DEBUG 1713303419, 22:36:59 DEBUG "0" 22:36:59 DEBUG 1713303419, 22:36:59 DEBUG ] 22:36:59 DEBUG "value": [ 22:36:59 DEBUG "665018368" 22:36:59 DEBUG 1713303419, 22:36:59 DEBUG ] 22:36:59 DEBUG 1713303419, 22:36:59 DEBUG ] 22:36:59 DEBUG ] 22:36:59 DEBUG "0" 22:36:59 DEBUG "value": [ 22:36:59 DEBUG "19963649024" 22:36:59 DEBUG "2001.7" 22:36:59 DEBUG "0" 22:36:59 DEBUG "0" 22:36:59 DEBUG "2001.7333333333333" 22:36:59 DEBUG "value": [ 22:36:59 DEBUG "0.025500000000000002" 22:36:59 DEBUG }, 22:36:59 DEBUG } 22:36:59 DEBUG ] 22:36:59 DEBUG } 22:36:59 DEBUG ] 22:36:59 DEBUG "0" 22:36:59 DEBUG ] 22:36:59 DEBUG "0" 22:36:59 DEBUG } 22:36:59 DEBUG 1713303419, 22:36:59 DEBUG ] 22:36:59 DEBUG "2001.6" 22:36:59 DEBUG } 22:36:59 DEBUG "0" 22:36:59 DEBUG } 22:36:59 DEBUG } 22:36:59 DEBUG ] 22:36:59 DEBUG 1713303419, 22:36:59 DEBUG ] 22:36:59 DEBUG ] 22:36:59 DEBUG ] 22:36:59 DEBUG ] 22:36:59 DEBUG ] 22:36:59 DEBUG 1713303419, 22:36:59 DEBUG ] 22:36:59 DEBUG "value": [ 22:36:59 DEBUG ] 22:36:59 DEBUG } 22:36:59 DEBUG ] 22:36:59 DEBUG } 22:36:59 DEBUG ] 22:36:59 DEBUG } 22:36:59 DEBUG ] 22:36:59 DEBUG ] 22:36:59 DEBUG "0" 22:36:59 DEBUG } 22:36:59 DEBUG ] 22:36:59 DEBUG ] 22:36:59 DEBUG ] 22:36:59 DEBUG ] 22:36:59 DEBUG ] 22:36:59 DEBUG } 22:36:59 DEBUG "4.066338383800992" 22:36:59 DEBUG } 22:36:59 DEBUG } 22:36:59 DEBUG } 22:36:59 DEBUG } 22:36:59 DEBUG } 22:36:59 DEBUG "45281170.9" 22:36:59 DEBUG } 22:36:59 DEBUG 1713303419, 22:36:59 DEBUG } 22:36:59 DEBUG ] 22:36:59 DEBUG } 22:36:59 DEBUG ] 22:36:59 DEBUG } 22:36:59 DEBUG ] 22:36:59 DEBUG } 22:36:59 DEBUG } 22:36:59 DEBUG ] 22:36:59 DEBUG ] 22:36:59 DEBUG } 22:36:59 DEBUG } 22:36:59 DEBUG } 22:36:59 DEBUG } 22:36:59 DEBUG } 22:36:59 DEBUG ] 22:36:59 DEBUG ] 22:36:59 DEBUG ] 22:36:59 DEBUG ] 22:36:59 DEBUG ] 22:36:59 DEBUG ] 22:36:59 DEBUG ] 22:36:59 DEBUG ] 22:36:59 DEBUG ] 22:36:59 DEBUG "0.00033333333333333365" 22:36:59 DEBUG } 22:36:59 DEBUG } 22:36:59 DEBUG } 22:36:59 DEBUG } 22:36:59 DEBUG ] 22:36:59 DEBUG } 22:36:59 DEBUG ] 22:36:59 DEBUG } 22:36:59 DEBUG } 22:36:59 DEBUG } 22:36:59 DEBUG ] 22:36:59 DEBUG } 22:36:59 DEBUG ] 22:36:59 DEBUG } 22:36:59 DEBUG } 22:36:59 DEBUG } 22:36:59 DEBUG } 22:36:59 DEBUG } 22:36:59 DEBUG } 22:36:59 DEBUG } 22:36:59 DEBUG } 22:36:59 DEBUG } 22:36:59 DEBUG } 22:36:59 DEBUG } 22:36:59 DEBUG ] 22:36:59 DEBUG } 22:36:59 DEBUG } 22:36:59 DEBUG } 22:36:59 DEBUG } 22:36:59 DEBUG } 22:36:59 DEBUG ] 22:36:59 DEBUG } 22:36:59 DEBUG } 22:36:59 DEBUG } 22:36:59 DEBUG } 22:36:59 DEBUG } 22:36:59 DEBUG ] 22:36:59 DEBUG } 22:36:59 DEBUG } 22:36:59 DEBUG } 22:36:59 DEBUG } 22:36:59 DEBUG } 22:36:59 DEBUG ] 22:36:59 DEBUG } 22:36:59 DEBUG } 22:36:59 DEBUG } 22:36:59 INFO [loop_until]: OK (rc = 0) 22:36:59 DEBUG } 22:36:59 DEBUG } 22:36:59 DEBUG } 22:36:59 DEBUG } 22:36:59 DEBUG } 22:36:59 DEBUG ] 22:36:59 DEBUG --- stdout --- 22:36:59 DEBUG } 22:36:59 DEBUG } 22:36:59 DEBUG } 22:36:59 DEBUG } 22:36:59 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 5098Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7536m 25% 14826Mi 13% gke-xlou-cdm-default-pool-1b0769c8-nhlf 9278m 31% 10823Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 83m 0% 2637Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 4543m 15% 6505Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 8760m 29% 16697Mi 14% gke-xlou-cdm-default-pool-1b0769c8-xnmp 62m 0% 3149Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 9468m 31% 10378Mi 9% gke-xlou-cdm-ds-0f601751-hmm7 4720m 15% 5631Mi 5% gke-xlou-cdm-ds-0f601751-j488 7652m 25% 14158Mi 12% gke-xlou-cdm-ds-0f601751-mc5d 10580m 35% 3247Mi 2% gke-xlou-cdm-ds-0f601751-nj6q 9698m 32% 10431Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 4751m 15% 7344Mi 6% gke-xlou-cdm-frontend-a8771548-i82n 6461m 40% 5947Mi 10% 22:36:59 DEBUG } 22:36:59 DEBUG --- stderr --- 22:36:59 DEBUG 22:37:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713303476" 22:37:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303476" 22:37:56 ERROR [http_cmd]: ERROR 22:37:56 ERROR --- status code --- 22:37:56 ERROR http status code is 400 (expected [200, 401, 403, 404]) 22:37:56 ERROR --- http response --- 22:37:56 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 22:37:56 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 22:37:56 INFO [http_cmd]: http status code OK 22:37:56 DEBUG --- status code --- 22:37:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:37:56 DEBUG --- http response --- 22:37:56 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 22:37:58 INFO 22:37:58 INFO [loop_until]: kubectl --namespace=xlou top pods 22:37:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:37:58 INFO [loop_until]: OK (rc = 0) 22:37:58 DEBUG --- stdout --- 22:37:58 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 9Mi am-555f4db595-4lbss 9487m 8604Mi am-555f4db595-fz8sn 9884m 8621Mi am-555f4db595-kzxsp 9510m 8564Mi ds-cts-0 8403m 12177Mi ds-cts-1 7455m 12267Mi ds-cts-2 7509m 12230Mi ds-idrepo-0 4773m 5027Mi ds-idrepo-1 4523m 2875Mi ds-idrepo-2 4506m 3171Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 3m 1207Mi idm-b85f9d978-rl76f 4m 1226Mi lodemon-bc756c647-cr7x6 2m 63Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-7c9b667974-9mr2s 6304m 547Mi 22:37:58 DEBUG --- stderr --- 22:37:58 DEBUG 22:37:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303479" 22:37:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303479" 22:37:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303479" 22:37:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303479" 22:37:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303479" 22:37:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303479" 22:37:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303479" 22:37:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303479" 22:37:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303479" 22:37:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713303479" 22:37:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713303479" 22:37:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303479" 22:37:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303479" 22:37:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303479" 22:37:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303479" 22:37:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713303479" 22:37:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303479" 22:37:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713303479" 22:37:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303479" 22:37:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303479" 22:37:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303479" 22:37:59 INFO 22:37:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303479" 22:37:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303479" 22:37:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303479" 22:37:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303479" 22:37:59 INFO [http_cmd]: http status code OK 22:37:59 INFO [http_cmd]: http status code OK 22:37:59 INFO [http_cmd]: http status code OK 22:37:59 INFO [http_cmd]: http status code OK 22:37:59 INFO [http_cmd]: http status code OK 22:37:59 INFO [http_cmd]: http status code OK 22:37:59 INFO [http_cmd]: http status code OK 22:37:59 INFO [http_cmd]: http status code OK 22:37:59 INFO [http_cmd]: http status code OK 22:37:59 INFO [http_cmd]: http status code OK 22:37:59 INFO [http_cmd]: http status code OK 22:37:59 INFO [http_cmd]: http status code OK 22:37:59 INFO [http_cmd]: http status code OK 22:37:59 INFO [http_cmd]: http status code OK 22:37:59 INFO [http_cmd]: http status code OK 22:37:59 INFO [http_cmd]: http status code OK 22:37:59 INFO [loop_until]: kubectl --namespace=xlou top node 22:37:59 INFO [http_cmd]: http status code OK 22:37:59 DEBUG --- status code --- 22:37:59 DEBUG --- status code --- 22:37:59 INFO [http_cmd]: http status code OK 22:37:59 DEBUG --- status code --- 22:37:59 DEBUG --- status code --- 22:37:59 DEBUG --- status code --- 22:37:59 DEBUG --- status code --- 22:37:59 DEBUG --- status code --- 22:37:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:37:59 INFO [http_cmd]: http status code OK 22:37:59 DEBUG --- status code --- 22:37:59 INFO [http_cmd]: http status code OK 22:37:59 DEBUG --- status code --- 22:37:59 DEBUG --- status code --- 22:37:59 DEBUG --- status code --- 22:37:59 DEBUG --- status code --- 22:37:59 DEBUG --- status code --- 22:37:59 DEBUG --- status code --- 22:37:59 DEBUG --- status code --- 22:37:59 DEBUG --- status code --- 22:37:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:37:59 INFO [http_cmd]: http status code OK 22:37:59 DEBUG --- status code --- 22:37:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:37:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:37:59 DEBUG --- status code --- 22:37:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:37:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:37:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:37:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:37:59 DEBUG --- http response --- 22:37:59 INFO [http_cmd]: http status code OK 22:37:59 DEBUG --- status code --- 22:37:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:37:59 DEBUG --- status code --- 22:37:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:37:59 INFO [http_cmd]: http status code OK 22:37:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:37:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:37:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:37:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:37:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:37:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:37:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:37:59 DEBUG --- status code --- 22:37:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:37:59 INFO [http_cmd]: http status code OK 22:37:59 DEBUG --- http response --- 22:37:59 DEBUG --- http response --- 22:37:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:37:59 DEBUG --- http response --- 22:37:59 DEBUG --- http response --- 22:37:59 DEBUG --- http response --- 22:37:59 DEBUG --- http response --- 22:37:59 DEBUG --- status code --- 22:37:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:37:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:37:59 INFO [http_cmd]: http status code OK 22:37:59 DEBUG --- http response --- 22:37:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:37:59 DEBUG --- http response --- 22:37:59 DEBUG --- status code --- 22:37:59 DEBUG --- http response --- 22:37:59 DEBUG --- http response --- 22:37:59 DEBUG --- http response --- 22:37:59 DEBUG --- http response --- 22:37:59 DEBUG --- http response --- 22:37:59 DEBUG --- http response --- 22:37:59 DEBUG --- http response --- 22:37:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:37:59 DEBUG --- http response --- 22:37:59 DEBUG --- status code --- 22:37:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:37:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:37:59 DEBUG --- http response --- 22:37:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 22:37:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:37:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 22:37:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:37:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:37:59 DEBUG ----- output ----- 22:37:59 DEBUG { 22:37:59 DEBUG --- status code --- 22:37:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:37:59 DEBUG --- http response --- 22:37:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:37:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:37:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:37:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:37:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:37:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:37:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:37:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:37:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:37:59 DEBUG --- http response --- 22:37:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:37:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:37:59 DEBUG ----- output ----- 22:37:59 DEBUG ----- output ----- 22:37:59 DEBUG ----- output ----- 22:37:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:37:59 DEBUG ----- output ----- 22:37:59 DEBUG ----- output ----- 22:37:59 DEBUG --- http response --- 22:37:59 DEBUG ----- output ----- 22:37:59 DEBUG "status": "success", 22:37:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:37:59 DEBUG --- http response --- 22:37:59 DEBUG ----- output ----- 22:37:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:37:59 DEBUG --- http response --- 22:37:59 DEBUG ----- output ----- 22:37:59 DEBUG ----- output ----- 22:37:59 DEBUG ----- output ----- 22:37:59 DEBUG ----- output ----- 22:37:59 DEBUG ----- output ----- 22:37:59 DEBUG ----- output ----- 22:37:59 DEBUG ----- output ----- 22:37:59 DEBUG ----- output ----- 22:37:59 DEBUG --- http response --- 22:37:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:37:59 DEBUG ----- output ----- 22:37:59 DEBUG { 22:37:59 DEBUG { 22:37:59 DEBUG { 22:37:59 DEBUG ----- output ----- 22:37:59 DEBUG { 22:37:59 DEBUG { 22:37:59 DEBUG { 22:37:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:37:59 DEBUG "data": { 22:37:59 DEBUG --- http response --- 22:37:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 22:37:59 DEBUG { 22:37:59 DEBUG ----- output ----- 22:37:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:37:59 DEBUG ----- output ----- 22:37:59 DEBUG { 22:37:59 DEBUG "status": "success", 22:37:59 DEBUG { 22:37:59 DEBUG { 22:37:59 DEBUG { 22:37:59 DEBUG { 22:37:59 DEBUG { 22:37:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:37:59 DEBUG ----- output ----- 22:37:59 DEBUG { 22:37:59 DEBUG "status": "success", 22:37:59 DEBUG "status": "success", 22:37:59 DEBUG "status": "success", 22:37:59 DEBUG { 22:37:59 DEBUG "status": "success", 22:37:59 DEBUG "status": "success", 22:37:59 DEBUG "status": "success", 22:37:59 DEBUG ----- output ----- 22:37:59 DEBUG "resultType": "vector", 22:37:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 22:37:59 DEBUG ----- output ----- 22:37:59 DEBUG "status": "success", 22:37:59 DEBUG { 22:37:59 DEBUG { 22:37:59 DEBUG { 22:37:59 DEBUG { 22:37:59 DEBUG "data": { 22:37:59 DEBUG "status": "success", 22:37:59 DEBUG "status": "success", 22:37:59 DEBUG "status": "success", 22:37:59 DEBUG "status": "success", 22:37:59 DEBUG "status": "success", 22:37:59 DEBUG ----- output ----- 22:37:59 DEBUG { 22:37:59 DEBUG "status": "success", 22:37:59 DEBUG "data": { 22:37:59 DEBUG "data": { 22:37:59 DEBUG "data": { 22:37:59 DEBUG "status": "success", 22:37:59 DEBUG "data": { 22:37:59 DEBUG "data": { 22:37:59 DEBUG "data": { 22:37:59 DEBUG { 22:37:59 DEBUG "result": [ 22:37:59 DEBUG ----- output ----- 22:37:59 DEBUG { 22:37:59 DEBUG "data": { 22:37:59 DEBUG "status": "success", 22:37:59 DEBUG "status": "success", 22:37:59 DEBUG "status": "success", 22:37:59 DEBUG "status": "success", 22:37:59 DEBUG "resultType": "vector", 22:37:59 DEBUG "data": { 22:37:59 DEBUG "data": { 22:37:59 DEBUG "data": { 22:37:59 DEBUG "data": { 22:37:59 DEBUG "data": { 22:37:59 DEBUG { 22:37:59 DEBUG "status": "success", 22:37:59 DEBUG "data": { 22:37:59 DEBUG "resultType": "vector", 22:37:59 DEBUG "resultType": "vector", 22:37:59 DEBUG "resultType": "vector", 22:37:59 DEBUG "data": { 22:37:59 DEBUG "resultType": "vector", 22:37:59 DEBUG "resultType": "vector", 22:37:59 DEBUG "resultType": "vector", 22:37:59 DEBUG "status": "success", 22:37:59 DEBUG { 22:37:59 DEBUG { 22:37:59 DEBUG "status": "success", 22:37:59 DEBUG "resultType": "vector", 22:37:59 DEBUG "data": { 22:37:59 DEBUG "data": { 22:37:59 DEBUG "resultType": "vector", 22:37:59 DEBUG "data": { 22:37:59 DEBUG "result": [ 22:37:59 DEBUG "resultType": "vector", 22:37:59 DEBUG "resultType": "vector", 22:37:59 DEBUG "resultType": "vector", 22:37:59 DEBUG "resultType": "vector", 22:37:59 DEBUG "resultType": "vector", 22:37:59 DEBUG "status": "success", 22:37:59 DEBUG "data": { 22:37:59 DEBUG "resultType": "vector", 22:37:59 DEBUG "result": [ 22:37:59 DEBUG "result": [ 22:37:59 DEBUG "result": [ 22:37:59 DEBUG "resultType": "vector", 22:37:59 DEBUG "result": [ 22:37:59 DEBUG "result": [ 22:37:59 DEBUG "result": [ 22:37:59 DEBUG "data": { 22:37:59 DEBUG "metric": { 22:37:59 DEBUG "status": "success", 22:37:59 DEBUG "data": { 22:37:59 DEBUG "result": [ 22:37:59 DEBUG { 22:37:59 DEBUG "result": [ 22:37:59 DEBUG "data": { 22:37:59 DEBUG "resultType": "vector", 22:37:59 DEBUG { 22:37:59 DEBUG "result": [ 22:37:59 DEBUG "result": [ 22:37:59 DEBUG "result": [ 22:37:59 DEBUG "result": [ 22:37:59 DEBUG "result": [ 22:37:59 DEBUG "data": { 22:37:59 DEBUG "resultType": "vector", 22:37:59 DEBUG "result": [ 22:37:59 DEBUG { 22:37:59 DEBUG { 22:37:59 DEBUG { 22:37:59 DEBUG "result": [ 22:37:59 DEBUG { 22:37:59 DEBUG { 22:37:59 DEBUG { 22:37:59 DEBUG "resultType": "vector", 22:37:59 DEBUG "pod": "am-555f4db595-fz8sn" 22:37:59 DEBUG "data": { 22:37:59 DEBUG "resultType": "vector", 22:37:59 DEBUG "metric": { 22:37:59 DEBUG "resultType": "vector", 22:37:59 DEBUG { 22:37:59 DEBUG "resultType": "vector", 22:37:59 DEBUG "result": [ 22:37:59 DEBUG "metric": { 22:37:59 DEBUG { 22:37:59 DEBUG { 22:37:59 DEBUG { 22:37:59 DEBUG { 22:37:59 DEBUG { 22:37:59 DEBUG "resultType": "vector", 22:37:59 DEBUG "result": [ 22:37:59 DEBUG { 22:37:59 DEBUG "metric": { 22:37:59 DEBUG "metric": { 22:37:59 DEBUG "metric": { 22:37:59 DEBUG { 22:37:59 DEBUG "metric": { 22:37:59 DEBUG "metric": { 22:37:59 DEBUG "metric": { 22:37:59 DEBUG "result": [ 22:37:59 DEBUG }, 22:37:59 DEBUG "resultType": "vector", 22:37:59 DEBUG "result": [ 22:37:59 DEBUG "pod": "am-555f4db595-fz8sn" 22:37:59 DEBUG "result": [ 22:37:59 DEBUG "metric": { 22:37:59 DEBUG "result": [ 22:37:59 DEBUG { 22:37:59 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:37:59 DEBUG "metric": { 22:37:59 DEBUG "metric": { 22:37:59 DEBUG "metric": { 22:37:59 DEBUG "metric": { 22:37:59 DEBUG "metric": { 22:37:59 DEBUG "result": [ 22:37:59 DEBUG { 22:37:59 DEBUG "metric": { 22:37:59 DEBUG "pod": "am-555f4db595-fz8sn" 22:37:59 DEBUG "pod": "am-555f4db595-fz8sn" 22:37:59 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:37:59 DEBUG "metric": { 22:37:59 DEBUG "instance": "10.142.0.107:9100" 22:37:59 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:37:59 DEBUG "instance": "10.142.0.107:9100" 22:37:59 DEBUG { 22:37:59 DEBUG "value": [ 22:37:59 DEBUG "result": [ 22:37:59 DEBUG { 22:37:59 DEBUG }, 22:37:59 DEBUG { 22:37:59 DEBUG "instance": "10.142.0.107:9100" 22:37:59 DEBUG { 22:37:59 DEBUG "metric": { 22:37:59 DEBUG }, 22:37:59 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:37:59 DEBUG }, 22:37:59 DEBUG "pod": "ds-idrepo-2" 22:37:59 DEBUG "pod": "ds-idrepo-2" 22:37:59 DEBUG "pod": "ds-idrepo-2" 22:37:59 DEBUG { 22:37:59 DEBUG "metric": { 22:37:59 DEBUG "pod": "am-555f4db595-fz8sn" 22:37:59 DEBUG }, 22:37:59 DEBUG }, 22:37:59 DEBUG }, 22:37:59 DEBUG "pod": "ds-idrepo-2" 22:37:59 DEBUG }, 22:37:59 DEBUG }, 22:37:59 DEBUG }, 22:37:59 DEBUG "metric": { 22:37:59 DEBUG 22:37:59 DEBUG { 22:37:59 DEBUG "metric": { 22:37:59 DEBUG "value": [ 22:37:59 DEBUG "metric": { 22:37:59 DEBUG }, 22:37:59 DEBUG "metric": { 22:37:59 DEBUG "pod": "am-555f4db595-fz8sn" 22:37:59 DEBUG "value": [ 22:37:59 DEBUG 22:37:59 DEBUG "value": [ 22:37:59 DEBUG }, 22:37:59 DEBUG }, 22:37:59 DEBUG }, 22:37:59 DEBUG "metric": { 22:37:59 DEBUG "pod": "ds-idrepo-2" 22:37:59 DEBUG }, 22:37:59 DEBUG "value": [ 22:37:59 DEBUG "value": [ 22:37:59 DEBUG "value": [ 22:37:59 DEBUG }, 22:37:59 DEBUG "value": [ 22:37:59 DEBUG "value": [ 22:37:59 DEBUG "value": [ 22:37:59 DEBUG "pod": "am-555f4db595-fz8sn" 22:37:59 DEBUG 22:37:59 DEBUG "metric": { 22:37:59 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:37:59 DEBUG 22:37:59 DEBUG "pod": "ds-idrepo-2" 22:37:59 DEBUG "value": [ 22:37:59 DEBUG "instance": "10.142.0.107:9100" 22:37:59 DEBUG }, 22:37:59 DEBUG "pod": "ds-idrepo-2" 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG "value": [ 22:37:59 DEBUG "value": [ 22:37:59 DEBUG "value": [ 22:37:59 DEBUG 22:37:59 DEBUG }, 22:37:59 DEBUG "value": [ 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG "value": [ 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG }, 22:37:59 DEBUG 22:37:59 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:37:59 DEBUG }, 22:37:59 DEBUG 22:37:59 DEBUG }, 22:37:59 DEBUG 22:37:59 DEBUG }, 22:37:59 DEBUG "value": [ 22:37:59 DEBUG }, 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG "value": [ 22:37:59 DEBUG "instance": "10.142.0.107:9100" 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG "value": [ 22:37:59 DEBUG [...] 22:37:59 DEBUG }, 22:37:59 DEBUG "value": [ 22:37:59 DEBUG 22:37:59 DEBUG "value": [ 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG "value": [ 22:37:59 DEBUG [...] 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG }, 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG "value": [ 22:37:59 DEBUG 22:37:59 DEBUG [...] 22:37:59 DEBUG 22:37:59 DEBUG [...] 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG "value": [ 22:37:59 DEBUG [...] 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG [...] 22:37:59 DEBUG 22:37:59 DEBUG "value": [ 22:37:59 DEBUG 22:37:59 DEBUG [...] 22:37:59 DEBUG [...] 22:37:59 DEBUG [...] 22:37:59 DEBUG 22:37:59 DEBUG [...] 22:37:59 DEBUG [...] 22:37:59 DEBUG [...] 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG [...] 22:37:59 DEBUG [...] 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG [...] 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG [...] 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG [...] 22:37:59 DEBUG 22:37:59 DEBUG [...] 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG [...] 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG [...] 22:37:59 DEBUG "pod": "am-555f4db595-kzxsp" 22:37:59 DEBUG 22:37:59 DEBUG [...] 22:37:59 DEBUG 22:37:59 DEBUG [...] 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG }, 22:37:59 DEBUG [...] 22:37:59 DEBUG 22:37:59 DEBUG "pod": "am-555f4db595-kzxsp" 22:37:59 DEBUG 22:37:59 DEBUG "instance": "10.142.0.43:9100" 22:37:59 DEBUG 22:37:59 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:37:59 DEBUG 22:37:59 DEBUG [...] 22:37:59 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG "pod": "ds-cts-2" 22:37:59 DEBUG 22:37:59 DEBUG [...] 22:37:59 DEBUG 22:37:59 DEBUG "pod": "am-555f4db595-kzxsp" 22:37:59 DEBUG "pod": "am-555f4db595-kzxsp" 22:37:59 DEBUG "pod": "idm-b85f9d978-j5pbm" 22:37:59 DEBUG 22:37:59 DEBUG "instance": "10.142.0.43:9100" 22:37:59 DEBUG "pod": "idm-b85f9d978-j5pbm" 22:37:59 DEBUG "instance": "10.142.0.43:9100" 22:37:59 DEBUG 22:37:59 DEBUG "value": [ 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG }, 22:37:59 DEBUG 22:37:59 DEBUG }, 22:37:59 DEBUG 22:37:59 DEBUG }, 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG }, 22:37:59 DEBUG "pod": "ds-idrepo-0" 22:37:59 DEBUG "pod": "ds-idrepo-0" 22:37:59 DEBUG }, 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG "pod": "am-555f4db595-kzxsp" 22:37:59 DEBUG }, 22:37:59 DEBUG }, 22:37:59 DEBUG }, 22:37:59 DEBUG "pod": "ds-idrepo-0" 22:37:59 DEBUG }, 22:37:59 DEBUG }, 22:37:59 DEBUG }, 22:37:59 DEBUG 22:37:59 DEBUG 1713303479, 22:37:59 DEBUG 22:37:59 DEBUG 22:37:59 DEBUG "value": [ 22:37:59 DEBUG 22:37:59 DEBUG "value": [ 22:37:59 DEBUG "pod": "am-555f4db595-kzxsp" 22:37:59 DEBUG "value": [ 22:37:59 DEBUG "pod": "ds-idrepo-0" 22:37:59 DEBUG 22:37:59 DEBUG "value": [ 22:37:59 DEBUG }, 22:37:59 DEBUG }, 22:37:59 DEBUG "value": [ 22:37:59 DEBUG "pod": "ds-idrepo-0" 22:37:59 DEBUG 22:37:59 DEBUG }, 22:37:59 DEBUG "value": [ 22:37:59 DEBUG "value": [ 22:37:59 DEBUG "value": [ 22:37:59 DEBUG }, 22:37:59 DEBUG "value": [ 22:37:59 DEBUG "value": [ 22:37:59 DEBUG "value": [ 22:37:59 DEBUG "pod": "am-555f4db595-kzxsp" 22:37:59 DEBUG "2003.7666666666667" 22:37:59 DEBUG 22:37:59 DEBUG "pod": "idm-b85f9d978-j5pbm" 22:37:59 DEBUG 1713303479, 22:37:59 DEBUG "pod": "ds-idrepo-0" 22:37:59 DEBUG 1713303479, 22:37:59 DEBUG }, 22:37:59 DEBUG 1713303479, 22:37:59 DEBUG }, 22:37:59 DEBUG 22:37:59 DEBUG 1713303479, 22:37:59 DEBUG "value": [ 22:37:59 DEBUG "value": [ 22:37:59 DEBUG 1713303479, 22:37:59 DEBUG }, 22:37:59 DEBUG 22:37:59 DEBUG "value": [ 22:37:59 DEBUG 1713303479, 22:37:59 DEBUG 1713303479, 22:37:59 DEBUG 1713303479, 22:37:59 DEBUG "value": [ 22:37:59 DEBUG 1713303479, 22:37:59 DEBUG 1713303479, 22:37:59 DEBUG 1713303479, 22:37:59 DEBUG }, 22:37:59 DEBUG ] 22:37:59 DEBUG "pod": "idm-b85f9d978-j5pbm" 22:37:59 DEBUG }, 22:37:59 DEBUG "2003.6333333333332" 22:37:59 DEBUG }, 22:37:59 DEBUG "45575241.49999999" 22:37:59 DEBUG "value": [ 22:37:59 DEBUG "6.4223614838724" 22:37:59 DEBUG "value": [ 22:37:59 DEBUG "instance": "10.142.0.43:9100" 22:37:59 DEBUG "666533888" 22:37:59 DEBUG 1713303479, 22:37:59 DEBUG 1713303479, 22:37:59 DEBUG "0" 22:37:59 DEBUG "value": [ 22:37:59 DEBUG "instance": "10.142.0.43:9100" 22:37:59 DEBUG 1713303479, 22:37:59 DEBUG "0" 22:37:59 DEBUG "3139173" 22:37:59 DEBUG "0" 22:37:59 DEBUG 1713303479, 22:37:59 DEBUG "45567672.8" 22:37:59 DEBUG "80425763.95326488" 22:37:59 DEBUG "0.0005666666666665302" 22:37:59 DEBUG "value": [ 22:37:59 DEBUG } 22:37:59 DEBUG }, 22:37:59 DEBUG "value": [ 22:37:59 DEBUG ] 22:37:59 DEBUG "value": [ 22:37:59 DEBUG ] 22:37:59 DEBUG 1713303479, 22:37:59 DEBUG ] 22:37:59 DEBUG 1713303479, 22:37:59 DEBUG }, 22:37:59 DEBUG ] 22:37:59 DEBUG "0" 22:37:59 DEBUG "0" 22:37:59 DEBUG ] 22:37:59 DEBUG 1713303479, 22:37:59 DEBUG }, 22:37:59 DEBUG "2003.1666666666667" 22:37:59 DEBUG ] 22:37:59 DEBUG ] 22:37:59 DEBUG ] 22:37:59 DEBUG "0" 22:37:59 DEBUG ] 22:37:59 DEBUG ] 22:37:59 DEBUG ] 22:37:59 DEBUG 1713303479, 22:37:59 DEBUG ] 22:37:59 DEBUG "value": [ 22:37:59 DEBUG 1713303479, 22:37:59 DEBUG } 22:37:59 DEBUG 1713303479, 22:37:59 DEBUG } 22:37:59 DEBUG "0.03333333333333333" 22:37:59 DEBUG } 22:37:59 DEBUG "0" 22:37:59 DEBUG "value": [ 22:37:59 DEBUG } 22:37:59 DEBUG ] 22:37:59 DEBUG ] 22:37:59 DEBUG } 22:37:59 DEBUG "0" 22:37:59 DEBUG "value": [ 22:37:59 DEBUG ] 22:37:59 DEBUG } 22:37:59 DEBUG } 22:37:59 DEBUG } 22:37:59 DEBUG ] 22:37:59 DEBUG } 22:37:59 DEBUG } 22:37:59 DEBUG } 22:37:59 DEBUG ] 22:37:59 DEBUG } 22:37:59 DEBUG 1713303479, 22:37:59 DEBUG "0" 22:37:59 DEBUG ] 22:37:59 DEBUG "0" 22:37:59 DEBUG ] 22:37:59 DEBUG ] 22:37:59 DEBUG ] 22:37:59 DEBUG ] 22:37:59 DEBUG } 22:37:59 DEBUG ] 22:37:59 DEBUG } 22:37:59 DEBUG } 22:37:59 DEBUG ] 22:37:59 DEBUG ] 22:37:59 DEBUG 1713303479, 22:37:59 DEBUG } 22:37:59 DEBUG ] 22:37:59 DEBUG ] 22:37:59 DEBUG ] 22:37:59 DEBUG } 22:37:59 DEBUG ] 22:37:59 DEBUG ] 22:37:59 DEBUG "0.026" 22:37:59 DEBUG } 22:37:59 DEBUG } 22:37:59 DEBUG "0.003930477168886634" 22:37:59 DEBUG ] 22:37:59 DEBUG } 22:37:59 DEBUG ] 22:37:59 DEBUG } 22:37:59 DEBUG } 22:37:59 DEBUG } 22:37:59 DEBUG 1713303479, 22:37:59 DEBUG ] 22:37:59 DEBUG } 22:37:59 DEBUG ] 22:37:59 DEBUG ] 22:37:59 DEBUG } 22:37:59 DEBUG } 22:37:59 DEBUG "0" 22:37:59 DEBUG ] 22:37:59 DEBUG } 22:37:59 DEBUG } 22:37:59 DEBUG } 22:37:59 DEBUG ] 22:37:59 DEBUG } 22:37:59 DEBUG } 22:37:59 DEBUG ] 22:37:59 DEBUG } 22:37:59 DEBUG ] 22:37:59 DEBUG } 22:37:59 DEBUG } 22:37:59 DEBUG } 22:37:59 DEBUG } 22:37:59 DEBUG ] 22:37:59 DEBUG } 22:37:59 DEBUG "19964281856" 22:37:59 DEBUG } 22:37:59 DEBUG } 22:37:59 DEBUG } 22:37:59 DEBUG } 22:37:59 DEBUG } 22:37:59 DEBUG ] 22:37:59 DEBUG ] 22:37:59 DEBUG } 22:37:59 DEBUG } 22:37:59 DEBUG } 22:37:59 DEBUG } 22:37:59 DEBUG } 22:37:59 DEBUG } 22:37:59 DEBUG } 22:37:59 DEBUG } 22:37:59 DEBUG } 22:37:59 DEBUG ] 22:37:59 DEBUG ] 22:37:59 DEBUG } 22:37:59 DEBUG ] 22:37:59 DEBUG } 22:37:59 DEBUG } 22:37:59 DEBUG } 22:37:59 DEBUG } 22:37:59 DEBUG } 22:37:59 DEBUG } 22:37:59 DEBUG } 22:37:59 DEBUG ] 22:37:59 DEBUG ] 22:37:59 DEBUG } 22:37:59 DEBUG } 22:37:59 DEBUG } 22:37:59 DEBUG } 22:37:59 INFO [loop_until]: OK (rc = 0) 22:37:59 DEBUG } 22:37:59 DEBUG ] 22:37:59 DEBUG } 22:37:59 DEBUG } 22:37:59 DEBUG } 22:37:59 DEBUG } 22:37:59 DEBUG ] 22:37:59 DEBUG --- stdout --- 22:37:59 DEBUG } 22:37:59 DEBUG } 22:37:59 DEBUG } 22:37:59 DEBUG } 22:37:59 DEBUG } 22:37:59 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 63m 0% 5095Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7626m 25% 14828Mi 13% gke-xlou-cdm-default-pool-1b0769c8-nhlf 9608m 32% 10827Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 81m 0% 2640Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 4788m 16% 6505Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 8836m 29% 16698Mi 14% gke-xlou-cdm-default-pool-1b0769c8-xnmp 61m 0% 3143Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 9620m 32% 10380Mi 9% gke-xlou-cdm-ds-0f601751-hmm7 4643m 15% 5619Mi 5% gke-xlou-cdm-ds-0f601751-j488 7727m 25% 14172Mi 12% gke-xlou-cdm-ds-0f601751-mc5d 10625m 35% 3243Mi 2% gke-xlou-cdm-ds-0f601751-nj6q 9592m 32% 10431Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 4785m 16% 7352Mi 6% gke-xlou-cdm-frontend-a8771548-i82n 6472m 40% 5950Mi 10% 22:37:59 DEBUG } 22:37:59 DEBUG --- stderr --- 22:37:59 DEBUG 22:38:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713303536" 22:38:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303536" 22:38:56 ERROR [http_cmd]: ERROR 22:38:56 ERROR --- status code --- 22:38:56 ERROR http status code is 400 (expected [200, 401, 403, 404]) 22:38:56 ERROR --- http response --- 22:38:56 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 22:38:56 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 22:38:56 INFO [http_cmd]: http status code OK 22:38:56 DEBUG --- status code --- 22:38:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:38:56 DEBUG --- http response --- 22:38:56 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 22:38:58 INFO 22:38:58 INFO [loop_until]: kubectl --namespace=xlou top pods 22:38:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:38:58 INFO [loop_until]: OK (rc = 0) 22:38:58 DEBUG --- stdout --- 22:38:58 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 9Mi am-555f4db595-4lbss 9291m 8607Mi am-555f4db595-fz8sn 9450m 8621Mi am-555f4db595-kzxsp 9398m 8566Mi ds-cts-0 8320m 12184Mi ds-cts-1 7635m 12274Mi ds-cts-2 7604m 12236Mi ds-idrepo-0 4696m 5026Mi ds-idrepo-1 4411m 2875Mi ds-idrepo-2 4453m 3171Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 4m 1207Mi idm-b85f9d978-rl76f 3m 1225Mi lodemon-bc756c647-cr7x6 2m 63Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-7c9b667974-9mr2s 6273m 548Mi 22:38:58 DEBUG --- stderr --- 22:38:58 DEBUG 22:38:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303539" 22:38:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303539" 22:38:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303539" 22:38:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713303539" 22:38:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303539" 22:38:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713303539" 22:38:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303539" 22:38:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303539" 22:38:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303539" 22:38:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303539" 22:38:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303539" 22:38:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713303539" 22:38:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303539" 22:38:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303539" 22:38:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303539" 22:38:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303539" 22:38:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303539" 22:38:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303539" 22:38:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303539" 22:38:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303539" 22:38:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713303539" 22:38:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303539" 22:38:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303539" 22:38:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303539" 22:38:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303539" 22:38:59 INFO 22:38:59 INFO [http_cmd]: http status code OK 22:38:59 INFO [http_cmd]: http status code OK 22:38:59 INFO [http_cmd]: http status code OK 22:38:59 INFO [http_cmd]: http status code OK 22:38:59 INFO [http_cmd]: http status code OK 22:38:59 INFO [http_cmd]: http status code OK 22:38:59 INFO [http_cmd]: http status code OK 22:38:59 INFO [http_cmd]: http status code OK 22:38:59 INFO [http_cmd]: http status code OK 22:38:59 INFO [http_cmd]: http status code OK 22:38:59 INFO [http_cmd]: http status code OK 22:38:59 INFO [http_cmd]: http status code OK 22:38:59 INFO [http_cmd]: http status code OK 22:38:59 INFO [http_cmd]: http status code OK 22:38:59 INFO [http_cmd]: http status code OK 22:38:59 INFO [loop_until]: kubectl --namespace=xlou top node 22:38:59 INFO [http_cmd]: http status code OK 22:38:59 INFO [http_cmd]: http status code OK 22:38:59 DEBUG --- status code --- 22:38:59 INFO [http_cmd]: http status code OK 22:38:59 DEBUG --- status code --- 22:38:59 INFO [http_cmd]: http status code OK 22:38:59 DEBUG --- status code --- 22:38:59 INFO [http_cmd]: http status code OK 22:38:59 DEBUG --- status code --- 22:38:59 DEBUG --- status code --- 22:38:59 INFO [http_cmd]: http status code OK 22:38:59 INFO [http_cmd]: http status code OK 22:38:59 DEBUG --- status code --- 22:38:59 DEBUG --- status code --- 22:38:59 DEBUG --- status code --- 22:38:59 DEBUG --- status code --- 22:38:59 DEBUG --- status code --- 22:38:59 DEBUG --- status code --- 22:38:59 DEBUG --- status code --- 22:38:59 DEBUG --- status code --- 22:38:59 DEBUG --- status code --- 22:38:59 DEBUG --- status code --- 22:38:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:38:59 DEBUG --- status code --- 22:38:59 DEBUG --- status code --- 22:38:59 INFO [http_cmd]: http status code OK 22:38:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:38:59 INFO [http_cmd]: http status code OK 22:38:59 DEBUG --- status code --- 22:38:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:38:59 DEBUG --- status code --- 22:38:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:38:59 DEBUG --- status code --- 22:38:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:38:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:38:59 DEBUG --- status code --- 22:38:59 DEBUG --- status code --- 22:38:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:38:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:38:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:38:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:38:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:38:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:38:59 INFO [http_cmd]: http status code OK 22:38:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:38:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:38:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:38:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:38:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:38:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:38:59 DEBUG --- status code --- 22:38:59 DEBUG --- http response --- 22:38:59 DEBUG --- status code --- 22:38:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:38:59 DEBUG --- http response --- 22:38:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:38:59 DEBUG --- http response --- 22:38:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:38:59 DEBUG --- http response --- 22:38:59 DEBUG --- http response --- 22:38:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:38:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:38:59 DEBUG --- http response --- 22:38:59 DEBUG --- http response --- 22:38:59 DEBUG --- http response --- 22:38:59 DEBUG --- http response --- 22:38:59 DEBUG --- http response --- 22:38:59 DEBUG --- http response --- 22:38:59 DEBUG --- status code --- 22:38:59 DEBUG --- http response --- 22:38:59 DEBUG --- http response --- 22:38:59 DEBUG --- http response --- 22:38:59 DEBUG --- http response --- 22:38:59 DEBUG --- http response --- 22:38:59 DEBUG --- http response --- 22:38:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:38:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:38:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:38:59 DEBUG --- http response --- 22:38:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:38:59 DEBUG --- http response --- 22:38:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:38:59 DEBUG --- http response --- 22:38:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:38:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:38:59 DEBUG --- http response --- 22:38:59 DEBUG --- http response --- 22:38:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:38:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:38:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:38:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:38:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:38:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:38:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:38:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:38:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:38:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:38:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:38:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:38:59 DEBUG --- http response --- 22:38:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:38:59 DEBUG --- http response --- 22:38:59 DEBUG ----- output ----- 22:38:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:38:59 DEBUG ----- output ----- 22:38:59 DEBUG ----- output ----- 22:38:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:38:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:38:59 DEBUG ----- output ----- 22:38:59 DEBUG ----- output ----- 22:38:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:38:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:38:59 DEBUG ----- output ----- 22:38:59 DEBUG ----- output ----- 22:38:59 DEBUG ----- output ----- 22:38:59 DEBUG ----- output ----- 22:38:59 DEBUG ----- output ----- 22:38:59 DEBUG ----- output ----- 22:38:59 DEBUG --- http response --- 22:38:59 DEBUG ----- output ----- 22:38:59 DEBUG ----- output ----- 22:38:59 DEBUG ----- output ----- 22:38:59 DEBUG ----- output ----- 22:38:59 DEBUG ----- output ----- 22:38:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:38:59 DEBUG ----- output ----- 22:38:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:38:59 DEBUG { 22:38:59 DEBUG ----- output ----- 22:38:59 DEBUG { 22:38:59 DEBUG { 22:38:59 DEBUG ----- output ----- 22:38:59 DEBUG ----- output ----- 22:38:59 DEBUG { 22:38:59 DEBUG { 22:38:59 DEBUG ----- output ----- 22:38:59 DEBUG ----- output ----- 22:38:59 DEBUG { 22:38:59 DEBUG { 22:38:59 DEBUG { 22:38:59 DEBUG { 22:38:59 DEBUG { 22:38:59 DEBUG { 22:38:59 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:38:59 DEBUG { 22:38:59 DEBUG { 22:38:59 DEBUG { 22:38:59 DEBUG { 22:38:59 DEBUG { 22:38:59 DEBUG ----- output ----- 22:38:59 DEBUG { 22:38:59 DEBUG ----- output ----- 22:38:59 DEBUG "status": "success", 22:38:59 DEBUG { 22:38:59 DEBUG "status": "success", 22:38:59 DEBUG "status": "success", 22:38:59 DEBUG { 22:38:59 DEBUG { 22:38:59 DEBUG "status": "success", 22:38:59 DEBUG "status": "success", 22:38:59 DEBUG { 22:38:59 DEBUG { 22:38:59 DEBUG "status": "success", 22:38:59 DEBUG "status": "success", 22:38:59 DEBUG "status": "success", 22:38:59 DEBUG "status": "success", 22:38:59 DEBUG "status": "success", 22:38:59 DEBUG "status": "success", 22:38:59 DEBUG ----- output ----- 22:38:59 DEBUG "status": "success", 22:38:59 DEBUG "status": "success", 22:38:59 DEBUG "status": "success", 22:38:59 DEBUG "status": "success", 22:38:59 DEBUG "status": "success", 22:38:59 DEBUG { 22:38:59 DEBUG "status": "success", 22:38:59 DEBUG { 22:38:59 DEBUG "data": { 22:38:59 DEBUG "status": "success", 22:38:59 DEBUG "data": { 22:38:59 DEBUG "data": { 22:38:59 DEBUG "status": "success", 22:38:59 DEBUG "status": "success", 22:38:59 DEBUG "data": { 22:38:59 DEBUG "data": { 22:38:59 DEBUG "status": "success", 22:38:59 DEBUG "status": "success", 22:38:59 DEBUG "data": { 22:38:59 DEBUG "data": { 22:38:59 DEBUG "data": { 22:38:59 DEBUG "data": { 22:38:59 DEBUG "data": { 22:38:59 DEBUG "data": { 22:38:59 DEBUG { 22:38:59 DEBUG "data": { 22:38:59 DEBUG "data": { 22:38:59 DEBUG "data": { 22:38:59 DEBUG "data": { 22:38:59 DEBUG "data": { 22:38:59 DEBUG "status": "success", 22:38:59 DEBUG "data": { 22:38:59 DEBUG "status": "success", 22:38:59 DEBUG "resultType": "vector", 22:38:59 DEBUG "data": { 22:38:59 DEBUG "resultType": "vector", 22:38:59 DEBUG "resultType": "vector", 22:38:59 DEBUG "data": { 22:38:59 DEBUG "data": { 22:38:59 DEBUG "resultType": "vector", 22:38:59 DEBUG "resultType": "vector", 22:38:59 DEBUG "data": { 22:38:59 DEBUG "data": { 22:38:59 DEBUG "resultType": "vector", 22:38:59 DEBUG "resultType": "vector", 22:38:59 DEBUG "resultType": "vector", 22:38:59 DEBUG "resultType": "vector", 22:38:59 DEBUG "resultType": "vector", 22:38:59 DEBUG "resultType": "vector", 22:38:59 DEBUG "status": "success", 22:38:59 DEBUG "resultType": "vector", 22:38:59 DEBUG "resultType": "vector", 22:38:59 DEBUG "resultType": "vector", 22:38:59 DEBUG "resultType": "vector", 22:38:59 DEBUG "result": [ 22:38:59 DEBUG "data": { 22:38:59 DEBUG "resultType": "vector", 22:38:59 DEBUG "data": { 22:38:59 DEBUG "result": [ 22:38:59 DEBUG "resultType": "vector", 22:38:59 DEBUG "result": [ 22:38:59 DEBUG "result": [ 22:38:59 DEBUG "resultType": "vector", 22:38:59 DEBUG "resultType": "vector", 22:38:59 DEBUG "result": [ 22:38:59 DEBUG "result": [ 22:38:59 DEBUG "resultType": "vector", 22:38:59 DEBUG "resultType": "vector", 22:38:59 DEBUG "result": [ 22:38:59 DEBUG "result": [ 22:38:59 DEBUG "result": [ 22:38:59 DEBUG "result": [ 22:38:59 DEBUG "result": [ 22:38:59 DEBUG "result": [ 22:38:59 DEBUG "data": { 22:38:59 DEBUG "result": [ 22:38:59 DEBUG "result": [ 22:38:59 DEBUG "result": [ 22:38:59 DEBUG "resultType": "vector", 22:38:59 DEBUG { 22:38:59 DEBUG "resultType": "vector", 22:38:59 DEBUG "result": [ 22:38:59 DEBUG "resultType": "vector", 22:38:59 DEBUG { 22:38:59 DEBUG "result": [ 22:38:59 DEBUG { 22:38:59 DEBUG { 22:38:59 DEBUG "result": [ 22:38:59 DEBUG "result": [ 22:38:59 DEBUG { 22:38:59 DEBUG { 22:38:59 DEBUG "result": [ 22:38:59 DEBUG "result": [ 22:38:59 DEBUG { 22:38:59 DEBUG { 22:38:59 DEBUG { 22:38:59 DEBUG { 22:38:59 DEBUG { 22:38:59 DEBUG { 22:38:59 DEBUG "resultType": "vector", 22:38:59 DEBUG { 22:38:59 DEBUG { 22:38:59 DEBUG { 22:38:59 DEBUG "result": [ 22:38:59 DEBUG "metric": { 22:38:59 DEBUG "result": [ 22:38:59 DEBUG { 22:38:59 DEBUG "result": [ 22:38:59 DEBUG "metric": { 22:38:59 DEBUG { 22:38:59 DEBUG "metric": { 22:38:59 DEBUG "metric": { 22:38:59 DEBUG { 22:38:59 DEBUG "metric": { 22:38:59 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:38:59 DEBUG "metric": { 22:38:59 DEBUG { 22:38:59 DEBUG { 22:38:59 DEBUG "metric": { 22:38:59 DEBUG "metric": { 22:38:59 DEBUG "metric": { 22:38:59 DEBUG "metric": { 22:38:59 DEBUG "metric": { 22:38:59 DEBUG "metric": { 22:38:59 DEBUG "pod": "ds-idrepo-2" 22:38:59 DEBUG "metric": { 22:38:59 DEBUG "metric": { 22:38:59 DEBUG "metric": { 22:38:59 DEBUG { 22:38:59 DEBUG "pod": "ds-idrepo-2" 22:38:59 DEBUG { 22:38:59 DEBUG "metric": { 22:38:59 DEBUG { 22:38:59 DEBUG "instance": "10.142.0.107:9100" 22:38:59 DEBUG "metric": { 22:38:59 DEBUG "pod": "am-555f4db595-fz8sn" 22:38:59 DEBUG "pod": "am-555f4db595-fz8sn" 22:38:59 DEBUG { 22:38:59 DEBUG "metric": { 22:38:59 DEBUG }, 22:38:59 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:38:59 DEBUG "metric": { 22:38:59 DEBUG "metric": { 22:38:59 DEBUG "instance": "10.142.0.107:9100" 22:38:59 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:38:59 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:38:59 DEBUG "pod": "am-555f4db595-fz8sn" 22:38:59 DEBUG "pod": "ds-idrepo-2" 22:38:59 DEBUG "result": [ 22:38:59 DEBUG }, 22:38:59 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:38:59 DEBUG "pod": "ds-idrepo-2" 22:38:59 DEBUG "pod": "am-555f4db595-fz8sn" 22:38:59 DEBUG "metric": { 22:38:59 DEBUG }, 22:38:59 DEBUG "metric": { 22:38:59 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:38:59 DEBUG "metric": { 22:38:59 DEBUG }, 22:38:59 DEBUG "pod": "am-555f4db595-fz8sn" 22:38:59 DEBUG }, 22:38:59 DEBUG }, 22:38:59 DEBUG "metric": { 22:38:59 DEBUG "pod": "am-555f4db595-fz8sn" 22:38:59 DEBUG "value": [ 22:38:59 DEBUG }, 22:38:59 DEBUG "pod": "ds-idrepo-2" 22:38:59 DEBUG "pod": "ds-idrepo-2" 22:38:59 DEBUG }, 22:38:59 DEBUG }, 22:38:59 DEBUG }, 22:38:59 DEBUG }, 22:38:59 DEBUG }, 22:38:59 DEBUG { 22:38:59 DEBUG "value": [ 22:38:59 DEBUG }, 22:38:59 DEBUG }, 22:38:59 DEBUG }, 22:38:59 DEBUG "pod": "ds-idrepo-2" 22:38:59 DEBUG "value": [ 22:38:59 DEBUG "instance": "10.142.0.107:9100" 22:38:59 DEBUG }, 22:38:59 DEBUG "instance": "10.142.0.107:9100" 22:38:59 DEBUG "value": [ 22:38:59 DEBUG }, 22:38:59 DEBUG "value": [ 22:38:59 DEBUG "value": [ 22:38:59 DEBUG "pod": "am-555f4db595-fz8sn" 22:38:59 DEBUG }, 22:38:59 DEBUG 22:38:59 DEBUG "value": [ 22:38:59 DEBUG }, 22:38:59 DEBUG }, 22:38:59 DEBUG "value": [ 22:38:59 DEBUG "value": [ 22:38:59 DEBUG "value": [ 22:38:59 DEBUG "value": [ 22:38:59 DEBUG "value": [ 22:38:59 DEBUG "metric": { 22:38:59 DEBUG 22:38:59 DEBUG "value": [ 22:38:59 DEBUG "value": [ 22:38:59 DEBUG "value": [ 22:38:59 DEBUG }, 22:38:59 DEBUG 22:38:59 DEBUG }, 22:38:59 DEBUG "value": [ 22:38:59 DEBUG }, 22:38:59 DEBUG 22:38:59 DEBUG "value": [ 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG }, 22:38:59 DEBUG "value": [ 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG "value": [ 22:38:59 DEBUG "value": [ 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG "instance": "10.142.0.107:9100" 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG "value": [ 22:38:59 DEBUG 22:38:59 DEBUG "value": [ 22:38:59 DEBUG 22:38:59 DEBUG "value": [ 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG "value": [ 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG }, 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG [...] 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG "value": [ 22:38:59 DEBUG [...] 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG [...] 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG [...] 22:38:59 DEBUG 22:38:59 DEBUG [...] 22:38:59 DEBUG [...] 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG [...] 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG [...] 22:38:59 DEBUG [...] 22:38:59 DEBUG [...] 22:38:59 DEBUG [...] 22:38:59 DEBUG [...] 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG [...] 22:38:59 DEBUG [...] 22:38:59 DEBUG [...] 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG [...] 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG [...] 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG [...] 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG [...] 22:38:59 DEBUG [...] 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG [...] 22:38:59 DEBUG 22:38:59 DEBUG [...] 22:38:59 DEBUG 22:38:59 DEBUG [...] 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG [...] 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG "pod": "am-555f4db595-kzxsp" 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG }, 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG [...] 22:38:59 DEBUG "pod": "ds-cts-2" 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG "pod": "ds-idrepo-0" 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG "instance": "10.142.0.43:9100" 22:38:59 DEBUG 22:38:59 DEBUG "pod": "am-555f4db595-kzxsp" 22:38:59 DEBUG "pod": "am-555f4db595-kzxsp" 22:38:59 DEBUG 22:38:59 DEBUG "pod": "idm-b85f9d978-j5pbm" 22:38:59 DEBUG 22:38:59 DEBUG "value": [ 22:38:59 DEBUG 22:38:59 DEBUG 22:38:59 DEBUG "instance": "10.142.0.43:9100" 22:38:59 DEBUG "pod": "idm-b85f9d978-j5pbm" 22:38:59 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:38:59 DEBUG "pod": "am-555f4db595-kzxsp" 22:38:59 DEBUG "pod": "ds-idrepo-0" 22:38:59 DEBUG 22:38:59 DEBUG }, 22:38:59 DEBUG "pod": "idm-b85f9d978-j5pbm" 22:38:59 DEBUG "pod": "ds-idrepo-0" 22:38:59 DEBUG "pod": "am-555f4db595-kzxsp" 22:38:59 DEBUG }, 22:38:59 DEBUG }, 22:38:59 DEBUG "value": [ 22:38:59 DEBUG "pod": "idm-b85f9d978-j5pbm" 22:38:59 DEBUG 22:38:59 DEBUG }, 22:38:59 DEBUG "pod": "am-555f4db595-kzxsp" 22:38:59 DEBUG }, 22:38:59 DEBUG }, 22:38:59 DEBUG 22:38:59 DEBUG }, 22:38:59 DEBUG 22:38:59 DEBUG 1713303539, 22:38:59 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:38:59 DEBUG "pod": "ds-idrepo-0" 22:38:59 DEBUG }, 22:38:59 DEBUG }, 22:38:59 DEBUG }, 22:38:59 DEBUG }, 22:38:59 DEBUG }, 22:38:59 DEBUG 22:38:59 DEBUG "value": [ 22:38:59 DEBUG }, 22:38:59 DEBUG }, 22:38:59 DEBUG 22:38:59 DEBUG "value": [ 22:38:59 DEBUG 1713303539, 22:38:59 DEBUG 22:38:59 DEBUG }, 22:38:59 DEBUG "instance": "10.142.0.43:9100" 22:38:59 DEBUG "value": [ 22:38:59 DEBUG }, 22:38:59 DEBUG "value": [ 22:38:59 DEBUG "value": [ 22:38:59 DEBUG "pod": "am-555f4db595-kzxsp" 22:38:59 DEBUG "value": [ 22:38:59 DEBUG "pod": "ds-idrepo-0" 22:38:59 DEBUG "1994.0666666666666" 22:38:59 DEBUG }, 22:38:59 DEBUG }, 22:38:59 DEBUG "value": [ 22:38:59 DEBUG "value": [ 22:38:59 DEBUG "value": [ 22:38:59 DEBUG "value": [ 22:38:59 DEBUG "value": [ 22:38:59 DEBUG 22:38:59 DEBUG 1713303539, 22:38:59 DEBUG "value": [ 22:38:59 DEBUG "value": [ 22:38:59 DEBUG "pod": "ds-idrepo-0" 22:38:59 DEBUG 1713303539, 22:38:59 DEBUG "0" 22:38:59 DEBUG "instance": "10.142.0.43:9100" 22:38:59 DEBUG "value": [ 22:38:59 DEBUG }, 22:38:59 DEBUG 1713303539, 22:38:59 DEBUG "value": [ 22:38:59 DEBUG 1713303539, 22:38:59 DEBUG 1713303539, 22:38:59 DEBUG }, 22:38:59 DEBUG 1713303539, 22:38:59 DEBUG }, 22:38:59 DEBUG ] 22:38:59 DEBUG "value": [ 22:38:59 DEBUG "value": [ 22:38:59 DEBUG 1713303539, 22:38:59 DEBUG 1713303539, 22:38:59 DEBUG 1713303539, 22:38:59 DEBUG 1713303539, 22:38:59 DEBUG 1713303539, 22:38:59 DEBUG "instance": "10.142.0.43:9100" 22:38:59 DEBUG "0" 22:38:59 DEBUG 1713303539, 22:38:59 DEBUG 1713303539, 22:38:59 DEBUG }, 22:38:59 DEBUG "1993.9333333333334" 22:38:59 DEBUG ] 22:38:59 DEBUG }, 22:38:59 DEBUG 1713303539, 22:38:59 DEBUG "value": [ 22:38:59 DEBUG "0.00046666666666700016" 22:38:59 DEBUG 1713303539, 22:38:59 DEBUG "1994.1" 22:38:59 DEBUG "0" 22:38:59 DEBUG "value": [ 22:38:59 DEBUG "0.002767703094598498" 22:38:59 DEBUG ] 22:38:59 DEBUG } 22:38:59 DEBUG 1713303539, 22:38:59 DEBUG 1713303539, 22:38:59 DEBUG "45431958.233333334" 22:38:59 DEBUG "300.72317462648215" 22:38:59 DEBUG "668995584" 22:38:59 DEBUG "3259151.5" 22:38:59 DEBUG "0" 22:38:59 DEBUG }, 22:38:59 DEBUG "value": [ 22:38:59 DEBUG "3007.2317462648216" 22:38:59 DEBUG "0" 22:38:59 DEBUG "value": [ 22:38:59 DEBUG ] 22:38:59 DEBUG } 22:38:59 DEBUG "value": [ 22:38:59 DEBUG "0" 22:38:59 DEBUG 1713303539, 22:38:59 DEBUG ] 22:38:59 DEBUG "0" 22:38:59 DEBUG ] 22:38:59 DEBUG ] 22:38:59 DEBUG 1713303539, 22:38:59 DEBUG } 22:38:59 DEBUG "value": [ 22:38:59 DEBUG ] 22:38:59 DEBUG "6.310810150493849" 22:38:59 DEBUG "0" 22:38:59 DEBUG ] 22:38:59 DEBUG ] 22:38:59 DEBUG ] 22:38:59 DEBUG ] 22:38:59 DEBUG ] 22:38:59 DEBUG ] 22:38:59 DEBUG 1713303539, 22:38:59 DEBUG ] 22:38:59 DEBUG ] 22:38:59 DEBUG 1713303539, 22:38:59 DEBUG } 22:38:59 DEBUG ] 22:38:59 DEBUG 1713303539, 22:38:59 DEBUG ] 22:38:59 DEBUG "19965043712" 22:38:59 DEBUG } 22:38:59 DEBUG ] 22:38:59 DEBUG } 22:38:59 DEBUG } 22:38:59 DEBUG "0.026" 22:38:59 DEBUG ] 22:38:59 DEBUG 1713303539, 22:38:59 DEBUG } 22:38:59 DEBUG ] 22:38:59 DEBUG ] 22:38:59 DEBUG } 22:38:59 DEBUG } 22:38:59 DEBUG } 22:38:59 DEBUG } 22:38:59 DEBUG } 22:38:59 DEBUG } 22:38:59 DEBUG "0" 22:38:59 DEBUG } 22:38:59 DEBUG } 22:38:59 DEBUG "0" 22:38:59 DEBUG ] 22:38:59 DEBUG } 22:38:59 DEBUG "45422890.4" 22:38:59 DEBUG } 22:38:59 DEBUG ] 22:38:59 DEBUG ] 22:38:59 DEBUG } 22:38:59 DEBUG ] 22:38:59 DEBUG ] 22:38:59 DEBUG ] 22:38:59 DEBUG } 22:38:59 DEBUG "0" 22:38:59 DEBUG } 22:38:59 DEBUG } 22:38:59 DEBUG } 22:38:59 DEBUG ] 22:38:59 DEBUG ] 22:38:59 DEBUG ] 22:38:59 DEBUG ] 22:38:59 DEBUG ] 22:38:59 DEBUG ] 22:38:59 DEBUG ] 22:38:59 DEBUG ] 22:38:59 DEBUG ] 22:38:59 DEBUG ] 22:38:59 DEBUG } 22:38:59 DEBUG } 22:38:59 DEBUG ] 22:38:59 DEBUG ] 22:38:59 DEBUG } 22:38:59 DEBUG } 22:38:59 DEBUG ] 22:38:59 DEBUG } 22:38:59 DEBUG } 22:38:59 DEBUG } 22:38:59 DEBUG } 22:38:59 DEBUG ] 22:38:59 DEBUG ] 22:38:59 DEBUG ] 22:38:59 DEBUG } 22:38:59 DEBUG } 22:38:59 DEBUG } 22:38:59 DEBUG } 22:38:59 DEBUG } 22:38:59 DEBUG } 22:38:59 DEBUG } 22:38:59 DEBUG } 22:38:59 DEBUG } 22:38:59 DEBUG } 22:38:59 DEBUG } 22:38:59 DEBUG } 22:38:59 DEBUG } 22:38:59 DEBUG ] 22:38:59 DEBUG } 22:38:59 DEBUG } 22:38:59 DEBUG } 22:38:59 DEBUG } 22:38:59 DEBUG ] 22:38:59 DEBUG } 22:38:59 DEBUG } 22:38:59 DEBUG } 22:38:59 DEBUG } 22:38:59 DEBUG } 22:38:59 DEBUG } 22:38:59 DEBUG } 22:38:59 DEBUG } 22:38:59 DEBUG } 22:38:59 DEBUG ] 22:38:59 DEBUG } 22:38:59 DEBUG } 22:38:59 DEBUG ] 22:38:59 DEBUG ] 22:38:59 DEBUG } 22:38:59 DEBUG } 22:38:59 DEBUG } 22:38:59 DEBUG } 22:38:59 DEBUG ] 22:38:59 DEBUG } 22:38:59 DEBUG } 22:38:59 INFO [loop_until]: OK (rc = 0) 22:38:59 DEBUG } 22:38:59 DEBUG } 22:38:59 DEBUG } 22:38:59 DEBUG } 22:38:59 DEBUG } 22:38:59 DEBUG } 22:38:59 DEBUG --- stdout --- 22:38:59 DEBUG } 22:38:59 DEBUG } 22:38:59 DEBUG } 22:38:59 DEBUG } 22:38:59 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 66m 0% 5098Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7331m 24% 14831Mi 13% gke-xlou-cdm-default-pool-1b0769c8-nhlf 9183m 30% 10836Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 81m 0% 2644Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 4757m 15% 6523Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 8897m 29% 16688Mi 14% gke-xlou-cdm-default-pool-1b0769c8-xnmp 56m 0% 3143Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 9468m 31% 10375Mi 9% gke-xlou-cdm-ds-0f601751-hmm7 4523m 15% 5624Mi 5% gke-xlou-cdm-ds-0f601751-j488 7645m 25% 14183Mi 12% gke-xlou-cdm-ds-0f601751-mc5d 10563m 35% 3237Mi 2% gke-xlou-cdm-ds-0f601751-nj6q 9701m 32% 10433Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 4672m 15% 7353Mi 6% gke-xlou-cdm-frontend-a8771548-i82n 6439m 40% 5952Mi 10% 22:38:59 DEBUG --- stderr --- 22:38:59 DEBUG 22:39:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713303596" 22:39:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303596" 22:39:56 ERROR [http_cmd]: ERROR 22:39:56 INFO [http_cmd]: http status code OK 22:39:56 ERROR --- status code --- 22:39:56 DEBUG --- status code --- 22:39:56 ERROR http status code is 400 (expected [200, 401, 403, 404]) 22:39:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:39:56 ERROR --- http response --- 22:39:56 DEBUG --- http response --- 22:39:56 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 22:39:56 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 22:39:56 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 22:39:58 INFO 22:39:58 INFO [loop_until]: kubectl --namespace=xlou top pods 22:39:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:39:58 INFO [loop_until]: OK (rc = 0) 22:39:58 DEBUG --- stdout --- 22:39:58 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 9Mi am-555f4db595-4lbss 9171m 8608Mi am-555f4db595-fz8sn 9797m 8623Mi am-555f4db595-kzxsp 9028m 8567Mi ds-cts-0 8581m 12190Mi ds-cts-1 7565m 12279Mi ds-cts-2 7320m 12242Mi ds-idrepo-0 4671m 5028Mi ds-idrepo-1 4343m 2875Mi ds-idrepo-2 4345m 3160Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 3m 1207Mi idm-b85f9d978-rl76f 4m 1225Mi lodemon-bc756c647-cr7x6 2m 63Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-7c9b667974-9mr2s 6182m 549Mi 22:39:58 DEBUG --- stderr --- 22:39:58 DEBUG 22:39:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303599" 22:39:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303599" 22:39:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303599" 22:39:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303599" 22:39:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303599" 22:39:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303599" 22:39:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303599" 22:39:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303599" 22:39:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303599" 22:39:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713303599" 22:39:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303599" 22:39:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303599" 22:39:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713303599" 22:39:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303599" 22:39:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303599" 22:39:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303599" 22:39:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713303599" 22:39:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303599" 22:39:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303599" 22:39:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303599" 22:39:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303599" 22:39:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303599" 22:39:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303599" 22:39:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303599" 22:39:59 INFO 22:39:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713303599" 22:40:00 INFO [http_cmd]: http status code OK 22:40:00 INFO [http_cmd]: http status code OK 22:40:00 INFO [http_cmd]: http status code OK 22:40:00 INFO [http_cmd]: http status code OK 22:40:00 INFO [http_cmd]: http status code OK 22:40:00 INFO [http_cmd]: http status code OK 22:40:00 INFO [http_cmd]: http status code OK 22:40:00 INFO [http_cmd]: http status code OK 22:40:00 INFO [http_cmd]: http status code OK 22:40:00 INFO [http_cmd]: http status code OK 22:40:00 INFO [http_cmd]: http status code OK 22:40:00 INFO [http_cmd]: http status code OK 22:40:00 INFO [http_cmd]: http status code OK 22:40:00 INFO [http_cmd]: http status code OK 22:40:00 INFO [http_cmd]: http status code OK 22:40:00 INFO [http_cmd]: http status code OK 22:40:00 INFO [http_cmd]: http status code OK 22:40:00 INFO [loop_until]: kubectl --namespace=xlou top node 22:40:00 INFO [http_cmd]: http status code OK 22:40:00 DEBUG --- status code --- 22:40:00 INFO [http_cmd]: http status code OK 22:40:00 DEBUG --- status code --- 22:40:00 DEBUG --- status code --- 22:40:00 INFO [http_cmd]: http status code OK 22:40:00 DEBUG --- status code --- 22:40:00 INFO [http_cmd]: http status code OK 22:40:00 INFO [http_cmd]: http status code OK 22:40:00 INFO [http_cmd]: http status code OK 22:40:00 DEBUG --- status code --- 22:40:00 DEBUG --- status code --- 22:40:00 DEBUG --- status code --- 22:40:00 DEBUG --- status code --- 22:40:00 DEBUG --- status code --- 22:40:00 DEBUG --- status code --- 22:40:00 DEBUG --- status code --- 22:40:00 DEBUG --- status code --- 22:40:00 DEBUG --- status code --- 22:40:00 DEBUG --- status code --- 22:40:00 DEBUG --- status code --- 22:40:00 DEBUG --- status code --- 22:40:00 DEBUG --- status code --- 22:40:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:40:00 DEBUG --- status code --- 22:40:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:40:00 DEBUG --- status code --- 22:40:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:40:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:40:00 DEBUG --- status code --- 22:40:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:40:00 DEBUG --- status code --- 22:40:00 DEBUG --- status code --- 22:40:00 DEBUG --- status code --- 22:40:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:40:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:40:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:40:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:40:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:40:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:40:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:40:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:40:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:40:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:40:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:40:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:40:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:40:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:40:00 DEBUG --- http response --- 22:40:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:40:00 DEBUG --- http response --- 22:40:00 INFO [http_cmd]: http status code OK 22:40:00 DEBUG --- http response --- 22:40:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:40:00 DEBUG --- http response --- 22:40:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:40:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:40:00 INFO [http_cmd]: http status code OK 22:40:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:40:00 DEBUG --- http response --- 22:40:00 DEBUG --- http response --- 22:40:00 DEBUG --- http response --- 22:40:00 DEBUG --- http response --- 22:40:00 DEBUG --- http response --- 22:40:00 DEBUG --- http response --- 22:40:00 DEBUG --- http response --- 22:40:00 DEBUG --- http response --- 22:40:00 DEBUG --- http response --- 22:40:00 DEBUG --- http response --- 22:40:00 DEBUG --- http response --- 22:40:00 DEBUG --- http response --- 22:40:00 DEBUG --- http response --- 22:40:00 DEBUG --- http response --- 22:40:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:40:00 DEBUG --- http response --- 22:40:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:40:00 DEBUG --- status code --- 22:40:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:40:00 DEBUG --- http response --- 22:40:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:40:00 DEBUG --- http response --- 22:40:00 DEBUG --- http response --- 22:40:00 DEBUG --- status code --- 22:40:00 DEBUG --- http response --- 22:40:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:40:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:40:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:40:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:40:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:40:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:40:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:40:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:40:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:40:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:40:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:40:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:40:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:40:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:40:00 DEBUG ----- output ----- 22:40:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:40:00 DEBUG ----- output ----- 22:40:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:40:00 DEBUG ----- output ----- 22:40:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:40:00 DEBUG ----- output ----- 22:40:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:40:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:40:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:40:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:40:00 DEBUG ----- output ----- 22:40:00 DEBUG ----- output ----- 22:40:00 DEBUG ----- output ----- 22:40:00 DEBUG ----- output ----- 22:40:00 DEBUG ----- output ----- 22:40:00 DEBUG ----- output ----- 22:40:00 DEBUG ----- output ----- 22:40:00 DEBUG ----- output ----- 22:40:00 DEBUG ----- output ----- 22:40:00 DEBUG ----- output ----- 22:40:00 DEBUG ----- output ----- 22:40:00 DEBUG ----- output ----- 22:40:00 DEBUG ----- output ----- 22:40:00 DEBUG ----- output ----- 22:40:00 DEBUG { 22:40:00 DEBUG ----- output ----- 22:40:00 DEBUG { 22:40:00 DEBUG --- http response --- 22:40:00 DEBUG { 22:40:00 DEBUG ----- output ----- 22:40:00 DEBUG { 22:40:00 DEBUG ----- output ----- 22:40:00 DEBUG ----- output ----- 22:40:00 DEBUG --- http response --- 22:40:00 DEBUG ----- output ----- 22:40:00 DEBUG { 22:40:00 DEBUG { 22:40:00 DEBUG { 22:40:00 DEBUG { 22:40:00 DEBUG { 22:40:00 DEBUG { 22:40:00 DEBUG { 22:40:00 DEBUG { 22:40:00 DEBUG { 22:40:00 DEBUG { 22:40:00 DEBUG "status": "success", 22:40:00 DEBUG { 22:40:00 DEBUG { 22:40:00 DEBUG { 22:40:00 DEBUG "status": "success", 22:40:00 DEBUG { 22:40:00 DEBUG "status": "success", 22:40:00 DEBUG "status": "success", 22:40:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:40:00 DEBUG { 22:40:00 DEBUG "status": "success", 22:40:00 DEBUG { 22:40:00 DEBUG { 22:40:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:40:00 DEBUG { 22:40:00 DEBUG "status": "success", 22:40:00 DEBUG "status": "success", 22:40:00 DEBUG "status": "success", 22:40:00 DEBUG "status": "success", 22:40:00 DEBUG "status": "success", 22:40:00 DEBUG "status": "success", 22:40:00 DEBUG "status": "success", 22:40:00 DEBUG "status": "success", 22:40:00 DEBUG "status": "success", 22:40:00 DEBUG { 22:40:00 DEBUG "data": { 22:40:00 DEBUG "status": "success", 22:40:00 DEBUG "status": "success", 22:40:00 DEBUG "status": "success", 22:40:00 DEBUG "data": { 22:40:00 DEBUG "status": "success", 22:40:00 DEBUG "data": { 22:40:00 DEBUG "data": { 22:40:00 DEBUG ----- output ----- 22:40:00 DEBUG "status": "success", 22:40:00 DEBUG "data": { 22:40:00 DEBUG "status": "success", 22:40:00 DEBUG "status": "success", 22:40:00 DEBUG ----- output ----- 22:40:00 DEBUG "status": "success", 22:40:00 DEBUG "data": { 22:40:00 DEBUG "data": { 22:40:00 DEBUG "data": { 22:40:00 DEBUG "data": { 22:40:00 DEBUG "data": { 22:40:00 DEBUG "data": { 22:40:00 DEBUG "data": { 22:40:00 DEBUG "data": { 22:40:00 DEBUG "data": { 22:40:00 DEBUG "status": "success", 22:40:00 DEBUG "resultType": "vector", 22:40:00 DEBUG "data": { 22:40:00 DEBUG "data": { 22:40:00 DEBUG "data": { 22:40:00 DEBUG "resultType": "vector", 22:40:00 DEBUG "data": { 22:40:00 DEBUG "resultType": "vector", 22:40:00 DEBUG "resultType": "vector", 22:40:00 DEBUG { 22:40:00 DEBUG "data": { 22:40:00 DEBUG "resultType": "vector", 22:40:00 DEBUG "data": { 22:40:00 DEBUG "resultType": "vector", 22:40:00 DEBUG "data": { 22:40:00 DEBUG "data": { 22:40:00 DEBUG "resultType": "vector", 22:40:00 DEBUG "resultType": "vector", 22:40:00 DEBUG "resultType": "vector", 22:40:00 DEBUG "resultType": "vector", 22:40:00 DEBUG "resultType": "vector", 22:40:00 DEBUG "resultType": "vector", 22:40:00 DEBUG "resultType": "vector", 22:40:00 DEBUG "resultType": "vector", 22:40:00 DEBUG "resultType": "vector", 22:40:00 DEBUG "data": { 22:40:00 DEBUG "result": [ 22:40:00 DEBUG "resultType": "vector", 22:40:00 DEBUG "resultType": "vector", 22:40:00 DEBUG "resultType": "vector", 22:40:00 DEBUG "result": [ 22:40:00 DEBUG "resultType": "vector", 22:40:00 DEBUG "result": [ 22:40:00 DEBUG "result": [ 22:40:00 DEBUG "status": "success", 22:40:00 DEBUG "resultType": "vector", 22:40:00 DEBUG "result": [ 22:40:00 DEBUG { 22:40:00 DEBUG "result": [ 22:40:00 DEBUG "resultType": "vector", 22:40:00 DEBUG "resultType": "vector", 22:40:00 DEBUG "result": [ 22:40:00 DEBUG "result": [ 22:40:00 DEBUG "result": [ 22:40:00 DEBUG "result": [ 22:40:00 DEBUG "result": [ 22:40:00 DEBUG "result": [ 22:40:00 DEBUG "result": [ 22:40:00 DEBUG "result": [ 22:40:00 DEBUG "result": [ 22:40:00 DEBUG "resultType": "vector", 22:40:00 DEBUG { 22:40:00 DEBUG "metric": { 22:40:00 DEBUG "result": [ 22:40:00 DEBUG "result": [ 22:40:00 DEBUG { 22:40:00 DEBUG "metric": { 22:40:00 DEBUG { 22:40:00 DEBUG { 22:40:00 DEBUG "data": { 22:40:00 DEBUG "result": [ 22:40:00 DEBUG { 22:40:00 DEBUG "status": "success", 22:40:00 DEBUG "data": { 22:40:00 DEBUG "result": [ 22:40:00 DEBUG "result": [ 22:40:00 DEBUG { 22:40:00 DEBUG { 22:40:00 DEBUG { 22:40:00 DEBUG { 22:40:00 DEBUG { 22:40:00 DEBUG { 22:40:00 DEBUG "metric": { 22:40:00 DEBUG { 22:40:00 DEBUG { 22:40:00 DEBUG "result": [ 22:40:00 DEBUG "result": [ 22:40:00 DEBUG "pod": "ds-idrepo-2" 22:40:00 DEBUG { 22:40:00 DEBUG { 22:40:00 DEBUG "pod": "am-555f4db595-fz8sn" 22:40:00 DEBUG "metric": { 22:40:00 DEBUG "metric": { 22:40:00 DEBUG "resultType": "vector", 22:40:00 DEBUG { 22:40:00 DEBUG "metric": { 22:40:00 DEBUG "resultType": "vector", 22:40:00 DEBUG { 22:40:00 DEBUG { 22:40:00 DEBUG "metric": { 22:40:00 DEBUG "metric": { 22:40:00 DEBUG "metric": { 22:40:00 DEBUG "metric": { 22:40:00 DEBUG "metric": { 22:40:00 DEBUG "pod": "am-555f4db595-fz8sn" 22:40:00 DEBUG { 22:40:00 DEBUG "result": [ 22:40:00 DEBUG "metric": { 22:40:00 DEBUG "metric": { 22:40:00 DEBUG { 22:40:00 DEBUG { 22:40:00 DEBUG }, 22:40:00 DEBUG "metric": { 22:40:00 DEBUG "metric": { 22:40:00 DEBUG { 22:40:00 DEBUG }, 22:40:00 DEBUG "instance": "10.142.0.107:9100" 22:40:00 DEBUG }, 22:40:00 DEBUG "result": [ 22:40:00 DEBUG "metric": { 22:40:00 DEBUG "pod": "am-555f4db595-fz8sn" 22:40:00 DEBUG "result": [ 22:40:00 DEBUG "metric": { 22:40:00 DEBUG "metric": { 22:40:00 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:40:00 DEBUG "pod": "ds-idrepo-2" 22:40:00 DEBUG "instance": "10.142.0.107:9100" 22:40:00 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:40:00 DEBUG "pod": "am-555f4db595-fz8sn" 22:40:00 DEBUG }, 22:40:00 DEBUG "metric": { 22:40:00 DEBUG { 22:40:00 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:40:00 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:40:00 DEBUG "metric": { 22:40:00 DEBUG "metric": { 22:40:00 DEBUG "value": [ 22:40:00 DEBUG "pod": "ds-idrepo-2" 22:40:00 DEBUG "pod": "ds-idrepo-2" 22:40:00 DEBUG }, 22:40:00 DEBUG "value": [ 22:40:00 DEBUG "value": [ 22:40:00 DEBUG { 22:40:00 DEBUG "instance": "10.142.0.107:9100" 22:40:00 DEBUG }, 22:40:00 DEBUG { 22:40:00 DEBUG "pod": "ds-idrepo-2" 22:40:00 DEBUG "pod": "ds-idrepo-2" 22:40:00 DEBUG }, 22:40:00 DEBUG }, 22:40:00 DEBUG }, 22:40:00 DEBUG }, 22:40:00 DEBUG }, 22:40:00 DEBUG "value": [ 22:40:00 DEBUG "pod": "ds-idrepo-2" 22:40:00 DEBUG "metric": { 22:40:00 DEBUG }, 22:40:00 DEBUG }, 22:40:00 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:40:00 DEBUG "pod": "am-555f4db595-fz8sn" 22:40:00 DEBUG 22:40:00 DEBUG }, 22:40:00 DEBUG "metric": { 22:40:00 DEBUG "value": [ 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG "metric": { 22:40:00 DEBUG }, 22:40:00 DEBUG "value": [ 22:40:00 DEBUG "metric": { 22:40:00 DEBUG }, 22:40:00 DEBUG }, 22:40:00 DEBUG "value": [ 22:40:00 DEBUG "pod": "am-555f4db595-fz8sn" 22:40:00 DEBUG "value": [ 22:40:00 DEBUG "value": [ 22:40:00 DEBUG "value": [ 22:40:00 DEBUG "value": [ 22:40:00 DEBUG 22:40:00 DEBUG }, 22:40:00 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:40:00 DEBUG "value": [ 22:40:00 DEBUG "value": [ 22:40:00 DEBUG }, 22:40:00 DEBUG }, 22:40:00 DEBUG 22:40:00 DEBUG "value": [ 22:40:00 DEBUG "instance": "10.142.0.107:9100" 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG "instance": "10.142.0.107:9100" 22:40:00 DEBUG "value": [ 22:40:00 DEBUG 22:40:00 DEBUG "pod": "am-555f4db595-fz8sn" 22:40:00 DEBUG "value": [ 22:40:00 DEBUG "value": [ 22:40:00 DEBUG 22:40:00 DEBUG }, 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG "value": [ 22:40:00 DEBUG }, 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG "value": [ 22:40:00 DEBUG "value": [ 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG }, 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG }, 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG }, 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG "value": [ 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG "value": [ 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG [...] 22:40:00 DEBUG 22:40:00 DEBUG "value": [ 22:40:00 DEBUG 22:40:00 DEBUG [...] 22:40:00 DEBUG [...] 22:40:00 DEBUG "value": [ 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG "value": [ 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG [...] 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG [...] 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG [...] 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG [...] 22:40:00 DEBUG 22:40:00 DEBUG [...] 22:40:00 DEBUG [...] 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG [...] 22:40:00 DEBUG [...] 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG [...] 22:40:00 DEBUG [...] 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG [...] 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG [...] 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG [...] 22:40:00 DEBUG [...] 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG [...] 22:40:00 DEBUG [...] 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG [...] 22:40:00 DEBUG [...] 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG [...] 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG "pod": "ds-idrepo-0" 22:40:00 DEBUG 22:40:00 DEBUG [...] 22:40:00 DEBUG 22:40:00 DEBUG "pod": "am-555f4db595-kzxsp" 22:40:00 DEBUG "instance": "10.142.0.43:9100" 22:40:00 DEBUG [...] 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG [...] 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG "pod": "ds-idrepo-0" 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG "pod": "am-555f4db595-kzxsp" 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG }, 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG "pod": "ds-idrepo-0" 22:40:00 DEBUG }, 22:40:00 DEBUG }, 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG "pod": "am-555f4db595-kzxsp" 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG "pod": "ds-idrepo-0" 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG }, 22:40:00 DEBUG "pod": "lodemon-bc756c647-cr7x6" 22:40:00 DEBUG "pod": "am-555f4db595-kzxsp" 22:40:00 DEBUG }, 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:40:00 DEBUG "pod": "lodemon-bc756c647-cr7x6" 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG "value": [ 22:40:00 DEBUG "pod": "ds-idrepo-0" 22:40:00 DEBUG 22:40:00 DEBUG }, 22:40:00 DEBUG "value": [ 22:40:00 DEBUG "value": [ 22:40:00 DEBUG 22:40:00 DEBUG "instance": "10.142.0.43:9100" 22:40:00 DEBUG }, 22:40:00 DEBUG "pod": "ds-idrepo-2" 22:40:00 DEBUG 22:40:00 DEBUG }, 22:40:00 DEBUG 22:40:00 DEBUG 22:40:00 DEBUG "value": [ 22:40:00 DEBUG }, 22:40:00 DEBUG }, 22:40:00 DEBUG "value": [ 22:40:00 DEBUG "pod": "ds-cts-2" 22:40:00 DEBUG "instance": "10.142.0.43:9100" 22:40:00 DEBUG 22:40:00 DEBUG }, 22:40:00 DEBUG }, 22:40:00 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:40:00 DEBUG "pod": "am-555f4db595-kzxsp" 22:40:00 DEBUG 1713303599, 22:40:00 DEBUG }, 22:40:00 DEBUG 22:40:00 DEBUG "value": [ 22:40:00 DEBUG 1713303599, 22:40:00 DEBUG 1713303599, 22:40:00 DEBUG 22:40:00 DEBUG }, 22:40:00 DEBUG "value": [ 22:40:00 DEBUG }, 22:40:00 DEBUG "pod": "am-555f4db595-kzxsp" 22:40:00 DEBUG "value": [ 22:40:00 DEBUG "pod": "am-555f4db595-kzxsp" 22:40:00 DEBUG "pod": "ds-idrepo-0" 22:40:00 DEBUG 1713303599, 22:40:00 DEBUG "value": [ 22:40:00 DEBUG "value": [ 22:40:00 DEBUG 1713303599, 22:40:00 DEBUG }, 22:40:00 DEBUG }, 22:40:00 DEBUG "pod": "ds-idrepo-2" 22:40:00 DEBUG "value": [ 22:40:00 DEBUG "value": [ 22:40:00 DEBUG }, 22:40:00 DEBUG }, 22:40:00 DEBUG "0" 22:40:00 DEBUG ] 22:40:00 DEBUG } 22:40:00 DEBUG 1713303599, 22:40:00 DEBUG "0" 22:40:00 DEBUG "instance": "10.142.0.43:9100" 22:40:00 DEBUG "instance": "10.142.0.43:9100" 22:40:00 DEBUG "value": [ 22:40:00 DEBUG 1713303599, 22:40:00 DEBUG "value": [ 22:40:00 DEBUG }, 22:40:00 DEBUG 1713303599, 22:40:00 DEBUG }, 22:40:00 DEBUG }, 22:40:00 DEBUG "0" 22:40:00 DEBUG 1713303599, 22:40:00 DEBUG 1713303599, 22:40:00 DEBUG "0" 22:40:00 DEBUG "value": [ 22:40:00 DEBUG "value": [ 22:40:00 DEBUG }, 22:40:00 DEBUG 1713303599, 22:40:00 DEBUG 1713303599, 22:40:00 DEBUG "value": [ 22:40:00 DEBUG "value": [ 22:40:00 DEBUG ] 22:40:00 DEBUG "value": [ 22:40:00 DEBUG "1992.4333333333334" 22:40:00 DEBUG "0.0005000000000004927" 22:40:00 DEBUG ] 22:40:00 DEBUG }, 22:40:00 DEBUG }, 22:40:00 DEBUG 1713303599, 22:40:00 DEBUG "1992.1666666666667" 22:40:00 DEBUG 1713303599, 22:40:00 DEBUG "value": [ 22:40:00 DEBUG "0" 22:40:00 DEBUG "value": [ 22:40:00 DEBUG "value": [ 22:40:00 DEBUG ] 22:40:00 DEBUG "4016.9603968963625" 22:40:00 DEBUG "3378980.5" 22:40:00 DEBUG ] 22:40:00 DEBUG 1713303599, 22:40:00 DEBUG 1713303599, 22:40:00 DEBUG "value": [ 22:40:00 DEBUG "671236096" 22:40:00 DEBUG "0" 22:40:00 DEBUG 1713303599, 22:40:00 DEBUG 1713303599, 22:40:00 DEBUG } 22:40:00 DEBUG 1713303599, 22:40:00 DEBUG ] 22:40:00 DEBUG ] 22:40:00 DEBUG } 22:40:00 DEBUG "value": [ 22:40:00 DEBUG "value": [ 22:40:00 DEBUG "19965637632" 22:40:00 DEBUG ] 22:40:00 DEBUG "0" 22:40:00 DEBUG 1713303599, 22:40:00 DEBUG ] 22:40:00 DEBUG 1713303599, 22:40:00 DEBUG 1713303599, 22:40:00 DEBUG } 22:40:00 DEBUG ] 22:40:00 DEBUG ] 22:40:00 DEBUG } 22:40:00 DEBUG "0" 22:40:00 DEBUG "45365683.599999994" 22:40:00 DEBUG 1713303599, 22:40:00 DEBUG ] 22:40:00 DEBUG ] 22:40:00 DEBUG "6.315603861646959" 22:40:00 DEBUG "0" 22:40:00 DEBUG } 22:40:00 DEBUG "0" 22:40:00 DEBUG } 22:40:00 DEBUG } 22:40:00 DEBUG ] 22:40:00 DEBUG 1713303599, 22:40:00 DEBUG 1713303599, 22:40:00 DEBUG ] 22:40:00 DEBUG } 22:40:00 DEBUG ] 22:40:00 DEBUG "0.026" 22:40:00 DEBUG } 22:40:00 DEBUG "1992.1333333333332" 22:40:00 DEBUG "0" 22:40:00 DEBUG ] 22:40:00 DEBUG } 22:40:00 DEBUG } 22:40:00 DEBUG ] 22:40:00 DEBUG ] 22:40:00 DEBUG ] 22:40:00 DEBUG "0" 22:40:00 DEBUG } 22:40:00 DEBUG } 22:40:00 DEBUG ] 22:40:00 DEBUG ] 22:40:00 DEBUG ] 22:40:00 DEBUG ] 22:40:00 DEBUG ] 22:40:00 DEBUG } 22:40:00 DEBUG "45356124.666666664" 22:40:00 DEBUG "0.00033333333333333365" 22:40:00 DEBUG } 22:40:00 DEBUG ] 22:40:00 DEBUG } 22:40:00 DEBUG ] 22:40:00 DEBUG ] 22:40:00 DEBUG ] 22:40:00 DEBUG ] 22:40:00 DEBUG } 22:40:00 DEBUG ] 22:40:00 DEBUG ] 22:40:00 DEBUG } 22:40:00 DEBUG } 22:40:00 DEBUG } 22:40:00 DEBUG ] 22:40:00 DEBUG ] 22:40:00 DEBUG ] 22:40:00 DEBUG } 22:40:00 DEBUG } 22:40:00 DEBUG } 22:40:00 DEBUG } 22:40:00 DEBUG } 22:40:00 DEBUG } 22:40:00 DEBUG ] 22:40:00 DEBUG ] 22:40:00 DEBUG ] 22:40:00 DEBUG } 22:40:00 DEBUG ] 22:40:00 DEBUG } 22:40:00 DEBUG } 22:40:00 DEBUG } 22:40:00 DEBUG } 22:40:00 DEBUG } 22:40:00 DEBUG } 22:40:00 DEBUG } 22:40:00 DEBUG } 22:40:00 DEBUG ] 22:40:00 DEBUG ] 22:40:00 DEBUG } 22:40:00 DEBUG } 22:40:00 DEBUG } 22:40:00 DEBUG ] 22:40:00 DEBUG ] 22:40:00 DEBUG ] 22:40:00 DEBUG } 22:40:00 DEBUG } 22:40:00 DEBUG } 22:40:00 DEBUG } 22:40:00 DEBUG ] 22:40:00 DEBUG } 22:40:00 DEBUG } 22:40:00 DEBUG ] 22:40:00 DEBUG } 22:40:00 DEBUG ] 22:40:00 DEBUG ] 22:40:00 DEBUG } 22:40:00 DEBUG } 22:40:00 DEBUG } 22:40:00 DEBUG } 22:40:00 DEBUG ] 22:40:00 DEBUG } 22:40:00 DEBUG } 22:40:00 DEBUG } 22:40:00 DEBUG } 22:40:00 DEBUG } 22:40:00 DEBUG ] 22:40:00 DEBUG } 22:40:00 DEBUG } 22:40:00 DEBUG } 22:40:00 DEBUG } 22:40:00 DEBUG } 22:40:00 DEBUG } 22:40:00 DEBUG } 22:40:00 DEBUG } 22:40:00 DEBUG } 22:40:00 DEBUG } 22:40:00 DEBUG } 22:40:00 DEBUG } 22:40:00 DEBUG } 22:40:00 DEBUG } 22:40:00 DEBUG } 22:40:00 DEBUG } 22:40:00 DEBUG } 22:40:00 DEBUG } 22:40:00 DEBUG } 22:40:00 INFO [loop_until]: OK (rc = 0) 22:40:00 DEBUG } 22:40:00 DEBUG --- stdout --- 22:40:00 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 62m 0% 5099Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7653m 25% 14844Mi 13% gke-xlou-cdm-default-pool-1b0769c8-nhlf 9537m 31% 10844Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 85m 0% 2646Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 4721m 15% 6502Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 8830m 29% 16696Mi 14% gke-xlou-cdm-default-pool-1b0769c8-xnmp 56m 0% 3140Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 9487m 31% 10376Mi 9% gke-xlou-cdm-ds-0f601751-hmm7 4618m 15% 5625Mi 5% gke-xlou-cdm-ds-0f601751-j488 7721m 25% 14183Mi 12% gke-xlou-cdm-ds-0f601751-mc5d 10498m 35% 3242Mi 2% gke-xlou-cdm-ds-0f601751-nj6q 9471m 31% 10433Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 4723m 15% 7352Mi 6% gke-xlou-cdm-frontend-a8771548-i82n 6426m 40% 5953Mi 10% 22:40:00 DEBUG --- stderr --- 22:40:00 DEBUG 22:40:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713303656" 22:40:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303656" 22:40:56 ERROR [http_cmd]: ERROR 22:40:56 INFO [http_cmd]: http status code OK 22:40:56 ERROR --- status code --- 22:40:56 DEBUG --- status code --- 22:40:56 ERROR http status code is 400 (expected [200, 401, 403, 404]) 22:40:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:40:56 ERROR --- http response --- 22:40:56 DEBUG --- http response --- 22:40:56 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 22:40:56 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 22:40:56 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 22:40:58 INFO 22:40:58 INFO [loop_until]: kubectl --namespace=xlou top pods 22:40:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:40:58 INFO [loop_until]: OK (rc = 0) 22:40:58 DEBUG --- stdout --- 22:40:58 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 9Mi am-555f4db595-4lbss 9587m 8608Mi am-555f4db595-fz8sn 9528m 8623Mi am-555f4db595-kzxsp 9396m 8568Mi ds-cts-0 8431m 12196Mi ds-cts-1 7910m 12286Mi ds-cts-2 7834m 12249Mi ds-idrepo-0 4793m 5028Mi ds-idrepo-1 4436m 2875Mi ds-idrepo-2 4488m 3160Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 3m 1207Mi idm-b85f9d978-rl76f 4m 1226Mi lodemon-bc756c647-cr7x6 2m 63Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-7c9b667974-9mr2s 6209m 551Mi 22:40:58 DEBUG --- stderr --- 22:40:58 DEBUG 22:41:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303660" 22:41:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303660" 22:41:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303660" 22:41:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303660" 22:41:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303660" 22:41:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303660" 22:41:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303660" 22:41:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303660" 22:41:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303660" 22:41:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713303660" 22:41:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303660" 22:41:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303660" 22:41:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303660" 22:41:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713303660" 22:41:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303660" 22:41:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303660" 22:41:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303660" 22:41:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303660" 22:41:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303660" 22:41:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713303660" 22:41:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713303660" 22:41:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303660" 22:41:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303660" 22:41:00 INFO 22:41:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303660" 22:41:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303660" 22:41:00 INFO [http_cmd]: http status code OK 22:41:00 INFO [http_cmd]: http status code OK 22:41:00 INFO [http_cmd]: http status code OK 22:41:00 INFO [http_cmd]: http status code OK 22:41:00 INFO [http_cmd]: http status code OK 22:41:00 INFO [http_cmd]: http status code OK 22:41:00 INFO [http_cmd]: http status code OK 22:41:00 INFO [http_cmd]: http status code OK 22:41:00 INFO [http_cmd]: http status code OK 22:41:00 INFO [http_cmd]: http status code OK 22:41:00 INFO [http_cmd]: http status code OK 22:41:00 INFO [http_cmd]: http status code OK 22:41:00 INFO [http_cmd]: http status code OK 22:41:00 INFO [http_cmd]: http status code OK 22:41:00 INFO [http_cmd]: http status code OK 22:41:00 INFO [loop_until]: kubectl --namespace=xlou top node 22:41:00 INFO [http_cmd]: http status code OK 22:41:00 INFO [http_cmd]: http status code OK 22:41:00 DEBUG --- status code --- 22:41:00 INFO [http_cmd]: http status code OK 22:41:00 DEBUG --- status code --- 22:41:00 DEBUG --- status code --- 22:41:00 DEBUG --- status code --- 22:41:00 DEBUG --- status code --- 22:41:00 INFO [http_cmd]: http status code OK 22:41:00 DEBUG --- status code --- 22:41:00 INFO [http_cmd]: http status code OK 22:41:00 DEBUG --- status code --- 22:41:00 INFO [http_cmd]: http status code OK 22:41:00 DEBUG --- status code --- 22:41:00 INFO [http_cmd]: http status code OK 22:41:00 DEBUG --- status code --- 22:41:00 DEBUG --- status code --- 22:41:00 DEBUG --- status code --- 22:41:00 DEBUG --- status code --- 22:41:00 DEBUG --- status code --- 22:41:00 DEBUG --- status code --- 22:41:00 DEBUG --- status code --- 22:41:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:41:00 INFO [http_cmd]: http status code OK 22:41:00 INFO [http_cmd]: http status code OK 22:41:00 DEBUG --- status code --- 22:41:00 DEBUG --- status code --- 22:41:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:41:00 DEBUG --- status code --- 22:41:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:41:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:41:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:41:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:41:00 DEBUG --- status code --- 22:41:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:41:00 DEBUG --- status code --- 22:41:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:41:00 DEBUG --- status code --- 22:41:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:41:00 DEBUG --- status code --- 22:41:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:41:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:41:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:41:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:41:00 DEBUG --- http response --- 22:41:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:41:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:41:00 DEBUG --- status code --- 22:41:00 DEBUG --- status code --- 22:41:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:41:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:41:00 INFO [http_cmd]: http status code OK 22:41:00 DEBUG --- http response --- 22:41:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:41:00 DEBUG --- http response --- 22:41:00 DEBUG --- http response --- 22:41:00 DEBUG --- http response --- 22:41:00 DEBUG --- http response --- 22:41:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:41:00 DEBUG --- http response --- 22:41:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:41:00 DEBUG --- http response --- 22:41:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:41:00 DEBUG --- http response --- 22:41:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:41:00 DEBUG --- http response --- 22:41:00 DEBUG --- http response --- 22:41:00 DEBUG --- http response --- 22:41:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:41:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:41:00 DEBUG --- http response --- 22:41:00 DEBUG --- http response --- 22:41:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:41:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:41:00 DEBUG --- http response --- 22:41:00 DEBUG --- http response --- 22:41:00 DEBUG --- status code --- 22:41:00 DEBUG --- http response --- 22:41:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:41:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:41:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:41:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:41:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:41:00 DEBUG --- http response --- 22:41:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:41:00 DEBUG --- http response --- 22:41:00 DEBUG --- http response --- 22:41:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:41:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:41:00 DEBUG --- http response --- 22:41:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:41:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:41:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:41:00 DEBUG --- http response --- 22:41:00 DEBUG ----- output ----- 22:41:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:41:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:41:00 DEBUG --- http response --- 22:41:00 DEBUG --- http response --- 22:41:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:41:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:41:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:41:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:41:00 DEBUG ----- output ----- 22:41:00 DEBUG ----- output ----- 22:41:00 DEBUG ----- output ----- 22:41:00 DEBUG ----- output ----- 22:41:00 DEBUG ----- output ----- 22:41:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:41:00 DEBUG ----- output ----- 22:41:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:41:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:41:00 DEBUG ----- output ----- 22:41:00 DEBUG ----- output ----- 22:41:00 DEBUG ----- output ----- 22:41:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:41:00 DEBUG ----- output ----- 22:41:00 DEBUG ----- output ----- 22:41:00 DEBUG { 22:41:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:41:00 DEBUG ----- output ----- 22:41:00 DEBUG ----- output ----- 22:41:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:41:00 DEBUG ----- output ----- 22:41:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:41:00 DEBUG --- http response --- 22:41:00 DEBUG ----- output ----- 22:41:00 DEBUG ----- output ----- 22:41:00 DEBUG { 22:41:00 DEBUG { 22:41:00 DEBUG "status": "success", 22:41:00 DEBUG { 22:41:00 DEBUG { 22:41:00 DEBUG "status": "success", 22:41:00 DEBUG "data": { 22:41:00 DEBUG "resultType": "vector", 22:41:00 DEBUG ----- output ----- 22:41:00 DEBUG { 22:41:00 DEBUG "status": "success", 22:41:00 DEBUG "data": { 22:41:00 DEBUG "resultType": "vector", 22:41:00 DEBUG { 22:41:00 DEBUG { 22:41:00 DEBUG "status": "success", 22:41:00 DEBUG ----- output ----- 22:41:00 DEBUG { 22:41:00 DEBUG { 22:41:00 DEBUG ----- output ----- 22:41:00 DEBUG { 22:41:00 DEBUG ----- output ----- 22:41:00 DEBUG { 22:41:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:41:00 DEBUG { 22:41:00 DEBUG "status": "success", 22:41:00 DEBUG "data": { 22:41:00 DEBUG { 22:41:00 DEBUG "status": "success", 22:41:00 DEBUG { 22:41:00 DEBUG "status": "success", 22:41:00 DEBUG ----- output ----- 22:41:00 DEBUG "result": [ 22:41:00 DEBUG { 22:41:00 DEBUG { 22:41:00 DEBUG { 22:41:00 DEBUG "result": [ 22:41:00 DEBUG "status": "success", 22:41:00 DEBUG "status": "success", 22:41:00 DEBUG "data": { 22:41:00 DEBUG { 22:41:00 DEBUG "status": "success", 22:41:00 DEBUG "status": "success", 22:41:00 DEBUG { 22:41:00 DEBUG "status": "success", 22:41:00 DEBUG { 22:41:00 DEBUG "status": "success", 22:41:00 DEBUG ----- output ----- 22:41:00 DEBUG "status": "success", 22:41:00 DEBUG "data": { 22:41:00 DEBUG "resultType": "vector", 22:41:00 DEBUG "status": "success", 22:41:00 DEBUG "data": { 22:41:00 DEBUG "data": { 22:41:00 DEBUG ----- output ----- 22:41:00 DEBUG { 22:41:00 DEBUG { 22:41:00 DEBUG "status": "success", 22:41:00 DEBUG "status": "success", 22:41:00 DEBUG "status": "success", 22:41:00 DEBUG { 22:41:00 DEBUG "data": { 22:41:00 DEBUG "data": { 22:41:00 DEBUG "resultType": "vector", 22:41:00 DEBUG "status": "success", 22:41:00 DEBUG ----- output ----- 22:41:00 DEBUG "data": { 22:41:00 DEBUG "data": { 22:41:00 DEBUG "status": "success", 22:41:00 DEBUG "data": { 22:41:00 DEBUG "status": "success", 22:41:00 DEBUG "data": { 22:41:00 DEBUG { 22:41:00 DEBUG "data": { 22:41:00 DEBUG "resultType": "vector", 22:41:00 DEBUG "result": [ 22:41:00 DEBUG "data": { 22:41:00 DEBUG "resultType": "vector", 22:41:00 DEBUG "resultType": "vector", 22:41:00 DEBUG { 22:41:00 DEBUG "status": "success", 22:41:00 DEBUG "metric": { 22:41:00 DEBUG "data": { 22:41:00 DEBUG "data": { 22:41:00 DEBUG "data": { 22:41:00 DEBUG "metric": { 22:41:00 DEBUG "resultType": "vector", 22:41:00 DEBUG "resultType": "vector", 22:41:00 DEBUG "result": [ 22:41:00 DEBUG "data": { 22:41:00 DEBUG { 22:41:00 DEBUG "resultType": "vector", 22:41:00 DEBUG "resultType": "vector", 22:41:00 DEBUG "data": { 22:41:00 DEBUG "resultType": "vector", 22:41:00 DEBUG "data": { 22:41:00 DEBUG "resultType": "vector", 22:41:00 DEBUG "status": "success", 22:41:00 DEBUG "resultType": "vector", 22:41:00 DEBUG "result": [ 22:41:00 DEBUG { 22:41:00 DEBUG "resultType": "vector", 22:41:00 DEBUG "result": [ 22:41:00 DEBUG "result": [ 22:41:00 DEBUG "status": "success", 22:41:00 DEBUG "data": { 22:41:00 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:41:00 DEBUG "resultType": "vector", 22:41:00 DEBUG "resultType": "vector", 22:41:00 DEBUG "resultType": "vector", 22:41:00 DEBUG "pod": "ds-idrepo-2" 22:41:00 DEBUG "result": [ 22:41:00 DEBUG "result": [ 22:41:00 DEBUG { 22:41:00 DEBUG "resultType": "vector", 22:41:00 DEBUG "status": "success", 22:41:00 DEBUG "result": [ 22:41:00 DEBUG "result": [ 22:41:00 DEBUG "resultType": "vector", 22:41:00 DEBUG "result": [ 22:41:00 DEBUG "resultType": "vector", 22:41:00 DEBUG "result": [ 22:41:00 DEBUG "data": { 22:41:00 DEBUG "result": [ 22:41:00 DEBUG { 22:41:00 DEBUG "metric": { 22:41:00 DEBUG "result": [ 22:41:00 DEBUG { 22:41:00 DEBUG { 22:41:00 DEBUG "data": { 22:41:00 DEBUG "resultType": "vector", 22:41:00 DEBUG }, 22:41:00 DEBUG "result": [ 22:41:00 DEBUG "result": [ 22:41:00 DEBUG "result": [ 22:41:00 DEBUG }, 22:41:00 DEBUG { 22:41:00 DEBUG { 22:41:00 DEBUG "metric": { 22:41:00 DEBUG "result": [ 22:41:00 DEBUG "data": { 22:41:00 DEBUG { 22:41:00 DEBUG { 22:41:00 DEBUG "result": [ 22:41:00 DEBUG { 22:41:00 DEBUG "result": [ 22:41:00 DEBUG { 22:41:00 DEBUG "resultType": "vector", 22:41:00 DEBUG { 22:41:00 DEBUG "metric": { 22:41:00 DEBUG "instance": "10.142.0.107:9100" 22:41:00 DEBUG }, 22:41:00 DEBUG "metric": { 22:41:00 DEBUG "metric": { 22:41:00 DEBUG "resultType": "vector", 22:41:00 DEBUG "result": [ 22:41:00 DEBUG "value": [ 22:41:00 DEBUG { 22:41:00 DEBUG { 22:41:00 DEBUG { 22:41:00 DEBUG "value": [ 22:41:00 DEBUG "metric": { 22:41:00 DEBUG "metric": { 22:41:00 DEBUG "pod": "ds-idrepo-2" 22:41:00 DEBUG { 22:41:00 DEBUG "resultType": "vector", 22:41:00 DEBUG "metric": { 22:41:00 DEBUG "metric": { 22:41:00 DEBUG { 22:41:00 DEBUG "metric": { 22:41:00 DEBUG { 22:41:00 DEBUG "metric": { 22:41:00 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:41:00 DEBUG "metric": { 22:41:00 DEBUG "pod": "ds-idrepo-2" 22:41:00 DEBUG "value": [ 22:41:00 DEBUG { 22:41:00 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:41:00 DEBUG "pod": "am-555f4db595-fz8sn" 22:41:00 DEBUG "result": [ 22:41:00 DEBUG { 22:41:00 DEBUG 22:41:00 DEBUG "metric": { 22:41:00 DEBUG "metric": { 22:41:00 DEBUG "metric": { 22:41:00 DEBUG 22:41:00 DEBUG "instance": "10.142.0.107:9100" 22:41:00 DEBUG "pod": "ds-idrepo-2" 22:41:00 DEBUG }, 22:41:00 DEBUG "metric": { 22:41:00 DEBUG "result": [ 22:41:00 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:41:00 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:41:00 DEBUG "metric": { 22:41:00 DEBUG "instance": "10.142.0.107:9100" 22:41:00 DEBUG "result": [ 22:41:00 DEBUG }, 22:41:00 DEBUG "metric": { 22:41:00 DEBUG "instance": "10.142.0.107:9100" 22:41:00 DEBUG }, 22:41:00 DEBUG 22:41:00 DEBUG "metric": { 22:41:00 DEBUG }, 22:41:00 DEBUG }, 22:41:00 DEBUG { 22:41:00 DEBUG "metric": { 22:41:00 DEBUG 22:41:00 DEBUG "pod": "am-555f4db595-fz8sn" 22:41:00 DEBUG "pod": "am-555f4db595-fz8sn" 22:41:00 DEBUG "pod": "ds-idrepo-2" 22:41:00 DEBUG 22:41:00 DEBUG }, 22:41:00 DEBUG }, 22:41:00 DEBUG "value": [ 22:41:00 DEBUG "pod": "am-555f4db595-fz8sn" 22:41:00 DEBUG { 22:41:00 DEBUG }, 22:41:00 DEBUG }, 22:41:00 DEBUG "pod": "ds-idrepo-2" 22:41:00 DEBUG }, 22:41:00 DEBUG { 22:41:00 DEBUG "value": [ 22:41:00 DEBUG "pod": "ds-idrepo-2" 22:41:00 DEBUG }, 22:41:00 DEBUG "value": [ 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG "value": [ 22:41:00 DEBUG "pod": "am-555f4db595-fz8sn" 22:41:00 DEBUG "metric": { 22:41:00 DEBUG "pod": "am-555f4db595-fz8sn" 22:41:00 DEBUG 22:41:00 DEBUG }, 22:41:00 DEBUG }, 22:41:00 DEBUG }, 22:41:00 DEBUG 22:41:00 DEBUG "value": [ 22:41:00 DEBUG "value": [ 22:41:00 DEBUG 22:41:00 DEBUG }, 22:41:00 DEBUG "metric": { 22:41:00 DEBUG "value": [ 22:41:00 DEBUG "value": [ 22:41:00 DEBUG }, 22:41:00 DEBUG "value": [ 22:41:00 DEBUG "metric": { 22:41:00 DEBUG 22:41:00 DEBUG }, 22:41:00 DEBUG "value": [ 22:41:00 DEBUG 22:41:00 DEBUG [...] 22:41:00 DEBUG "value": [ 22:41:00 DEBUG 22:41:00 DEBUG }, 22:41:00 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:41:00 DEBUG }, 22:41:00 DEBUG [...] 22:41:00 DEBUG "value": [ 22:41:00 DEBUG "value": [ 22:41:00 DEBUG "value": [ 22:41:00 DEBUG [...] 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG "value": [ 22:41:00 DEBUG "pod": "am-555f4db595-fz8sn" 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG "value": [ 22:41:00 DEBUG 22:41:00 DEBUG "instance": "10.142.0.107:9100" 22:41:00 DEBUG 22:41:00 DEBUG "value": [ 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG "value": [ 22:41:00 DEBUG }, 22:41:00 DEBUG "value": [ 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG }, 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG }, 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG "value": [ 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG [...] 22:41:00 DEBUG 22:41:00 DEBUG "value": [ 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG "value": [ 22:41:00 DEBUG [...] 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG [...] 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG [...] 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG [...] 22:41:00 DEBUG [...] 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG [...] 22:41:00 DEBUG [...] 22:41:00 DEBUG 22:41:00 DEBUG [...] 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG [...] 22:41:00 DEBUG 22:41:00 DEBUG "instance": "10.142.0.43:9100" 22:41:00 DEBUG [...] 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:41:00 DEBUG [...] 22:41:00 DEBUG [...] 22:41:00 DEBUG [...] 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG [...] 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG [...] 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG [...] 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG }, 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG [...] 22:41:00 DEBUG 22:41:00 DEBUG [...] 22:41:00 DEBUG }, 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG "pod": "ds-idrepo-0" 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG "value": [ 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG [...] 22:41:00 DEBUG 22:41:00 DEBUG "value": [ 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG }, 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG "pod": "ds-cts-2" 22:41:00 DEBUG 22:41:00 DEBUG [...] 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG [...] 22:41:00 DEBUG "pod": "overseer-0-7c9b667974-9mr2s" 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG "pod": "ds-idrepo-0" 22:41:00 DEBUG 1713303660, 22:41:00 DEBUG 22:41:00 DEBUG "pod": "am-555f4db595-kzxsp" 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG 1713303660, 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG "value": [ 22:41:00 DEBUG "pod": "ds-idrepo-0" 22:41:00 DEBUG "instance": "10.142.0.43:9100" 22:41:00 DEBUG "pod": "ds-idrepo-0" 22:41:00 DEBUG }, 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG "pod": "am-555f4db595-kzxsp" 22:41:00 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:41:00 DEBUG 22:41:00 DEBUG "instance": "10.142.0.43:9100" 22:41:00 DEBUG 22:41:00 DEBUG }, 22:41:00 DEBUG "pod": "ds-idrepo-0" 22:41:00 DEBUG 22:41:00 DEBUG }, 22:41:00 DEBUG "0.0005333333333330377" 22:41:00 DEBUG "pod": "am-555f4db595-kzxsp" 22:41:00 DEBUG }, 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG 22:41:00 DEBUG "6.32250360484082" 22:41:00 DEBUG "pod": "am-555f4db595-kzxsp" 22:41:00 DEBUG "pod": "am-555f4db595-kzxsp" 22:41:00 DEBUG 1713303660, 22:41:00 DEBUG }, 22:41:00 DEBUG }, 22:41:00 DEBUG }, 22:41:00 DEBUG "value": [ 22:41:00 DEBUG "pod": "am-555f4db595-kzxsp" 22:41:00 DEBUG 22:41:00 DEBUG }, 22:41:00 DEBUG }, 22:41:00 DEBUG "pod": "ds-idrepo-0" 22:41:00 DEBUG }, 22:41:00 DEBUG 22:41:00 DEBUG "value": [ 22:41:00 DEBUG }, 22:41:00 DEBUG "instance": "10.142.0.43:9100" 22:41:00 DEBUG "value": [ 22:41:00 DEBUG ] 22:41:00 DEBUG }, 22:41:00 DEBUG "value": [ 22:41:00 DEBUG "pod": "am-555f4db595-kzxsp" 22:41:00 DEBUG 22:41:00 DEBUG "pod": "am-555f4db595-kzxsp" 22:41:00 DEBUG ] 22:41:00 DEBUG }, 22:41:00 DEBUG }, 22:41:00 DEBUG "0" 22:41:00 DEBUG "value": [ 22:41:00 DEBUG "value": [ 22:41:00 DEBUG "value": [ 22:41:00 DEBUG 1713303660, 22:41:00 DEBUG }, 22:41:00 DEBUG 22:41:00 DEBUG "value": [ 22:41:00 DEBUG "value": [ 22:41:00 DEBUG }, 22:41:00 DEBUG "value": [ 22:41:00 DEBUG 22:41:00 DEBUG 1713303660, 22:41:00 DEBUG "value": [ 22:41:00 DEBUG }, 22:41:00 DEBUG 1713303660, 22:41:00 DEBUG } 22:41:00 DEBUG "value": [ 22:41:00 DEBUG 1713303660, 22:41:00 DEBUG }, 22:41:00 DEBUG "pod": "overseer-0-7c9b667974-9mr2s" 22:41:00 DEBUG }, 22:41:00 DEBUG } 22:41:00 DEBUG "value": [ 22:41:00 DEBUG "value": [ 22:41:00 DEBUG ] 22:41:00 DEBUG 1713303660, 22:41:00 DEBUG 1713303660, 22:41:00 DEBUG 1713303660, 22:41:00 DEBUG "0" 22:41:00 DEBUG "value": [ 22:41:00 DEBUG "pod": "am-555f4db595-kzxsp" 22:41:00 DEBUG 1713303660, 22:41:00 DEBUG 1713303660, 22:41:00 DEBUG "value": [ 22:41:00 DEBUG 1713303660, 22:41:00 DEBUG "instance": "10.142.0.43:9100" 22:41:00 DEBUG "0" 22:41:00 DEBUG 1713303660, 22:41:00 DEBUG "value": [ 22:41:00 DEBUG "0" 22:41:00 DEBUG ] 22:41:00 DEBUG 1713303660, 22:41:00 DEBUG "508.81987577639745" 22:41:00 DEBUG "value": [ 22:41:00 DEBUG }, 22:41:00 DEBUG "value": [ 22:41:00 DEBUG ] 22:41:00 DEBUG 1713303660, 22:41:00 DEBUG 1713303660, 22:41:00 DEBUG } 22:41:00 DEBUG "0" 22:41:00 DEBUG "0" 22:41:00 DEBUG "0" 22:41:00 DEBUG ] 22:41:00 DEBUG 1713303660, 22:41:00 DEBUG }, 22:41:00 DEBUG "0" 22:41:00 DEBUG "671264768" 22:41:00 DEBUG 1713303660, 22:41:00 DEBUG "45434144.86666666" 22:41:00 DEBUG }, 22:41:00 DEBUG ] 22:41:00 DEBUG "0" 22:41:00 DEBUG 1713303660, 22:41:00 DEBUG ] 22:41:00 DEBUG } 22:41:00 DEBUG "3498829" 22:41:00 DEBUG ] 22:41:00 DEBUG 1713303660, 22:41:00 DEBUG "value": [ 22:41:00 DEBUG 1713303660, 22:41:00 DEBUG } 22:41:00 DEBUG "1996.2666666666667" 22:41:00 DEBUG "1996.1333333333332" 22:41:00 DEBUG ] 22:41:00 DEBUG ] 22:41:00 DEBUG ] 22:41:00 DEBUG ] 22:41:00 DEBUG } 22:41:00 DEBUG "0" 22:41:00 DEBUG "value": [ 22:41:00 DEBUG ] 22:41:00 DEBUG ] 22:41:00 DEBUG "0" 22:41:00 DEBUG ] 22:41:00 DEBUG "value": [ 22:41:00 DEBUG } 22:41:00 DEBUG ] 22:41:00 DEBUG "19966356480" 22:41:00 DEBUG } 22:41:00 DEBUG } 22:41:00 DEBUG ] 22:41:00 DEBUG } 22:41:00 DEBUG "0" 22:41:00 DEBUG 1713303660, 22:41:00 DEBUG "1996.6" 22:41:00 DEBUG } 22:41:00 DEBUG ] 22:41:00 DEBUG ] 22:41:00 DEBUG } 22:41:00 DEBUG } 22:41:00 DEBUG } 22:41:00 DEBUG } 22:41:00 DEBUG ] 22:41:00 DEBUG ] 22:41:00 DEBUG 1713303660, 22:41:00 DEBUG } 22:41:00 DEBUG } 22:41:00 DEBUG ] 22:41:00 DEBUG } 22:41:00 DEBUG 1713303660, 22:41:00 DEBUG ] 22:41:00 DEBUG } 22:41:00 DEBUG ] 22:41:00 DEBUG ] 22:41:00 DEBUG } 22:41:00 DEBUG ] 22:41:00 DEBUG ] 22:41:00 DEBUG "0" 22:41:00 DEBUG ] 22:41:00 DEBUG } 22:41:00 DEBUG } 22:41:00 DEBUG } 22:41:00 DEBUG ] 22:41:00 DEBUG ] 22:41:00 DEBUG ] 22:41:00 DEBUG } 22:41:00 DEBUG } 22:41:00 DEBUG "0.026" 22:41:00 DEBUG ] 22:41:00 DEBUG ] 22:41:00 DEBUG } 22:41:00 DEBUG ] 22:41:00 DEBUG "45424794.63333333" 22:41:00 DEBUG } 22:41:00 DEBUG ] 22:41:00 DEBUG } 22:41:00 DEBUG } 22:41:00 DEBUG ] 22:41:00 DEBUG } 22:41:00 DEBUG } 22:41:00 DEBUG ] 22:41:00 DEBUG } 22:41:00 DEBUG ] 22:41:00 DEBUG ] 22:41:00 DEBUG } 22:41:00 DEBUG } 22:41:00 DEBUG } 22:41:00 DEBUG } 22:41:00 DEBUG ] 22:41:00 DEBUG ] 22:41:00 DEBUG } 22:41:00 DEBUG } 22:41:00 DEBUG ] 22:41:00 DEBUG } 22:41:00 DEBUG ] 22:41:00 DEBUG } 22:41:00 DEBUG } 22:41:00 DEBUG ] 22:41:00 DEBUG } 22:41:00 DEBUG } 22:41:00 DEBUG } 22:41:00 DEBUG ] 22:41:00 DEBUG } 22:41:00 DEBUG ] 22:41:00 DEBUG } 22:41:00 DEBUG } 22:41:00 DEBUG } 22:41:00 DEBUG } 22:41:00 DEBUG } 22:41:00 DEBUG } 22:41:00 DEBUG } 22:41:00 DEBUG } 22:41:00 DEBUG } 22:41:00 DEBUG } 22:41:00 DEBUG } 22:41:00 DEBUG } 22:41:00 DEBUG } 22:41:00 DEBUG } 22:41:00 DEBUG } 22:41:00 DEBUG } 22:41:00 DEBUG ] 22:41:00 DEBUG } 22:41:00 DEBUG } 22:41:00 DEBUG } 22:41:00 DEBUG } 22:41:00 DEBUG ] 22:41:00 DEBUG } 22:41:00 DEBUG ] 22:41:00 DEBUG } 22:41:00 INFO [loop_until]: OK (rc = 0) 22:41:00 DEBUG } 22:41:00 DEBUG } 22:41:00 DEBUG } 22:41:00 DEBUG } 22:41:00 DEBUG } 22:41:00 DEBUG --- stdout --- 22:41:00 DEBUG } 22:41:00 DEBUG } 22:41:00 DEBUG } 22:41:00 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 62m 0% 5109Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7699m 25% 14848Mi 13% gke-xlou-cdm-default-pool-1b0769c8-nhlf 9419m 31% 10840Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 80m 0% 2651Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 4722m 15% 6501Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 8950m 29% 16724Mi 14% gke-xlou-cdm-default-pool-1b0769c8-xnmp 60m 0% 3145Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 9516m 31% 10381Mi 9% gke-xlou-cdm-ds-0f601751-hmm7 4613m 15% 5614Mi 5% gke-xlou-cdm-ds-0f601751-j488 7816m 26% 14194Mi 12% gke-xlou-cdm-ds-0f601751-mc5d 10550m 35% 3242Mi 2% gke-xlou-cdm-ds-0f601751-nj6q 9747m 32% 10430Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 4748m 15% 7352Mi 6% gke-xlou-cdm-frontend-a8771548-i82n 6439m 40% 5949Mi 10% 22:41:00 DEBUG --- stderr --- 22:41:00 DEBUG 22:41:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303716" 22:41:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713303716" 22:41:56 ERROR [http_cmd]: ERROR 22:41:56 INFO [http_cmd]: http status code OK 22:41:56 ERROR --- status code --- 22:41:56 DEBUG --- status code --- 22:41:56 ERROR http status code is 400 (expected [200, 401, 403, 404]) 22:41:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:41:56 ERROR --- http response --- 22:41:56 DEBUG --- http response --- 22:41:56 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 22:41:56 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 22:41:56 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 22:41:58 INFO 22:41:58 INFO [loop_until]: kubectl --namespace=xlou top pods 22:41:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:41:58 INFO [loop_until]: OK (rc = 0) 22:41:58 DEBUG --- stdout --- 22:41:58 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 9Mi am-555f4db595-4lbss 9961m 8641Mi am-555f4db595-fz8sn 10890m 8663Mi am-555f4db595-kzxsp 10289m 8626Mi ds-cts-0 14249m 12392Mi ds-cts-1 12003m 12513Mi ds-cts-2 11542m 12501Mi ds-idrepo-0 3969m 5028Mi ds-idrepo-1 4612m 2875Mi ds-idrepo-2 3946m 3160Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 4m 1208Mi idm-b85f9d978-rl76f 3m 1226Mi lodemon-bc756c647-cr7x6 2m 63Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-7c9b667974-9mr2s 5236m 64Mi 22:41:58 DEBUG --- stderr --- 22:41:58 DEBUG 22:42:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713303720" 22:42:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303720" 22:42:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303720" 22:42:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303720" 22:42:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303720" 22:42:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303720" 22:42:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303720" 22:42:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303720" 22:42:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303720" 22:42:00 INFO 22:42:00 INFO [loop_until]: kubectl --namespace=xlou top node 22:42:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713303720" 22:42:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:42:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303720" 22:42:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303720" 22:42:00 INFO [http_cmd]: http status code OK 22:42:00 INFO [http_cmd]: http status code OK 22:42:00 INFO [http_cmd]: http status code OK 22:42:00 INFO [http_cmd]: http status code OK 22:42:00 INFO [http_cmd]: http status code OK 22:42:00 DEBUG --- status code --- 22:42:00 INFO [http_cmd]: http status code OK 22:42:00 DEBUG --- status code --- 22:42:00 DEBUG --- status code --- 22:42:00 DEBUG --- status code --- 22:42:00 DEBUG --- status code --- 22:42:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:42:00 DEBUG --- status code --- 22:42:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:42:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:42:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:42:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:42:00 DEBUG --- http response --- 22:42:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:42:00 DEBUG --- http response --- 22:42:00 DEBUG --- http response --- 22:42:00 DEBUG --- http response --- 22:42:00 DEBUG --- http response --- 22:42:00 DEBUG --- http response --- 22:42:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:42:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:42:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:42:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:42:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:42:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:42:00 DEBUG ----- output ----- 22:42:00 DEBUG ----- output ----- 22:42:00 DEBUG ----- output ----- 22:42:00 DEBUG ----- output ----- 22:42:00 DEBUG ----- output ----- 22:42:00 DEBUG ----- output ----- 22:42:00 DEBUG { 22:42:00 DEBUG "status": "success", 22:42:00 DEBUG { 22:42:00 DEBUG { 22:42:00 DEBUG { 22:42:00 DEBUG { 22:42:00 DEBUG { 22:42:00 DEBUG "data": { 22:42:00 DEBUG "status": "success", 22:42:00 DEBUG "status": "success", 22:42:00 DEBUG "status": "success", 22:42:00 DEBUG "status": "success", 22:42:00 DEBUG "data": { 22:42:00 DEBUG "resultType": "vector", 22:42:00 DEBUG "data": { 22:42:00 DEBUG "data": { 22:42:00 DEBUG "data": { 22:42:00 DEBUG "status": "success", 22:42:00 DEBUG "resultType": "vector", 22:42:00 DEBUG "result": [ 22:42:00 DEBUG "resultType": "vector", 22:42:00 DEBUG "resultType": "vector", 22:42:00 DEBUG "resultType": "vector", 22:42:00 DEBUG "data": { 22:42:00 DEBUG "result": [ 22:42:00 DEBUG { 22:42:00 DEBUG "result": [ 22:42:00 DEBUG { 22:42:00 DEBUG "result": [ 22:42:00 DEBUG "resultType": "vector", 22:42:00 DEBUG { 22:42:00 DEBUG "metric": { 22:42:00 DEBUG "result": [ 22:42:00 DEBUG "metric": { 22:42:00 DEBUG { 22:42:00 DEBUG "result": [ 22:42:00 DEBUG "metric": { 22:42:00 INFO [http_cmd]: http status code OK 22:42:00 DEBUG "pod": "ds-idrepo-2" 22:42:00 INFO [http_cmd]: http status code OK 22:42:00 INFO [http_cmd]: http status code OK 22:42:00 INFO [http_cmd]: http status code OK 22:42:00 DEBUG { 22:42:00 DEBUG "pod": "ds-idrepo-2" 22:42:00 DEBUG "metric": { 22:42:00 DEBUG { 22:42:00 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:42:00 DEBUG --- status code --- 22:42:00 DEBUG }, 22:42:00 DEBUG --- status code --- 22:42:00 DEBUG --- status code --- 22:42:00 DEBUG --- status code --- 22:42:00 DEBUG "metric": { 22:42:00 DEBUG }, 22:42:00 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:42:00 DEBUG "metric": { 22:42:00 DEBUG }, 22:42:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:42:00 DEBUG "value": [ 22:42:00 INFO [http_cmd]: http status code OK 22:42:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:42:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:42:00 INFO [http_cmd]: http status code OK 22:42:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:42:00 DEBUG "pod": "ds-idrepo-2" 22:42:00 DEBUG "value": [ 22:42:00 DEBUG }, 22:42:00 DEBUG "pod": "am-555f4db595-fz8sn" 22:42:00 DEBUG "value": [ 22:42:00 DEBUG --- http response --- 22:42:00 DEBUG 22:42:00 DEBUG --- status code --- 22:42:00 DEBUG --- http response --- 22:42:00 DEBUG --- http response --- 22:42:00 DEBUG --- status code --- 22:42:00 DEBUG --- http response --- 22:42:00 DEBUG }, 22:42:00 DEBUG 22:42:00 DEBUG "value": [ 22:42:00 DEBUG }, 22:42:00 DEBUG 22:42:00 DEBUG 22:42:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:42:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:42:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:42:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:42:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:42:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:42:00 DEBUG "value": [ 22:42:00 DEBUG 22:42:00 DEBUG 22:42:00 DEBUG "value": [ 22:42:00 DEBUG 22:42:00 DEBUG 22:42:00 DEBUG ----- output ----- 22:42:00 DEBUG --- http response --- 22:42:00 DEBUG ----- output ----- 22:42:00 DEBUG --- http response --- 22:42:00 DEBUG ----- output ----- 22:42:00 DEBUG ----- output ----- 22:42:00 DEBUG 22:42:00 DEBUG 22:42:00 DEBUG 22:42:00 DEBUG 22:42:00 DEBUG 22:42:00 DEBUG [...] 22:42:00 DEBUG { 22:42:00 DEBUG { 22:42:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:42:00 DEBUG { 22:42:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:42:00 DEBUG { 22:42:00 DEBUG 22:42:00 DEBUG [...] 22:42:00 DEBUG 22:42:00 DEBUG 22:42:00 DEBUG [...] 22:42:00 DEBUG 22:42:00 DEBUG "status": "success", 22:42:00 DEBUG "status": "success", 22:42:00 DEBUG ----- output ----- 22:42:00 DEBUG "status": "success", 22:42:00 DEBUG ----- output ----- 22:42:00 DEBUG "status": "success", 22:42:00 DEBUG 22:42:00 DEBUG 22:42:00 DEBUG [...] 22:42:00 DEBUG 22:42:00 DEBUG 22:42:00 DEBUG 22:42:00 DEBUG "data": { 22:42:00 DEBUG "data": { 22:42:00 DEBUG { 22:42:00 DEBUG "data": { 22:42:00 DEBUG { 22:42:00 DEBUG "data": { 22:42:00 DEBUG [...] 22:42:00 DEBUG 22:42:00 DEBUG 22:42:00 DEBUG [...] 22:42:00 DEBUG 22:42:00 DEBUG 22:42:00 DEBUG "resultType": "vector", 22:42:00 DEBUG "resultType": "vector", 22:42:00 DEBUG "status": "success", 22:42:00 DEBUG "resultType": "vector", 22:42:00 DEBUG "status": "success", 22:42:00 DEBUG "resultType": "vector", 22:42:00 DEBUG 22:42:00 DEBUG 22:42:00 DEBUG 22:42:00 DEBUG 22:42:00 DEBUG 22:42:00 DEBUG "pod": "ds-idrepo-0" 22:42:00 DEBUG "result": [ 22:42:00 DEBUG "result": [ 22:42:00 DEBUG "data": { 22:42:00 DEBUG "result": [ 22:42:00 DEBUG "data": { 22:42:00 DEBUG "result": [ 22:42:00 DEBUG 22:42:00 DEBUG "pod": "ds-cts-2" 22:42:00 DEBUG 22:42:00 DEBUG 22:42:00 DEBUG "pod": "ds-cts-0" 22:42:00 DEBUG }, 22:42:00 DEBUG { 22:42:00 DEBUG { 22:42:00 DEBUG "resultType": "vector", 22:42:00 DEBUG { 22:42:00 DEBUG "resultType": "vector", 22:42:00 DEBUG { 22:42:00 DEBUG 22:42:00 DEBUG }, 22:42:00 DEBUG "pod": "ds-cts-0" 22:42:00 DEBUG 22:42:00 DEBUG }, 22:42:00 DEBUG "value": [ 22:42:00 DEBUG "metric": { 22:42:00 DEBUG "metric": { 22:42:00 DEBUG "result": [ 22:42:00 DEBUG "metric": { 22:42:00 DEBUG "result": [ 22:42:00 DEBUG "metric": { 22:42:00 DEBUG "pod": "ds-idrepo-0" 22:42:00 DEBUG "value": [ 22:42:00 DEBUG }, 22:42:00 DEBUG "pod": "am-555f4db595-kzxsp" 22:42:00 DEBUG "value": [ 22:42:00 DEBUG 1713303720, 22:42:00 DEBUG "pod": "ds-idrepo-2" 22:42:00 DEBUG "instance": "10.142.0.107:9100" 22:42:00 DEBUG { 22:42:00 DEBUG "pod": "am-555f4db595-fz8sn" 22:42:00 DEBUG { 22:42:00 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:42:00 DEBUG }, 22:42:00 DEBUG 1713303720, 22:42:00 DEBUG "value": [ 22:42:00 DEBUG }, 22:42:00 DEBUG 1713303720, 22:42:00 DEBUG "0" 22:42:00 DEBUG }, 22:42:00 DEBUG }, 22:42:00 DEBUG "metric": { 22:42:00 DEBUG }, 22:42:00 DEBUG "metric": { 22:42:00 DEBUG }, 22:42:00 DEBUG "value": [ 22:42:00 DEBUG "0" 22:42:00 DEBUG 1713303720, 22:42:00 DEBUG "value": [ 22:42:00 DEBUG "893166.544556296" 22:42:00 DEBUG ] 22:42:00 DEBUG "value": [ 22:42:00 DEBUG "value": [ 22:42:00 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:42:00 DEBUG "value": [ 22:42:00 DEBUG "pod": "ds-idrepo-2" 22:42:00 DEBUG "value": [ 22:42:00 DEBUG 1713303720, 22:42:00 DEBUG ] 22:42:00 DEBUG "24717038.705874566" 22:42:00 DEBUG 1713303720, 22:42:00 DEBUG ] 22:42:00 DEBUG } 22:42:00 DEBUG 22:42:00 DEBUG 22:42:00 DEBUG }, 22:42:00 DEBUG 22:42:00 DEBUG }, 22:42:00 DEBUG 22:42:00 DEBUG "0" 22:42:00 DEBUG } 22:42:00 DEBUG ] 22:42:00 DEBUG "1317.4" 22:42:00 DEBUG } 22:42:00 DEBUG ] 22:42:00 DEBUG 22:42:00 DEBUG 22:42:00 DEBUG "value": [ 22:42:00 DEBUG 22:42:00 DEBUG "value": [ 22:42:00 DEBUG 22:42:00 DEBUG ] 22:42:00 DEBUG ] 22:42:00 DEBUG } 22:42:00 DEBUG ] 22:42:00 DEBUG ] 22:42:00 DEBUG } 22:42:00 DEBUG 22:42:00 DEBUG 22:42:00 DEBUG 22:42:00 DEBUG 22:42:00 DEBUG 22:42:00 DEBUG 22:42:00 DEBUG } 22:42:00 DEBUG } 22:42:00 DEBUG ] 22:42:00 DEBUG } 22:42:00 DEBUG } 22:42:00 DEBUG } 22:42:00 DEBUG [...] 22:42:00 DEBUG 22:42:00 DEBUG 22:42:00 DEBUG [...] 22:42:00 DEBUG 22:42:00 DEBUG [...] 22:42:00 DEBUG ] 22:42:00 DEBUG } 22:42:00 DEBUG } 22:42:00 DEBUG ] 22:42:00 DEBUG } 22:42:00 DEBUG [...] 22:42:00 DEBUG 22:42:00 DEBUG 22:42:00 DEBUG 22:42:00 DEBUG 22:42:00 DEBUG 22:42:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303720" 22:42:00 DEBUG } 22:42:00 DEBUG } 22:42:00 DEBUG } 22:42:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713303720" 22:42:00 DEBUG 22:42:00 DEBUG 22:42:00 DEBUG [...] 22:42:00 DEBUG 22:42:00 DEBUG [...] 22:42:00 DEBUG 22:42:00 DEBUG } 22:42:00 DEBUG } 22:42:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303720" 22:42:00 DEBUG 22:42:00 DEBUG "pod": "ds-idrepo-0" 22:42:00 DEBUG 22:42:00 DEBUG 22:42:00 DEBUG 22:42:00 DEBUG 22:42:00 DEBUG 22:42:00 DEBUG }, 22:42:00 DEBUG 22:42:00 DEBUG "pod": "am-555f4db595-kzxsp" 22:42:00 DEBUG 22:42:00 DEBUG "pod": "ds-cts-0" 22:42:00 DEBUG "instance": "10.142.0.43:9100" 22:42:00 DEBUG "value": [ 22:42:00 DEBUG 22:42:00 DEBUG }, 22:42:00 DEBUG 22:42:00 DEBUG }, 22:42:00 DEBUG }, 22:42:00 DEBUG 1713303720, 22:42:00 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:42:00 DEBUG "value": [ 22:42:00 DEBUG "pod": "ds-idrepo-0" 22:42:00 DEBUG "value": [ 22:42:00 DEBUG "value": [ 22:42:00 DEBUG "0" 22:42:00 DEBUG }, 22:42:00 DEBUG 1713303720, 22:42:00 DEBUG }, 22:42:00 DEBUG 1713303720, 22:42:00 DEBUG 1713303720, 22:42:00 DEBUG ] 22:42:00 DEBUG "value": [ 22:42:00 DEBUG "952.5149333333333" 22:42:00 DEBUG "value": [ 22:42:00 DEBUG "882308.8117207808" 22:42:00 DEBUG "19967198208" 22:42:00 DEBUG } 22:42:00 DEBUG 1713303720, 22:42:00 DEBUG ] 22:42:00 DEBUG 1713303720, 22:42:00 DEBUG ] 22:42:00 DEBUG ] 22:42:00 DEBUG ] 22:42:00 DEBUG "674906112" 22:42:00 DEBUG } 22:42:00 DEBUG "0" 22:42:00 DEBUG } 22:42:00 DEBUG } 22:42:00 DEBUG } 22:42:00 DEBUG ] 22:42:00 DEBUG ] 22:42:00 DEBUG ] 22:42:00 DEBUG ] 22:42:00 DEBUG ] 22:42:00 DEBUG } 22:42:00 DEBUG } 22:42:00 DEBUG } 22:42:00 DEBUG } 22:42:00 DEBUG } 22:42:00 DEBUG } 22:42:00 DEBUG ] 22:42:00 DEBUG } 22:42:00 DEBUG ] 22:42:00 DEBUG } 22:42:00 DEBUG } 22:42:00 DEBUG } 22:42:00 DEBUG } 22:42:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303720" 22:42:00 DEBUG } 22:42:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303720" 22:42:00 DEBUG } 22:42:00 INFO [http_cmd]: http status code OK 22:42:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303720" 22:42:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303720" 22:42:00 DEBUG --- status code --- 22:42:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303720" 22:42:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:42:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303720" 22:42:00 INFO [http_cmd]: http status code OK 22:42:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713303720" 22:42:00 INFO [http_cmd]: http status code OK 22:42:00 DEBUG --- http response --- 22:42:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303720" 22:42:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303720" 22:42:00 DEBUG --- status code --- 22:42:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303720" 22:42:00 DEBUG --- status code --- 22:42:00 INFO [http_cmd]: http status code OK 22:42:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:42:00 INFO [http_cmd]: http status code OK 22:42:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:42:00 INFO [http_cmd]: http status code OK 22:42:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:42:00 INFO [http_cmd]: http status code OK 22:42:00 DEBUG --- status code --- 22:42:00 DEBUG ----- output ----- 22:42:00 DEBUG { 22:42:00 DEBUG --- http response --- 22:42:00 DEBUG --- status code --- 22:42:00 DEBUG --- http response --- 22:42:00 DEBUG --- status code --- 22:42:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:42:00 INFO [http_cmd]: http status code OK 22:42:00 INFO [http_cmd]: http status code OK 22:42:00 DEBUG --- status code --- 22:42:00 INFO [http_cmd]: http status code OK 22:42:00 DEBUG "status": "success", 22:42:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:42:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:42:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:42:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:42:00 DEBUG --- http response --- 22:42:00 DEBUG --- status code --- 22:42:00 DEBUG --- status code --- 22:42:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:42:00 DEBUG --- status code --- 22:42:00 DEBUG "data": { 22:42:00 DEBUG ----- output ----- 22:42:00 DEBUG --- http response --- 22:42:00 DEBUG --- http response --- 22:42:00 DEBUG ----- output ----- 22:42:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:42:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:42:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:42:00 DEBUG --- http response --- 22:42:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:42:00 INFO [http_cmd]: http status code OK 22:42:00 DEBUG "resultType": "vector", 22:42:00 DEBUG { 22:42:00 INFO [loop_until]: OK (rc = 0) 22:42:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:42:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:42:00 DEBUG { 22:42:00 DEBUG --- http response --- 22:42:00 INFO [http_cmd]: http status code OK 22:42:00 DEBUG ----- output ----- 22:42:00 DEBUG --- http response --- 22:42:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:42:00 DEBUG --- http response --- 22:42:00 DEBUG --- status code --- 22:42:00 DEBUG "result": [ 22:42:00 DEBUG "status": "success", 22:42:00 DEBUG --- stdout --- 22:42:00 DEBUG ----- output ----- 22:42:00 DEBUG ----- output ----- 22:42:00 DEBUG "status": "success", 22:42:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:42:00 DEBUG --- status code --- 22:42:00 DEBUG { 22:42:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:42:00 DEBUG ----- output ----- 22:42:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:42:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:42:00 DEBUG { 22:42:00 DEBUG "data": { 22:42:00 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 5103Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 12133m 40% 15037Mi 13% gke-xlou-cdm-default-pool-1b0769c8-nhlf 8192m 27% 10897Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 83m 0% 2647Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 4681m 15% 6508Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 14257m 47% 16859Mi 15% gke-xlou-cdm-default-pool-1b0769c8-xnmp 63m 0% 3159Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 10190m 34% 10423Mi 9% gke-xlou-cdm-ds-0f601751-hmm7 3708m 12% 5622Mi 5% gke-xlou-cdm-ds-0f601751-j488 12029m 40% 14376Mi 12% gke-xlou-cdm-ds-0f601751-mc5d 8723m 29% 3254Mi 2% gke-xlou-cdm-ds-0f601751-nj6q 9381m 31% 10504Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 4586m 15% 7353Mi 6% gke-xlou-cdm-frontend-a8771548-i82n 6070m 38% 5466Mi 9% 22:42:00 DEBUG { 22:42:00 DEBUG { 22:42:00 DEBUG "data": { 22:42:00 DEBUG ----- output ----- 22:42:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:42:00 DEBUG "status": "success", 22:42:00 DEBUG ----- output ----- 22:42:00 DEBUG { 22:42:00 DEBUG ----- output ----- 22:42:00 DEBUG --- http response --- 22:42:00 DEBUG "metric": { 22:42:00 DEBUG "resultType": "vector", 22:42:00 DEBUG --- stderr --- 22:42:00 DEBUG "status": "success", 22:42:00 DEBUG "status": "success", 22:42:00 INFO [http_cmd]: http status code OK 22:42:00 DEBUG "resultType": "vector", 22:42:00 DEBUG { 22:42:00 DEBUG --- http response --- 22:42:00 DEBUG "data": { 22:42:00 DEBUG { 22:42:00 DEBUG "status": "success", 22:42:00 DEBUG { 22:42:00 DEBUG "instance": "10.142.0.107:9100" 22:42:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:42:00 DEBUG "result": [ 22:42:00 DEBUG 22:42:00 DEBUG "data": { 22:42:00 DEBUG "data": { 22:42:00 DEBUG --- status code --- 22:42:00 DEBUG "result": [ 22:42:00 DEBUG "status": "success", 22:42:00 DEBUG "resultType": "vector", 22:42:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:42:00 DEBUG "status": "success", 22:42:00 DEBUG "data": { 22:42:00 DEBUG "status": "success", 22:42:00 DEBUG }, 22:42:00 DEBUG ----- output ----- 22:42:00 DEBUG { 22:42:00 DEBUG "resultType": "vector", 22:42:00 DEBUG "resultType": "vector", 22:42:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:42:00 DEBUG { 22:42:00 DEBUG "data": { 22:42:00 DEBUG "result": [ 22:42:00 DEBUG ----- output ----- 22:42:00 DEBUG "data": { 22:42:00 DEBUG "resultType": "vector", 22:42:00 DEBUG "data": { 22:42:00 DEBUG "value": [ 22:42:00 DEBUG { 22:42:00 DEBUG "metric": { 22:42:00 DEBUG "result": [ 22:42:00 DEBUG "result": [ 22:42:00 DEBUG --- http response --- 22:42:00 DEBUG "metric": { 22:42:00 DEBUG "resultType": "vector", 22:42:00 DEBUG { 22:42:00 DEBUG { 22:42:00 DEBUG "resultType": "vector", 22:42:00 DEBUG "result": [ 22:42:00 DEBUG "resultType": "vector", 22:42:00 DEBUG 22:42:00 DEBUG "status": "success", 22:42:00 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:42:00 DEBUG { 22:42:00 DEBUG { 22:42:00 DEBUG "pod": "ds-idrepo-2" 22:42:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:42:00 DEBUG "result": [ 22:42:00 DEBUG "metric": { 22:42:00 DEBUG "status": "success", 22:42:00 DEBUG "result": [ 22:42:00 DEBUG { 22:42:00 DEBUG "result": [ 22:42:00 DEBUG 22:42:00 DEBUG "data": { 22:42:00 DEBUG }, 22:42:00 DEBUG "metric": { 22:42:00 DEBUG "metric": { 22:42:00 DEBUG }, 22:42:00 DEBUG ----- output ----- 22:42:00 DEBUG { 22:42:00 DEBUG "pod": "ds-idrepo-2" 22:42:00 DEBUG "data": { 22:42:00 DEBUG { 22:42:00 DEBUG "metric": { 22:42:00 DEBUG { 22:42:00 DEBUG 22:42:00 DEBUG "resultType": "vector", 22:42:00 DEBUG "value": [ 22:42:00 DEBUG "instance": "10.142.0.107:9100" 22:42:00 DEBUG "pod": "am-555f4db595-fz8sn" 22:42:00 DEBUG "value": [ 22:42:00 DEBUG { 22:42:00 DEBUG "metric": { 22:42:00 DEBUG }, 22:42:00 DEBUG "resultType": "vector", 22:42:00 DEBUG "metric": { 22:42:00 DEBUG "pod": "am-555f4db595-fz8sn" 22:42:00 DEBUG "metric": { 22:42:00 DEBUG [...] 22:42:00 DEBUG "result": [ 22:42:00 DEBUG 22:42:00 DEBUG }, 22:42:00 DEBUG }, 22:42:00 DEBUG 22:42:00 DEBUG "status": "success", 22:42:00 DEBUG "pod": "am-555f4db595-fz8sn" 22:42:00 DEBUG "value": [ 22:42:00 DEBUG "result": [ 22:42:00 DEBUG "instance": "10.142.0.107:9100" 22:42:00 DEBUG }, 22:42:00 DEBUG "pod": "am-555f4db595-fz8sn" 22:42:00 DEBUG 22:42:00 DEBUG { 22:42:00 DEBUG 22:42:00 DEBUG "value": [ 22:42:00 DEBUG "value": [ 22:42:00 DEBUG 22:42:00 DEBUG "data": { 22:42:00 DEBUG }, 22:42:00 DEBUG 22:42:00 DEBUG { 22:42:00 DEBUG }, 22:42:00 DEBUG "value": [ 22:42:00 DEBUG }, 22:42:00 DEBUG 22:42:00 DEBUG "metric": { 22:42:00 DEBUG 22:42:00 DEBUG 22:42:00 DEBUG 22:42:00 DEBUG 22:42:00 DEBUG "resultType": "vector", 22:42:00 DEBUG "value": [ 22:42:00 DEBUG 22:42:00 DEBUG "metric": { 22:42:00 DEBUG "value": [ 22:42:00 DEBUG 22:42:00 DEBUG "value": [ 22:42:00 DEBUG 22:42:00 DEBUG "instance": "10.142.0.107:9100" 22:42:00 DEBUG [...] 22:42:00 DEBUG 22:42:00 DEBUG 22:42:00 DEBUG [...] 22:42:00 DEBUG "result": [ 22:42:00 DEBUG 22:42:00 DEBUG 22:42:00 DEBUG "pod": "am-555f4db595-fz8sn" 22:42:00 DEBUG 22:42:00 DEBUG 22:42:00 DEBUG 22:42:00 DEBUG "instance": "10.142.0.43:9100" 22:42:00 DEBUG }, 22:42:00 DEBUG 22:42:00 DEBUG 22:42:00 DEBUG 22:42:00 DEBUG 22:42:00 DEBUG { 22:42:00 DEBUG 22:42:00 DEBUG [...] 22:42:00 DEBUG }, 22:42:00 DEBUG 22:42:00 DEBUG 22:42:00 DEBUG 22:42:00 DEBUG }, 22:42:00 DEBUG "value": [ 22:42:00 DEBUG 22:42:00 DEBUG [...] 22:42:00 DEBUG [...] 22:42:00 DEBUG 22:42:00 DEBUG "metric": { 22:42:00 DEBUG 22:42:00 DEBUG 22:42:00 DEBUG "value": [ 22:42:00 DEBUG 22:42:00 DEBUG [...] 22:42:00 DEBUG 22:42:00 DEBUG "value": [ 22:42:00 DEBUG 22:42:00 DEBUG 22:42:00 DEBUG 22:42:00 DEBUG 22:42:00 DEBUG 22:42:00 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:42:00 DEBUG [...] 22:42:00 DEBUG 22:42:00 DEBUG 22:42:00 DEBUG [...] 22:42:00 DEBUG 22:42:00 DEBUG [...] 22:42:00 DEBUG 1713303720, 22:42:00 DEBUG 22:42:00 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:42:00 DEBUG 22:42:00 DEBUG 22:42:00 DEBUG "pod": "ds-idrepo-0" 22:42:00 DEBUG }, 22:42:00 DEBUG 22:42:00 DEBUG 22:42:00 DEBUG 22:42:00 DEBUG 22:42:00 DEBUG 22:42:00 DEBUG 22:42:00 DEBUG "0.0008666666666670153" 22:42:00 DEBUG 22:42:00 DEBUG }, 22:42:00 DEBUG 22:42:00 DEBUG 22:42:00 DEBUG }, 22:42:00 DEBUG "value": [ 22:42:00 DEBUG 22:42:00 DEBUG "pod": "ds-idrepo-0" 22:42:00 DEBUG 22:42:00 DEBUG 22:42:00 DEBUG 22:42:00 DEBUG 22:42:00 DEBUG ] 22:42:00 DEBUG [...] 22:42:00 DEBUG "value": [ 22:42:00 DEBUG "instance": "10.142.0.43:9100" 22:42:00 DEBUG "pod": "am-555f4db595-kzxsp" 22:42:00 DEBUG "value": [ 22:42:00 DEBUG 22:42:00 DEBUG 22:42:00 DEBUG }, 22:42:00 DEBUG [...] 22:42:00 DEBUG 22:42:00 DEBUG "pod": "am-555f4db595-kzxsp" 22:42:00 DEBUG 22:42:00 DEBUG } 22:42:00 DEBUG 22:42:00 DEBUG 1713303720, 22:42:00 DEBUG }, 22:42:00 DEBUG }, 22:42:00 DEBUG 1713303720, 22:42:00 DEBUG 22:42:00 DEBUG "pod": "am-555f4db595-kzxsp" 22:42:00 DEBUG "value": [ 22:42:00 DEBUG 22:42:00 DEBUG "instance": "10.142.0.43:9100" 22:42:00 DEBUG }, 22:42:00 DEBUG "pod": "am-555f4db595-kzxsp" 22:42:00 DEBUG ] 22:42:00 DEBUG 22:42:00 DEBUG "6.297271362783573" 22:42:00 DEBUG "value": [ 22:42:00 DEBUG "value": [ 22:42:00 DEBUG "0" 22:42:00 DEBUG 22:42:00 DEBUG }, 22:42:00 DEBUG 1713303720, 22:42:00 DEBUG 22:42:00 DEBUG }, 22:42:00 DEBUG "value": [ 22:42:00 DEBUG }, 22:42:00 DEBUG } 22:42:00 DEBUG 22:42:00 DEBUG ] 22:42:00 DEBUG 1713303720, 22:42:00 DEBUG 1713303720, 22:42:00 DEBUG ] 22:42:00 DEBUG [...] 22:42:00 DEBUG "value": [ 22:42:00 DEBUG "0" 22:42:00 DEBUG 22:42:00 DEBUG "value": [ 22:42:00 DEBUG 1713303720, 22:42:00 DEBUG "value": [ 22:42:00 DEBUG } 22:42:00 DEBUG "instance": "10.142.0.43:9100" 22:42:00 DEBUG } 22:42:00 DEBUG "0.00033333333333333365" 22:42:00 DEBUG "3581439" 22:42:00 DEBUG } 22:42:00 DEBUG 22:42:00 DEBUG 1713303720, 22:42:00 DEBUG ] 22:42:00 DEBUG "pod": "am-555f4db595-kzxsp" 22:42:00 DEBUG 1713303720, 22:42:00 DEBUG "1317.4" 22:42:00 DEBUG 1713303720, 22:42:00 DEBUG }, 22:42:00 DEBUG ] 22:42:00 DEBUG ] 22:42:00 DEBUG ] 22:42:00 DEBUG ] 22:42:00 DEBUG 22:42:00 DEBUG "0" 22:42:00 DEBUG } 22:42:00 DEBUG }, 22:42:00 DEBUG "45483892.83333333" 22:42:00 DEBUG ] 22:42:00 DEBUG "0.026" 22:42:00 DEBUG "value": [ 22:42:00 DEBUG } 22:42:00 DEBUG } 22:42:00 DEBUG } 22:42:00 DEBUG } 22:42:00 DEBUG 22:42:00 DEBUG ] 22:42:00 DEBUG ] 22:42:00 DEBUG "value": [ 22:42:00 DEBUG ] 22:42:00 DEBUG } 22:42:00 DEBUG ] 22:42:00 DEBUG 1713303720, 22:42:00 DEBUG } 22:42:00 DEBUG ] 22:42:00 DEBUG ] 22:42:00 DEBUG } 22:42:00 DEBUG "pod": "ds-cts-0" 22:42:00 DEBUG } 22:42:00 DEBUG } 22:42:00 DEBUG 1713303720, 22:42:00 DEBUG } 22:42:00 DEBUG ] 22:42:00 DEBUG } 22:42:00 DEBUG "45474924.06666666" 22:42:00 DEBUG } 22:42:00 DEBUG } 22:42:00 DEBUG }, 22:42:00 DEBUG ] 22:42:00 DEBUG } 22:42:00 DEBUG "1318.1" 22:42:00 DEBUG ] 22:42:00 DEBUG } 22:42:00 DEBUG ] 22:42:00 DEBUG ] 22:42:00 DEBUG } 22:42:00 DEBUG } 22:42:00 DEBUG "value": [ 22:42:00 DEBUG } 22:42:00 DEBUG ] 22:42:00 DEBUG } 22:42:00 DEBUG } 22:42:00 DEBUG } 22:42:00 DEBUG } 22:42:00 DEBUG 1713303720, 22:42:00 DEBUG } 22:42:00 DEBUG } 22:42:00 DEBUG } 22:42:00 DEBUG } 22:42:00 DEBUG ] 22:42:00 DEBUG "5.802296395049134" 22:42:00 DEBUG ] 22:42:00 DEBUG } 22:42:00 DEBUG ] 22:42:00 DEBUG } 22:42:00 DEBUG } 22:42:00 DEBUG } 22:42:00 DEBUG } 22:42:00 DEBUG ] 22:42:00 DEBUG } 22:42:00 DEBUG } 22:42:08 WARNING File /tmp/lodemon_data-20240416_220944/node_disk_read_bytes_total.json does not exist. Skipping average value computation... 127.0.0.1 - - [16/Apr/2024 22:42:08] "GET /monitoring/average?start_time=20240416_211256&stop_time=20240416_214008 HTTP/1.1" 200 - 22:42:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713303776" 22:42:56 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303776" 22:42:56 INFO [http_cmd]: http status code OK 22:42:56 DEBUG --- status code --- 22:42:56 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:42:56 DEBUG --- http response --- 22:42:56 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 22:42:56 ERROR [http_cmd]: ERROR 22:42:56 ERROR --- status code --- 22:42:56 ERROR http status code is 400 (expected [200, 401, 403, 404]) 22:42:56 ERROR --- http response --- 22:42:56 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 22:42:56 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 22:42:58 INFO 22:42:58 INFO [loop_until]: kubectl --namespace=xlou top pods 22:42:58 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:42:58 INFO [loop_until]: OK (rc = 0) 22:42:58 DEBUG --- stdout --- 22:42:58 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 9Mi am-555f4db595-4lbss 4689m 8649Mi am-555f4db595-fz8sn 2692m 8528Mi am-555f4db595-kzxsp 5390m 8630Mi ds-cts-0 10798m 13253Mi ds-cts-1 11476m 13324Mi ds-cts-2 10237m 12593Mi ds-idrepo-0 1922m 5028Mi ds-idrepo-1 1134m 2875Mi ds-idrepo-2 1438m 3159Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 3m 1208Mi idm-b85f9d978-rl76f 3m 1225Mi lodemon-bc756c647-cr7x6 2m 63Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-7c9b667974-9mr2s 3484m 613Mi 22:42:58 DEBUG --- stderr --- 22:42:58 DEBUG 22:43:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303780" 22:43:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713303780" 22:43:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303780" 22:43:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303780" 22:43:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303780" 22:43:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303780" 22:43:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303780" 22:43:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303780" 22:43:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303780" 22:43:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303780" 22:43:00 INFO [http_cmd]: http status code OK 22:43:00 DEBUG --- status code --- 22:43:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:43:00 DEBUG --- http response --- 22:43:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713303780" 22:43:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 22:43:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303780" 22:43:00 INFO [http_cmd]: http status code OK 22:43:00 INFO [http_cmd]: http status code OK 22:43:00 INFO [http_cmd]: http status code OK 22:43:00 DEBUG ----- output ----- 22:43:00 INFO [http_cmd]: http status code OK 22:43:00 DEBUG --- status code --- 22:43:00 INFO [http_cmd]: http status code OK 22:43:00 INFO [http_cmd]: http status code OK 22:43:00 DEBUG --- status code --- 22:43:00 DEBUG --- status code --- 22:43:00 DEBUG { 22:43:00 DEBUG --- status code --- 22:43:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:43:00 INFO [http_cmd]: http status code OK 22:43:00 DEBUG --- status code --- 22:43:00 INFO [http_cmd]: http status code OK 22:43:00 DEBUG --- status code --- 22:43:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:43:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:43:00 DEBUG "status": "success", 22:43:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:43:00 DEBUG --- http response --- 22:43:00 DEBUG --- status code --- 22:43:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:43:00 DEBUG --- status code --- 22:43:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:43:00 DEBUG --- http response --- 22:43:00 DEBUG --- http response --- 22:43:00 DEBUG "data": { 22:43:00 DEBUG --- http response --- 22:43:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:43:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:43:00 DEBUG --- http response --- 22:43:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:43:00 DEBUG --- http response --- 22:43:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:43:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:43:00 DEBUG "resultType": "vector", 22:43:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:43:00 DEBUG --- http response --- 22:43:00 DEBUG ----- output ----- 22:43:00 DEBUG --- http response --- 22:43:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:43:00 DEBUG ----- output ----- 22:43:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:43:00 DEBUG ----- output ----- 22:43:00 DEBUG "result": [ 22:43:00 DEBUG ----- output ----- 22:43:00 INFO [http_cmd]: http status code OK 22:43:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:43:00 DEBUG { 22:43:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:43:00 INFO [http_cmd]: http status code OK 22:43:00 DEBUG ----- output ----- 22:43:00 DEBUG { 22:43:00 DEBUG ----- output ----- 22:43:00 DEBUG { 22:43:00 DEBUG { 22:43:00 DEBUG { 22:43:00 DEBUG --- status code --- 22:43:00 DEBUG ----- output ----- 22:43:00 DEBUG "status": "success", 22:43:00 INFO [http_cmd]: http status code OK 22:43:00 DEBUG ----- output ----- 22:43:00 DEBUG --- status code --- 22:43:00 DEBUG { 22:43:00 DEBUG "status": "success", 22:43:00 DEBUG { 22:43:00 DEBUG "status": "success", 22:43:00 DEBUG "metric": { 22:43:00 DEBUG "status": "success", 22:43:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:43:00 DEBUG { 22:43:00 DEBUG "data": { 22:43:00 DEBUG --- status code --- 22:43:00 DEBUG { 22:43:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:43:00 DEBUG "status": "success", 22:43:00 DEBUG "data": { 22:43:00 DEBUG "status": "success", 22:43:00 DEBUG "data": { 22:43:00 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:43:00 DEBUG "data": { 22:43:00 DEBUG --- http response --- 22:43:00 DEBUG "status": "success", 22:43:00 DEBUG "resultType": "vector", 22:43:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:43:00 DEBUG "status": "success", 22:43:00 DEBUG --- http response --- 22:43:00 DEBUG "data": { 22:43:00 DEBUG "resultType": "vector", 22:43:00 DEBUG "data": { 22:43:00 DEBUG "resultType": "vector", 22:43:00 DEBUG }, 22:43:00 DEBUG "resultType": "vector", 22:43:00 INFO 22:43:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 22:43:00 DEBUG "data": { 22:43:00 DEBUG "result": [ 22:43:00 DEBUG --- http response --- 22:43:00 DEBUG "data": { 22:43:00 DEBUG "resultType": "vector", 22:43:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:43:00 DEBUG "result": [ 22:43:00 DEBUG "resultType": "vector", 22:43:00 DEBUG "result": [ 22:43:00 DEBUG "value": [ 22:43:00 DEBUG "result": [ 22:43:00 INFO [loop_until]: kubectl --namespace=xlou top node 22:43:00 DEBUG ----- output ----- 22:43:00 DEBUG "resultType": "vector", 22:43:00 DEBUG { 22:43:00 DEBUG "resultType": "vector", 22:43:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:43:00 DEBUG "result": [ 22:43:00 DEBUG ----- output ----- 22:43:00 DEBUG { 22:43:00 DEBUG "result": [ 22:43:00 DEBUG { 22:43:00 DEBUG 22:43:00 DEBUG { 22:43:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:43:00 DEBUG { 22:43:00 DEBUG "result": [ 22:43:00 DEBUG "metric": { 22:43:00 DEBUG "result": [ 22:43:00 DEBUG ----- output ----- 22:43:00 DEBUG { 22:43:00 DEBUG { 22:43:00 DEBUG "metric": { 22:43:00 DEBUG { 22:43:00 DEBUG "metric": { 22:43:00 DEBUG 22:43:00 DEBUG "metric": { 22:43:00 DEBUG "status": "success", 22:43:00 DEBUG { 22:43:00 DEBUG "pod": "ds-idrepo-2" 22:43:00 DEBUG { 22:43:00 DEBUG { 22:43:00 DEBUG "metric": { 22:43:00 DEBUG "status": "success", 22:43:00 DEBUG "pod": "ds-idrepo-2" 22:43:00 DEBUG "metric": { 22:43:00 DEBUG "pod": "am-555f4db595-fz8sn" 22:43:00 DEBUG 22:43:00 DEBUG "pod": "ds-idrepo-2" 22:43:00 DEBUG "data": { 22:43:00 DEBUG "metric": { 22:43:00 DEBUG }, 22:43:00 DEBUG "metric": { 22:43:00 DEBUG "status": "success", 22:43:00 DEBUG "pod": "ds-idrepo-2" 22:43:00 DEBUG "data": { 22:43:00 DEBUG }, 22:43:00 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:43:00 DEBUG }, 22:43:00 DEBUG [...] 22:43:00 DEBUG }, 22:43:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303780" 22:43:00 DEBUG "resultType": "vector", 22:43:00 DEBUG "pod": "am-555f4db595-fz8sn" 22:43:00 DEBUG "value": [ 22:43:00 DEBUG "instance": "10.142.0.107:9100" 22:43:00 DEBUG "data": { 22:43:00 DEBUG }, 22:43:00 DEBUG "resultType": "vector", 22:43:00 DEBUG "value": [ 22:43:00 DEBUG }, 22:43:00 DEBUG "value": [ 22:43:00 DEBUG 22:43:00 DEBUG "value": [ 22:43:00 DEBUG "result": [ 22:43:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303780" 22:43:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713303780" 22:43:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303780" 22:43:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303780" 22:43:00 DEBUG }, 22:43:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303780" 22:43:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303780" 22:43:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303780" 22:43:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713303780" 22:43:00 DEBUG 22:43:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303780" 22:43:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303780" 22:43:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303780" 22:43:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303780" 22:43:00 DEBUG }, 22:43:00 DEBUG "resultType": "vector", 22:43:00 DEBUG "value": [ 22:43:00 DEBUG "result": [ 22:43:00 DEBUG 22:43:00 DEBUG "value": [ 22:43:00 DEBUG 22:43:00 DEBUG 22:43:00 DEBUG 22:43:00 DEBUG { 22:43:00 DEBUG "value": [ 22:43:00 INFO [http_cmd]: http status code OK 22:43:00 DEBUG 22:43:00 INFO [http_cmd]: http status code OK 22:43:00 INFO [http_cmd]: http status code OK 22:43:00 DEBUG "value": [ 22:43:00 INFO [http_cmd]: http status code OK 22:43:00 INFO [http_cmd]: http status code OK 22:43:00 INFO [http_cmd]: http status code OK 22:43:00 INFO [http_cmd]: http status code OK 22:43:00 DEBUG "result": [ 22:43:00 DEBUG 22:43:00 DEBUG { 22:43:00 INFO [http_cmd]: http status code OK 22:43:00 DEBUG 22:43:00 DEBUG 22:43:00 DEBUG 22:43:00 DEBUG 22:43:00 INFO [http_cmd]: http status code OK 22:43:00 INFO [http_cmd]: http status code OK 22:43:00 DEBUG 22:43:00 DEBUG "metric": { 22:43:00 DEBUG 22:43:00 DEBUG --- status code --- 22:43:00 DEBUG 22:43:00 DEBUG --- status code --- 22:43:00 DEBUG --- status code --- 22:43:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:43:00 DEBUG --- status code --- 22:43:00 DEBUG --- status code --- 22:43:00 DEBUG --- status code --- 22:43:00 DEBUG --- status code --- 22:43:00 DEBUG { 22:43:00 DEBUG 22:43:00 INFO [http_cmd]: http status code OK 22:43:00 DEBUG "metric": { 22:43:00 DEBUG --- status code --- 22:43:00 DEBUG 22:43:00 DEBUG 22:43:00 DEBUG 22:43:00 DEBUG "pod": "lodemon-bc756c647-cr7x6" 22:43:00 DEBUG --- status code --- 22:43:00 DEBUG --- status code --- 22:43:00 DEBUG 22:43:00 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:43:00 DEBUG 22:43:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:43:00 DEBUG [...] 22:43:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:43:00 INFO [http_cmd]: http status code OK 22:43:00 DEBUG --- http response --- 22:43:00 DEBUG 22:43:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:43:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:43:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:43:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:43:00 DEBUG "metric": { 22:43:00 DEBUG 22:43:00 DEBUG --- status code --- 22:43:00 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:43:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:43:00 DEBUG [...] 22:43:00 DEBUG 22:43:00 DEBUG [...] 22:43:00 DEBUG }, 22:43:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:43:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:43:00 DEBUG [...] 22:43:00 DEBUG }, 22:43:00 INFO [http_cmd]: http status code OK 22:43:00 DEBUG 22:43:00 DEBUG --- http response --- 22:43:00 DEBUG 22:43:00 DEBUG --- http response --- 22:43:00 DEBUG --- status code --- 22:43:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:43:00 DEBUG 22:43:00 DEBUG --- http response --- 22:43:00 DEBUG --- http response --- 22:43:00 DEBUG --- http response --- 22:43:00 DEBUG --- http response --- 22:43:00 DEBUG "pod": "ds-idrepo-2" 22:43:00 DEBUG [...] 22:43:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:43:00 DEBUG }, 22:43:00 DEBUG --- http response --- 22:43:00 DEBUG 22:43:00 DEBUG [...] 22:43:00 DEBUG 22:43:00 DEBUG "value": [ 22:43:00 DEBUG --- http response --- 22:43:00 DEBUG --- http response --- 22:43:00 DEBUG 22:43:00 DEBUG "value": [ 22:43:00 DEBUG --- status code --- 22:43:00 DEBUG [...] 22:43:00 DEBUG 22:43:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:43:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:43:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:43:00 DEBUG ----- output ----- 22:43:00 DEBUG 22:43:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:43:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:43:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:43:00 DEBUG }, 22:43:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:43:00 DEBUG 22:43:00 DEBUG --- http response --- 22:43:00 DEBUG "value": [ 22:43:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:43:00 DEBUG 22:43:00 DEBUG 22:43:00 DEBUG 22:43:00 DEBUG 1713303780, 22:43:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:43:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:43:00 DEBUG 22:43:00 DEBUG 22:43:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:43:00 DEBUG 22:43:00 DEBUG 22:43:00 DEBUG ----- output ----- 22:43:00 DEBUG ----- output ----- 22:43:00 DEBUG --- http response --- 22:43:00 DEBUG { 22:43:00 DEBUG [...] 22:43:00 DEBUG ----- output ----- 22:43:00 DEBUG ----- output ----- 22:43:00 DEBUG ----- output ----- 22:43:00 DEBUG "value": [ 22:43:00 DEBUG ----- output ----- 22:43:00 DEBUG 22:43:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:43:00 DEBUG 22:43:00 DEBUG ----- output ----- 22:43:00 DEBUG 22:43:00 DEBUG 22:43:00 DEBUG 22:43:00 DEBUG "4770.4949854416045" 22:43:00 DEBUG ----- output ----- 22:43:00 DEBUG ----- output ----- 22:43:00 DEBUG 22:43:00 DEBUG 22:43:00 DEBUG --- http response --- 22:43:00 DEBUG 22:43:00 DEBUG "pod": "ds-idrepo-0" 22:43:00 DEBUG { 22:43:00 DEBUG { 22:43:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:43:00 DEBUG "status": "success", 22:43:00 DEBUG 22:43:00 DEBUG { 22:43:00 DEBUG { 22:43:00 DEBUG { 22:43:00 DEBUG 22:43:00 DEBUG { 22:43:00 DEBUG 22:43:00 DEBUG ----- output ----- 22:43:00 DEBUG 22:43:00 DEBUG { 22:43:00 DEBUG "pod": "ds-cts-2" 22:43:00 DEBUG 22:43:00 DEBUG "pod": "am-555f4db595-kzxsp" 22:43:00 DEBUG ] 22:43:00 DEBUG { 22:43:00 DEBUG { 22:43:00 DEBUG "pod": "ds-idrepo-0" 22:43:00 DEBUG 22:43:00 DEBUG 22:43:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:43:00 DEBUG }, 22:43:00 DEBUG "status": "success", 22:43:00 DEBUG "status": "success", 22:43:00 DEBUG ----- output ----- 22:43:00 DEBUG "data": { 22:43:00 DEBUG 22:43:00 DEBUG "status": "success", 22:43:00 DEBUG "status": "success", 22:43:00 DEBUG "status": "success", 22:43:00 DEBUG 22:43:00 DEBUG "status": "success", 22:43:00 DEBUG "pod": "ds-idrepo-0" 22:43:00 DEBUG { 22:43:00 DEBUG 22:43:00 DEBUG "status": "success", 22:43:00 DEBUG }, 22:43:00 DEBUG "pod": "overseer-0-7c9b667974-9mr2s" 22:43:00 DEBUG }, 22:43:00 DEBUG } 22:43:00 DEBUG "status": "success", 22:43:00 DEBUG "status": "success", 22:43:00 DEBUG }, 22:43:00 DEBUG [...] 22:43:00 DEBUG "pod": "am-555f4db595-kzxsp" 22:43:00 DEBUG ----- output ----- 22:43:00 DEBUG "value": [ 22:43:00 DEBUG "data": { 22:43:00 DEBUG "data": { 22:43:00 DEBUG { 22:43:00 DEBUG "resultType": "vector", 22:43:00 DEBUG 22:43:00 DEBUG "data": { 22:43:00 DEBUG "data": { 22:43:00 DEBUG "data": { 22:43:00 DEBUG 22:43:00 DEBUG "data": { 22:43:00 DEBUG }, 22:43:00 DEBUG "status": "success", 22:43:00 DEBUG [...] 22:43:00 DEBUG "data": { 22:43:00 DEBUG "value": [ 22:43:00 DEBUG }, 22:43:00 DEBUG "value": [ 22:43:00 DEBUG ] 22:43:00 DEBUG "data": { 22:43:00 DEBUG "data": { 22:43:00 DEBUG "value": [ 22:43:00 DEBUG 22:43:00 DEBUG }, 22:43:00 DEBUG { 22:43:00 DEBUG "status": "success", 22:43:00 DEBUG "data": { 22:43:00 DEBUG "resultType": "vector", 22:43:00 DEBUG "status": "success", 22:43:00 DEBUG "result": [ 22:43:00 DEBUG "instance": "10.142.0.43:9100" 22:43:00 DEBUG "resultType": "vector", 22:43:00 DEBUG "resultType": "vector", 22:43:00 DEBUG "resultType": "vector", 22:43:00 DEBUG [...] 22:43:00 DEBUG "resultType": "vector", 22:43:00 DEBUG "value": [ 22:43:00 DEBUG "data": { 22:43:00 DEBUG 22:43:00 DEBUG "resultType": "vector", 22:43:00 DEBUG 1713303780, 22:43:00 DEBUG "value": [ 22:43:00 DEBUG 1713303780, 22:43:00 DEBUG } 22:43:00 DEBUG "resultType": "vector", 22:43:00 DEBUG "resultType": "vector", 22:43:00 DEBUG 1713303780, 22:43:00 DEBUG 22:43:00 DEBUG "value": [ 22:43:00 DEBUG 1713303780, 22:43:00 DEBUG "resultType": "vector", 22:43:00 DEBUG "resultType": "vector", 22:43:00 DEBUG "result": [ 22:43:00 DEBUG "data": { 22:43:00 DEBUG { 22:43:00 DEBUG }, 22:43:00 DEBUG "result": [ 22:43:00 DEBUG "result": [ 22:43:00 DEBUG "result": [ 22:43:00 DEBUG 22:43:00 DEBUG "result": [ 22:43:00 DEBUG 1713303780, 22:43:00 DEBUG "resultType": "vector", 22:43:00 DEBUG 22:43:00 DEBUG "result": [ 22:43:00 DEBUG "0" 22:43:00 DEBUG 1713303780, 22:43:00 DEBUG "0" 22:43:00 DEBUG } 22:43:00 DEBUG "result": [ 22:43:00 DEBUG "result": [ 22:43:00 DEBUG "0" 22:43:00 DEBUG 22:43:00 DEBUG 1713303780, 22:43:00 DEBUG "0" 22:43:00 DEBUG "result": [ 22:43:00 DEBUG "result": [ 22:43:00 DEBUG { 22:43:00 DEBUG "resultType": "vector", 22:43:00 DEBUG "metric": { 22:43:00 DEBUG "value": [ 22:43:00 DEBUG { 22:43:00 DEBUG { 22:43:00 DEBUG { 22:43:00 DEBUG 22:43:00 DEBUG { 22:43:00 DEBUG "0" 22:43:00 DEBUG "result": [ 22:43:00 DEBUG 22:43:00 DEBUG { 22:43:00 DEBUG ] 22:43:00 DEBUG "913.5" 22:43:00 DEBUG ] 22:43:00 DEBUG { 22:43:00 DEBUG { 22:43:00 DEBUG ] 22:43:00 DEBUG "pod": "lodemon-bc756c647-cr7x6" 22:43:00 DEBUG "2006.7333333333333" 22:43:00 DEBUG ] 22:43:00 DEBUG { 22:43:00 DEBUG { 22:43:00 DEBUG "metric": { 22:43:00 DEBUG "result": [ 22:43:00 DEBUG "pod": "ds-idrepo-2" 22:43:00 DEBUG 1713303780, 22:43:00 DEBUG "metric": { 22:43:00 DEBUG "metric": { 22:43:00 DEBUG "metric": { 22:43:00 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:43:00 DEBUG "metric": { 22:43:00 DEBUG ] 22:43:00 DEBUG { 22:43:00 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:43:00 DEBUG "metric": { 22:43:00 DEBUG } 22:43:00 DEBUG ] 22:43:00 DEBUG } 22:43:00 DEBUG "metric": { 22:43:00 DEBUG "metric": { 22:43:00 DEBUG } 22:43:00 DEBUG }, 22:43:00 DEBUG ] 22:43:00 DEBUG } 22:43:00 DEBUG "metric": { 22:43:00 DEBUG "metric": { 22:43:00 DEBUG "pod": "am-555f4db595-fz8sn" 22:43:00 DEBUG { 22:43:00 DEBUG }, 22:43:00 DEBUG "19967992832" 22:43:00 DEBUG "pod": "ds-idrepo-2" 22:43:00 DEBUG "pod": "am-555f4db595-fz8sn" 22:43:00 DEBUG 22:43:00 DEBUG }, 22:43:00 DEBUG "pod": "am-555f4db595-fz8sn" 22:43:00 DEBUG } 22:43:00 DEBUG "metric": { 22:43:00 DEBUG }, 22:43:00 DEBUG "instance": "10.142.0.107:9100" 22:43:00 DEBUG ] 22:43:00 DEBUG } 22:43:00 DEBUG ] 22:43:00 DEBUG "pod": "am-555f4db595-fz8sn" 22:43:00 DEBUG "instance": "10.142.0.107:9100" 22:43:00 DEBUG ] 22:43:00 DEBUG "value": [ 22:43:00 DEBUG } 22:43:00 DEBUG ] 22:43:00 DEBUG "instance": "10.142.0.107:9100" 22:43:00 DEBUG "instance": "10.142.0.107:9100" 22:43:00 DEBUG }, 22:43:00 DEBUG "metric": { 22:43:00 DEBUG "value": [ 22:43:00 DEBUG ] 22:43:00 DEBUG }, 22:43:00 DEBUG }, 22:43:00 DEBUG "pod": "ds-idrepo-0" 22:43:00 DEBUG "value": [ 22:43:00 DEBUG }, 22:43:00 DEBUG ] 22:43:00 DEBUG "pod": "am-555f4db595-fz8sn" 22:43:00 DEBUG "value": [ 22:43:00 DEBUG }, 22:43:00 DEBUG } 22:43:00 DEBUG ] 22:43:00 DEBUG } 22:43:00 DEBUG }, 22:43:00 DEBUG }, 22:43:00 DEBUG } 22:43:00 DEBUG 1713303780, 22:43:00 DEBUG ] 22:43:00 DEBUG } 22:43:00 DEBUG }, 22:43:00 DEBUG }, 22:43:00 DEBUG "value": [ 22:43:00 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:43:00 DEBUG 22:43:00 DEBUG } 22:43:00 DEBUG "value": [ 22:43:00 DEBUG "value": [ 22:43:00 DEBUG }, 22:43:00 DEBUG 22:43:00 DEBUG "value": [ 22:43:00 DEBUG } 22:43:00 DEBUG }, 22:43:00 DEBUG 1713303780, 22:43:00 DEBUG "value": [ 22:43:00 DEBUG } 22:43:00 DEBUG } 22:43:00 DEBUG } 22:43:00 DEBUG "value": [ 22:43:00 DEBUG "value": [ 22:43:00 DEBUG } 22:43:00 DEBUG "0" 22:43:00 DEBUG } 22:43:00 DEBUG } 22:43:00 DEBUG "value": [ 22:43:00 DEBUG "value": [ 22:43:00 DEBUG 22:43:00 DEBUG }, 22:43:00 DEBUG 22:43:00 DEBUG ] 22:43:00 DEBUG 22:43:00 DEBUG 22:43:00 DEBUG "value": [ 22:43:00 DEBUG 22:43:00 DEBUG 22:43:00 DEBUG } 22:43:00 DEBUG "value": [ 22:43:00 DEBUG "428965888" 22:43:00 DEBUG 22:43:00 DEBUG } 22:43:00 DEBUG 22:43:00 DEBUG 22:43:00 DEBUG ] 22:43:00 DEBUG } 22:43:00 INFO [loop_until]: OK (rc = 0) 22:43:00 DEBUG 22:43:00 DEBUG 22:43:00 DEBUG 22:43:00 DEBUG "value": [ 22:43:00 DEBUG 22:43:00 DEBUG } 22:43:00 DEBUG 22:43:00 DEBUG 22:43:00 DEBUG 1713303780, 22:43:00 DEBUG 22:43:00 DEBUG 22:43:00 DEBUG 22:43:00 DEBUG ] 22:43:00 DEBUG 22:43:00 DEBUG 22:43:00 DEBUG 22:43:00 DEBUG } 22:43:00 DEBUG --- stdout --- 22:43:00 DEBUG 22:43:00 DEBUG 22:43:00 DEBUG 22:43:00 DEBUG 22:43:00 DEBUG [...] 22:43:00 DEBUG } 22:43:00 DEBUG 22:43:00 DEBUG 22:43:00 DEBUG "0" 22:43:00 DEBUG [...] 22:43:00 DEBUG 22:43:00 DEBUG 22:43:00 DEBUG } 22:43:00 DEBUG 22:43:00 DEBUG 22:43:00 DEBUG 22:43:00 DEBUG ] 22:43:00 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 65m 0% 5105Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 10835m 36% 15823Mi 14% gke-xlou-cdm-default-pool-1b0769c8-nhlf 6136m 20% 10778Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 79m 0% 2639Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 1597m 5% 6516Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 11038m 36% 17639Mi 15% gke-xlou-cdm-default-pool-1b0769c8-xnmp 61m 0% 3157Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 4379m 14% 10432Mi 9% gke-xlou-cdm-ds-0f601751-hmm7 2734m 9% 5623Mi 5% gke-xlou-cdm-ds-0f601751-j488 10689m 35% 14447Mi 12% gke-xlou-cdm-ds-0f601751-mc5d 3001m 10% 3236Mi 2% gke-xlou-cdm-ds-0f601751-nj6q 5430m 18% 10499Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 1892m 6% 7360Mi 6% gke-xlou-cdm-frontend-a8771548-i82n 3270m 20% 5936Mi 10% 22:43:00 DEBUG 22:43:00 DEBUG 22:43:00 DEBUG [...] 22:43:00 DEBUG 22:43:00 DEBUG 22:43:00 DEBUG [...] 22:43:00 DEBUG [...] 22:43:00 DEBUG ] 22:43:00 DEBUG 22:43:00 DEBUG [...] 22:43:00 DEBUG 22:43:00 DEBUG ] 22:43:00 DEBUG [...] 22:43:00 DEBUG [...] 22:43:00 DEBUG [...] 22:43:00 DEBUG } 22:43:00 DEBUG --- stderr --- 22:43:00 DEBUG [...] 22:43:00 DEBUG [...] 22:43:00 DEBUG 22:43:00 DEBUG 22:43:00 DEBUG 22:43:00 DEBUG 22:43:00 DEBUG 22:43:00 DEBUG } 22:43:00 DEBUG 22:43:00 DEBUG 22:43:00 DEBUG [...] 22:43:00 DEBUG } 22:43:00 DEBUG 22:43:00 DEBUG 22:43:00 DEBUG 22:43:00 DEBUG } 22:43:00 DEBUG 22:43:00 DEBUG 22:43:00 DEBUG 22:43:00 DEBUG 22:43:00 DEBUG [...] 22:43:00 DEBUG 22:43:00 DEBUG 22:43:00 DEBUG 22:43:00 DEBUG ] 22:43:00 DEBUG 22:43:00 DEBUG 22:43:00 DEBUG 22:43:00 DEBUG } 22:43:00 DEBUG 22:43:00 DEBUG 22:43:00 DEBUG 22:43:00 DEBUG 22:43:00 DEBUG 22:43:00 DEBUG 22:43:00 DEBUG 22:43:00 DEBUG "pod": "ds-idrepo-0" 22:43:00 DEBUG 22:43:00 DEBUG 22:43:00 DEBUG } 22:43:00 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:43:00 DEBUG 22:43:00 DEBUG 22:43:00 DEBUG 22:43:00 DEBUG 22:43:00 DEBUG 22:43:00 DEBUG 22:43:00 DEBUG 22:43:00 DEBUG "pod": "am-555f4db595-kzxsp" 22:43:00 DEBUG 22:43:00 DEBUG }, 22:43:00 DEBUG "pod": "ds-idrepo-0" 22:43:00 DEBUG "pod": "am-555f4db595-kzxsp" 22:43:00 DEBUG } 22:43:00 DEBUG }, 22:43:00 DEBUG "pod": "am-555f4db595-kzxsp" 22:43:00 DEBUG 22:43:00 DEBUG "instance": "10.142.0.43:9100" 22:43:00 DEBUG "pod": "am-555f4db595-kzxsp" 22:43:00 DEBUG "instance": "10.142.0.43:9100" 22:43:00 DEBUG "instance": "10.142.0.43:9100" 22:43:00 DEBUG "instance": "10.142.0.43:9100" 22:43:00 DEBUG }, 22:43:00 DEBUG 22:43:00 DEBUG "value": [ 22:43:00 DEBUG }, 22:43:00 DEBUG }, 22:43:00 DEBUG "value": [ 22:43:00 DEBUG }, 22:43:00 DEBUG "pod": "am-555f4db595-kzxsp" 22:43:00 DEBUG }, 22:43:00 DEBUG }, 22:43:00 DEBUG }, 22:43:00 DEBUG }, 22:43:00 DEBUG }, 22:43:00 DEBUG "value": [ 22:43:00 DEBUG "pod": "overseer-0-7c9b667974-9mr2s" 22:43:00 DEBUG 1713303780, 22:43:00 DEBUG "value": [ 22:43:00 DEBUG "value": [ 22:43:00 DEBUG 1713303780, 22:43:00 DEBUG "value": [ 22:43:00 DEBUG }, 22:43:00 DEBUG "value": [ 22:43:00 DEBUG "value": [ 22:43:00 DEBUG "value": [ 22:43:00 DEBUG "value": [ 22:43:00 DEBUG "value": [ 22:43:00 DEBUG 1713303780, 22:43:00 DEBUG }, 22:43:00 DEBUG "0" 22:43:00 DEBUG 1713303780, 22:43:00 DEBUG 1713303780, 22:43:00 DEBUG "1.5188089341318227" 22:43:00 DEBUG 1713303780, 22:43:00 DEBUG "value": [ 22:43:00 DEBUG 1713303780, 22:43:00 DEBUG 1713303780, 22:43:00 DEBUG 1713303780, 22:43:00 DEBUG "3043173.8" 22:43:00 DEBUG 1713303780, 22:43:00 DEBUG "3501113" 22:43:00 DEBUG "value": [ 22:43:00 DEBUG ] 22:43:00 DEBUG "0" 22:43:00 DEBUG "0" 22:43:00 DEBUG ] 22:43:00 DEBUG "912.4333333333333" 22:43:00 DEBUG 1713303780, 22:43:00 DEBUG "0.00033333333333333365" 22:43:00 DEBUG "0.026" 22:43:00 DEBUG 1713303780, 22:43:00 DEBUG ] 22:43:00 DEBUG "0.0005333333333330377" 22:43:00 DEBUG ] 22:43:00 DEBUG 1713303780, 22:43:00 DEBUG } 22:43:00 DEBUG ] 22:43:00 DEBUG ] 22:43:00 DEBUG } 22:43:00 DEBUG ] 22:43:00 DEBUG "911.6" 22:43:00 DEBUG ] 22:43:00 DEBUG ] 22:43:00 DEBUG "3035235.8" 22:43:00 DEBUG } 22:43:00 DEBUG ] 22:43:00 DEBUG } 22:43:00 DEBUG "0" 22:43:00 DEBUG ] 22:43:00 DEBUG } 22:43:00 DEBUG } 22:43:00 DEBUG ] 22:43:00 DEBUG } 22:43:00 DEBUG ] 22:43:00 DEBUG } 22:43:00 DEBUG } 22:43:00 DEBUG ] 22:43:00 DEBUG ] 22:43:00 DEBUG } 22:43:00 DEBUG ] 22:43:00 DEBUG ] 22:43:00 DEBUG } 22:43:00 DEBUG ] 22:43:00 DEBUG ] 22:43:00 DEBUG } 22:43:00 DEBUG ] 22:43:00 DEBUG } 22:43:00 DEBUG ] 22:43:00 DEBUG ] 22:43:00 DEBUG } 22:43:00 DEBUG } 22:43:00 DEBUG ] 22:43:00 DEBUG } 22:43:00 DEBUG } 22:43:00 DEBUG } 22:43:00 DEBUG } 22:43:00 DEBUG } 22:43:00 DEBUG } 22:43:00 DEBUG } 22:43:00 DEBUG ] 22:43:00 DEBUG } 22:43:00 DEBUG } 22:43:00 DEBUG ] 22:43:00 DEBUG } 22:43:00 DEBUG } 22:43:00 DEBUG } 22:43:00 DEBUG ] 22:43:00 DEBUG } 22:43:00 DEBUG } 22:43:00 DEBUG } 22:43:00 DEBUG } 22:43:00 DEBUG } 22:43:00 DEBUG } 22:43:00 DEBUG } 22:43:00 DEBUG } 22:43:00 DEBUG } 22:43:00 DEBUG } 22:43:00 DEBUG } 22:43:00 DEBUG } 22:43:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303837" 22:43:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713303837" 22:43:57 ERROR [http_cmd]: ERROR 22:43:57 INFO [http_cmd]: http status code OK 22:43:57 ERROR --- status code --- 22:43:57 DEBUG --- status code --- 22:43:57 ERROR http status code is 400 (expected [200, 401, 403, 404]) 22:43:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:43:57 ERROR --- http response --- 22:43:57 DEBUG --- http response --- 22:43:57 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 22:43:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 22:43:57 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 22:43:59 INFO 22:43:59 INFO [loop_until]: kubectl --namespace=xlou top pods 22:43:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:43:59 INFO [loop_until]: OK (rc = 0) 22:43:59 DEBUG --- stdout --- 22:43:59 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 9Mi am-555f4db595-4lbss 13260m 8672Mi am-555f4db595-fz8sn 13162m 8553Mi am-555f4db595-kzxsp 13392m 8660Mi ds-cts-0 19508m 12642Mi ds-cts-1 17934m 12662Mi ds-cts-2 17858m 12601Mi ds-idrepo-0 5891m 5028Mi ds-idrepo-1 5659m 2875Mi ds-idrepo-2 5648m 3160Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 4m 1208Mi idm-b85f9d978-rl76f 4m 1225Mi lodemon-bc756c647-cr7x6 2m 63Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-7c9b667974-9mr2s 6737m 712Mi 22:43:59 DEBUG --- stderr --- 22:43:59 DEBUG 22:44:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303840" 22:44:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303840" 22:44:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713303840" 22:44:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303840" 22:44:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303840" 22:44:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303840" 22:44:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303840" 22:44:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303840" 22:44:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303840" 22:44:00 INFO 22:44:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303840" 22:44:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713303840" 22:44:00 INFO [http_cmd]: http status code OK 22:44:00 INFO [http_cmd]: http status code OK 22:44:00 INFO [loop_until]: kubectl --namespace=xlou top node 22:44:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303840" 22:44:00 DEBUG --- status code --- 22:44:00 DEBUG --- status code --- 22:44:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:44:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:44:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:44:00 DEBUG --- http response --- 22:44:00 DEBUG --- http response --- 22:44:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713303840" 22:44:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303840" 22:44:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303840" 22:44:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303840" 22:44:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303840" 22:44:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303840" 22:44:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303840" 22:44:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:44:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303840" 22:44:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303840" 22:44:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303840" 22:44:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303840" 22:44:00 INFO [http_cmd]: http status code OK 22:44:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303840" 22:44:00 INFO [http_cmd]: http status code OK 22:44:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:44:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713303840" 22:44:00 INFO [http_cmd]: http status code OK 22:44:00 DEBUG ----- output ----- 22:44:00 DEBUG --- status code --- 22:44:00 INFO [http_cmd]: http status code OK 22:44:00 INFO [http_cmd]: http status code OK 22:44:00 DEBUG --- status code --- 22:44:00 DEBUG ----- output ----- 22:44:00 INFO [http_cmd]: http status code OK 22:44:00 INFO [http_cmd]: http status code OK 22:44:00 INFO [http_cmd]: http status code OK 22:44:00 INFO [http_cmd]: http status code OK 22:44:00 INFO [http_cmd]: http status code OK 22:44:00 INFO [http_cmd]: http status code OK 22:44:00 INFO [http_cmd]: http status code OK 22:44:00 INFO [http_cmd]: http status code OK 22:44:00 INFO [http_cmd]: http status code OK 22:44:00 INFO [http_cmd]: http status code OK 22:44:00 INFO [http_cmd]: http status code OK 22:44:00 INFO [http_cmd]: http status code OK 22:44:00 DEBUG --- status code --- 22:44:00 DEBUG { 22:44:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:44:00 DEBUG --- status code --- 22:44:00 DEBUG --- status code --- 22:44:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:44:00 DEBUG { 22:44:00 DEBUG --- status code --- 22:44:00 DEBUG --- status code --- 22:44:00 DEBUG --- status code --- 22:44:00 DEBUG --- status code --- 22:44:00 DEBUG --- status code --- 22:44:00 DEBUG --- status code --- 22:44:00 DEBUG --- status code --- 22:44:00 DEBUG --- status code --- 22:44:00 DEBUG --- status code --- 22:44:00 INFO [http_cmd]: http status code OK 22:44:00 INFO [http_cmd]: http status code OK 22:44:00 DEBUG --- status code --- 22:44:00 INFO [http_cmd]: http status code OK 22:44:00 DEBUG --- status code --- 22:44:00 INFO [http_cmd]: http status code OK 22:44:00 DEBUG --- status code --- 22:44:00 INFO [http_cmd]: http status code OK 22:44:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:44:00 DEBUG "status": "success", 22:44:00 DEBUG --- http response --- 22:44:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:44:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:44:00 DEBUG --- http response --- 22:44:00 DEBUG "status": "success", 22:44:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:44:00 INFO [http_cmd]: http status code OK 22:44:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:44:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:44:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:44:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:44:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:44:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:44:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:44:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:44:00 DEBUG --- status code --- 22:44:00 DEBUG --- status code --- 22:44:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:44:00 DEBUG --- status code --- 22:44:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:44:00 DEBUG --- status code --- 22:44:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:44:00 DEBUG --- status code --- 22:44:00 DEBUG --- http response --- 22:44:00 DEBUG "data": { 22:44:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:44:00 DEBUG --- http response --- 22:44:00 DEBUG --- http response --- 22:44:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:44:00 DEBUG "data": { 22:44:00 DEBUG --- http response --- 22:44:00 DEBUG --- status code --- 22:44:00 DEBUG --- http response --- 22:44:00 DEBUG --- http response --- 22:44:00 DEBUG --- http response --- 22:44:00 DEBUG --- http response --- 22:44:00 DEBUG --- http response --- 22:44:00 DEBUG --- http response --- 22:44:00 DEBUG --- http response --- 22:44:00 DEBUG --- http response --- 22:44:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:44:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:44:00 DEBUG --- http response --- 22:44:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:44:00 DEBUG --- http response --- 22:44:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:44:00 DEBUG --- http response --- 22:44:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:44:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:44:00 DEBUG "resultType": "vector", 22:44:00 DEBUG ----- output ----- 22:44:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:44:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:44:00 DEBUG ----- output ----- 22:44:00 DEBUG "resultType": "vector", 22:44:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:44:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:44:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:44:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:44:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:44:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:44:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:44:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:44:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:44:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:44:00 DEBUG --- http response --- 22:44:00 DEBUG --- http response --- 22:44:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:44:00 DEBUG --- http response --- 22:44:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:44:00 DEBUG --- http response --- 22:44:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:44:00 DEBUG --- http response --- 22:44:00 DEBUG ----- output ----- 22:44:00 DEBUG "result": [ 22:44:00 DEBUG { 22:44:00 DEBUG ----- output ----- 22:44:00 DEBUG ----- output ----- 22:44:00 DEBUG { 22:44:00 DEBUG "result": [ 22:44:00 DEBUG ----- output ----- 22:44:00 DEBUG --- http response --- 22:44:00 DEBUG ----- output ----- 22:44:00 DEBUG ----- output ----- 22:44:00 DEBUG ----- output ----- 22:44:00 DEBUG ----- output ----- 22:44:00 DEBUG ----- output ----- 22:44:00 DEBUG ----- output ----- 22:44:00 DEBUG ----- output ----- 22:44:00 DEBUG ----- output ----- 22:44:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:44:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:44:00 DEBUG ----- output ----- 22:44:00 DEBUG ----- output ----- 22:44:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:44:00 DEBUG ----- output ----- 22:44:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:44:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:44:00 DEBUG { 22:44:00 DEBUG { 22:44:00 DEBUG "status": "success", 22:44:00 DEBUG { 22:44:00 DEBUG { 22:44:00 DEBUG "status": "success", 22:44:00 DEBUG { 22:44:00 DEBUG { 22:44:00 DEBUG { 22:44:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:44:00 DEBUG { 22:44:00 DEBUG { 22:44:00 DEBUG { 22:44:00 DEBUG { 22:44:00 DEBUG { 22:44:00 DEBUG { 22:44:00 DEBUG "status": "success", 22:44:00 DEBUG "data": { 22:44:00 DEBUG ----- output ----- 22:44:00 DEBUG { 22:44:00 DEBUG { 22:44:00 DEBUG ----- output ----- 22:44:00 DEBUG { 22:44:00 DEBUG ----- output ----- 22:44:00 DEBUG ----- output ----- 22:44:00 DEBUG "status": "success", 22:44:00 DEBUG "metric": { 22:44:00 DEBUG "data": { 22:44:00 DEBUG "status": "success", 22:44:00 DEBUG "status": "success", 22:44:00 DEBUG "data": { 22:44:00 DEBUG "metric": { 22:44:00 DEBUG "status": "success", 22:44:00 DEBUG "status": "success", 22:44:00 DEBUG ----- output ----- 22:44:00 DEBUG "status": "success", 22:44:00 DEBUG "status": "success", 22:44:00 DEBUG "status": "success", 22:44:00 DEBUG "status": "success", 22:44:00 DEBUG "status": "success", 22:44:00 DEBUG ----- output ----- 22:44:00 DEBUG "resultType": "vector", 22:44:00 DEBUG { 22:44:00 DEBUG { 22:44:00 DEBUG "status": "success", 22:44:00 DEBUG "status": "success", 22:44:00 DEBUG { 22:44:00 DEBUG "status": "success", 22:44:00 DEBUG { 22:44:00 DEBUG { 22:44:00 DEBUG "data": { 22:44:00 DEBUG "pod": "ds-idrepo-2" 22:44:00 DEBUG "resultType": "vector", 22:44:00 DEBUG "data": { 22:44:00 DEBUG "data": { 22:44:00 DEBUG "resultType": "vector", 22:44:00 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:44:00 DEBUG "data": { 22:44:00 DEBUG "data": { 22:44:00 DEBUG { 22:44:00 DEBUG "data": { 22:44:00 DEBUG "data": { 22:44:00 DEBUG "data": { 22:44:00 DEBUG "data": { 22:44:00 DEBUG "data": { 22:44:00 DEBUG { 22:44:00 DEBUG "result": [ 22:44:00 DEBUG "status": "success", 22:44:00 DEBUG "status": "success", 22:44:00 DEBUG "data": { 22:44:00 DEBUG "data": { 22:44:00 DEBUG "status": "success", 22:44:00 DEBUG "data": { 22:44:00 DEBUG "status": "success", 22:44:00 DEBUG "status": "success", 22:44:00 DEBUG "resultType": "vector", 22:44:00 DEBUG }, 22:44:00 DEBUG "result": [ 22:44:00 DEBUG "resultType": "vector", 22:44:00 DEBUG "resultType": "vector", 22:44:00 DEBUG "result": [ 22:44:00 DEBUG }, 22:44:00 DEBUG "resultType": "vector", 22:44:00 DEBUG "resultType": "vector", 22:44:00 DEBUG "status": "success", 22:44:00 DEBUG "resultType": "vector", 22:44:00 DEBUG "resultType": "vector", 22:44:00 DEBUG "resultType": "vector", 22:44:00 DEBUG "resultType": "vector", 22:44:00 DEBUG "resultType": "vector", 22:44:00 DEBUG "status": "success", 22:44:00 DEBUG { 22:44:00 DEBUG "data": { 22:44:00 DEBUG "data": { 22:44:00 DEBUG "resultType": "vector", 22:44:00 DEBUG "resultType": "vector", 22:44:00 DEBUG "data": { 22:44:00 DEBUG "resultType": "vector", 22:44:00 DEBUG "data": { 22:44:00 DEBUG "data": { 22:44:00 DEBUG "result": [ 22:44:00 DEBUG "value": [ 22:44:00 DEBUG { 22:44:00 DEBUG "result": [ 22:44:00 DEBUG "result": [ 22:44:00 DEBUG { 22:44:00 DEBUG "value": [ 22:44:00 DEBUG "result": [ 22:44:00 DEBUG { 22:44:00 DEBUG "data": { 22:44:00 DEBUG "result": [ 22:44:00 DEBUG "result": [ 22:44:00 DEBUG "result": [ 22:44:00 DEBUG "result": [ 22:44:00 DEBUG "result": [ 22:44:00 DEBUG "data": { 22:44:00 DEBUG "metric": { 22:44:00 DEBUG "resultType": "vector", 22:44:00 DEBUG "resultType": "vector", 22:44:00 DEBUG "result": [ 22:44:00 DEBUG "result": [ 22:44:00 DEBUG "resultType": "vector", 22:44:00 DEBUG "result": [ 22:44:00 DEBUG { 22:44:00 DEBUG "resultType": "vector", 22:44:00 DEBUG { 22:44:00 DEBUG 22:44:00 DEBUG "metric": { 22:44:00 DEBUG { 22:44:00 DEBUG { 22:44:00 DEBUG "metric": { 22:44:00 DEBUG 22:44:00 DEBUG "metric": { 22:44:00 DEBUG "result": [ 22:44:00 DEBUG "resultType": "vector", 22:44:00 DEBUG { 22:44:00 DEBUG { 22:44:00 DEBUG { 22:44:00 DEBUG { 22:44:00 DEBUG { 22:44:00 DEBUG "resultType": "vector", 22:44:00 DEBUG "pod": "am-555f4db595-fz8sn" 22:44:00 DEBUG "result": [ 22:44:00 DEBUG "result": [ 22:44:00 DEBUG { 22:44:00 DEBUG { 22:44:00 DEBUG "result": [ 22:44:00 DEBUG "resultType": "vector", 22:44:00 DEBUG "metric": { 22:44:00 DEBUG "result": [ 22:44:00 DEBUG "metric": { 22:44:00 DEBUG 22:44:00 DEBUG "pod": "ds-idrepo-2" 22:44:00 DEBUG "metric": { 22:44:00 DEBUG "metric": { 22:44:00 DEBUG "pod": "ds-idrepo-2" 22:44:00 DEBUG 22:44:00 DEBUG "pod": "ds-idrepo-2" 22:44:00 DEBUG { 22:44:00 DEBUG "result": [ 22:44:00 DEBUG "metric": { 22:44:00 DEBUG "metric": { 22:44:00 DEBUG "metric": { 22:44:00 DEBUG "metric": { 22:44:00 DEBUG "metric": { 22:44:00 DEBUG "result": [ 22:44:00 DEBUG }, 22:44:00 DEBUG { 22:44:00 DEBUG { 22:44:00 DEBUG "metric": { 22:44:00 DEBUG "metric": { 22:44:00 DEBUG { 22:44:00 DEBUG "result": [ 22:44:00 DEBUG "pod": "am-555f4db595-fz8sn" 22:44:00 DEBUG { 22:44:00 DEBUG "pod": "ds-idrepo-2" 22:44:00 DEBUG 22:44:00 DEBUG }, 22:44:00 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:44:00 DEBUG "pod": "am-555f4db595-fz8sn" 22:44:00 DEBUG }, 22:44:00 DEBUG 22:44:00 DEBUG }, 22:44:00 DEBUG "metric": { 22:44:00 DEBUG { 22:44:00 DEBUG "instance": "10.142.0.107:9100" 22:44:00 DEBUG "instance": "10.142.0.107:9100" 22:44:00 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:44:00 DEBUG "pod": "ds-idrepo-2" 22:44:00 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:44:00 DEBUG { 22:44:00 INFO [loop_until]: OK (rc = 0) 22:44:00 DEBUG "value": [ 22:44:00 DEBUG "metric": { 22:44:00 DEBUG "metric": { 22:44:00 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:44:00 DEBUG "instance": "10.142.0.107:9100" 22:44:00 DEBUG "metric": { 22:44:00 DEBUG { 22:44:00 DEBUG }, 22:44:00 DEBUG "metric": { 22:44:00 DEBUG }, 22:44:00 DEBUG [...] 22:44:00 DEBUG "value": [ 22:44:00 DEBUG }, 22:44:00 DEBUG }, 22:44:00 DEBUG "value": [ 22:44:00 DEBUG [...] 22:44:00 DEBUG "value": [ 22:44:00 DEBUG "pod": "am-555f4db595-fz8sn" 22:44:00 DEBUG "metric": { 22:44:00 DEBUG }, 22:44:00 DEBUG }, 22:44:00 DEBUG }, 22:44:00 DEBUG }, 22:44:00 DEBUG }, 22:44:00 DEBUG "metric": { 22:44:00 DEBUG --- stdout --- 22:44:00 DEBUG 22:44:00 DEBUG "pod": "am-555f4db595-fz8sn" 22:44:00 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:44:00 DEBUG }, 22:44:00 DEBUG }, 22:44:00 DEBUG "pod": "am-555f4db595-fz8sn" 22:44:00 DEBUG "metric": { 22:44:00 DEBUG "value": [ 22:44:00 DEBUG "pod": "ds-idrepo-2" 22:44:00 DEBUG "value": [ 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG "value": [ 22:44:00 DEBUG "value": [ 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG }, 22:44:00 DEBUG "pod": "am-555f4db595-fz8sn" 22:44:00 DEBUG "value": [ 22:44:00 DEBUG "value": [ 22:44:00 DEBUG "value": [ 22:44:00 DEBUG "value": [ 22:44:00 DEBUG "value": [ 22:44:00 DEBUG "instance": "10.142.0.107:9100" 22:44:00 DEBUG 22:44:00 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 5091Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 18261m 61% 15272Mi 13% gke-xlou-cdm-default-pool-1b0769c8-nhlf 13113m 43% 10782Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 81m 0% 2656Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 5571m 18% 6489Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 20612m 69% 17057Mi 15% gke-xlou-cdm-default-pool-1b0769c8-xnmp 60m 0% 3148Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 13224m 44% 10452Mi 9% gke-xlou-cdm-ds-0f601751-hmm7 6097m 20% 5622Mi 5% gke-xlou-cdm-ds-0f601751-j488 18064m 60% 14623Mi 13% gke-xlou-cdm-ds-0f601751-mc5d 11231m 37% 3264Mi 2% gke-xlou-cdm-ds-0f601751-nj6q 13355m 44% 10523Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 6066m 20% 7354Mi 6% gke-xlou-cdm-frontend-a8771548-i82n 7001m 44% 6153Mi 10% 22:44:00 DEBUG }, 22:44:00 DEBUG }, 22:44:00 DEBUG "value": [ 22:44:00 DEBUG "value": [ 22:44:00 DEBUG }, 22:44:00 DEBUG "instance": "10.142.0.107:9100" 22:44:00 DEBUG 22:44:00 DEBUG }, 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG "value": [ 22:44:00 DEBUG }, 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG }, 22:44:00 DEBUG 22:44:00 DEBUG --- stderr --- 22:44:00 DEBUG "value": [ 22:44:00 DEBUG "value": [ 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG "value": [ 22:44:00 DEBUG }, 22:44:00 DEBUG 22:44:00 DEBUG "value": [ 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG "value": [ 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG "value": [ 22:44:00 DEBUG [...] 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG "value": [ 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG "pod": "ds-idrepo-0" 22:44:00 DEBUG [...] 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG [...] 22:44:00 DEBUG "pod": "overseer-0-7c9b667974-9mr2s" 22:44:00 DEBUG [...] 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG [...] 22:44:00 DEBUG 22:44:00 DEBUG [...] 22:44:00 DEBUG }, 22:44:00 DEBUG 22:44:00 DEBUG [...] 22:44:00 DEBUG [...] 22:44:00 DEBUG 22:44:00 DEBUG }, 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG [...] 22:44:00 DEBUG [...] 22:44:00 DEBUG [...] 22:44:00 DEBUG [...] 22:44:00 DEBUG [...] 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG [...] 22:44:00 DEBUG [...] 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG "value": [ 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG "value": [ 22:44:00 DEBUG 22:44:00 DEBUG [...] 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG [...] 22:44:00 DEBUG [...] 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG [...] 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG [...] 22:44:00 DEBUG 22:44:00 DEBUG 1713303840, 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG 1713303840, 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG [...] 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG [...] 22:44:00 DEBUG "pod": "am-555f4db595-kzxsp" 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG [...] 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG "0" 22:44:00 DEBUG "pod": "ds-cts-2" 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG "pod": "ds-idrepo-0" 22:44:00 DEBUG "0" 22:44:00 DEBUG "pod": "ds-idrepo-0" 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG }, 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG "pod": "am-555f4db595-kzxsp" 22:44:00 DEBUG 22:44:00 DEBUG "pod": "ds-idrepo-0" 22:44:00 DEBUG ] 22:44:00 DEBUG }, 22:44:00 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:44:00 DEBUG "pod": "am-555f4db595-kzxsp" 22:44:00 DEBUG }, 22:44:00 DEBUG ] 22:44:00 DEBUG }, 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG "instance": "10.142.0.43:9100" 22:44:00 DEBUG "instance": "10.142.0.43:9100" 22:44:00 DEBUG "pod": "overseer-0-7c9b667974-9mr2s" 22:44:00 DEBUG "pod": "ds-idrepo-0" 22:44:00 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG "pod": "overseer-0-7c9b667974-9mr2s" 22:44:00 DEBUG "instance": "10.142.0.43:9100" 22:44:00 DEBUG 22:44:00 DEBUG 22:44:00 DEBUG }, 22:44:00 DEBUG 22:44:00 DEBUG }, 22:44:00 DEBUG } 22:44:00 DEBUG "value": [ 22:44:00 DEBUG }, 22:44:00 DEBUG "value": [ 22:44:00 DEBUG }, 22:44:00 DEBUG } 22:44:00 DEBUG "value": [ 22:44:00 DEBUG "pod": "am-555f4db595-kzxsp" 22:44:00 DEBUG 22:44:00 DEBUG }, 22:44:00 DEBUG }, 22:44:00 DEBUG }, 22:44:00 DEBUG }, 22:44:00 DEBUG }, 22:44:00 DEBUG "instance": "10.142.0.43:9100" 22:44:00 DEBUG "pod": "am-555f4db595-kzxsp" 22:44:00 DEBUG "value": [ 22:44:00 DEBUG "pod": "overseer-0-7c9b667974-9mr2s" 22:44:00 DEBUG }, 22:44:00 DEBUG }, 22:44:00 DEBUG "pod": "am-555f4db595-kzxsp" 22:44:00 DEBUG 22:44:00 DEBUG "value": [ 22:44:00 DEBUG "pod": "ds-idrepo-0" 22:44:00 DEBUG "value": [ 22:44:00 DEBUG ] 22:44:00 DEBUG 1713303840, 22:44:00 DEBUG "value": [ 22:44:00 DEBUG 1713303840, 22:44:00 DEBUG "value": [ 22:44:00 DEBUG ] 22:44:00 DEBUG 1713303840, 22:44:00 DEBUG }, 22:44:00 DEBUG "pod": "am-555f4db595-kzxsp" 22:44:00 DEBUG "value": [ 22:44:00 DEBUG "value": [ 22:44:00 DEBUG "value": [ 22:44:00 DEBUG "value": [ 22:44:00 DEBUG "value": [ 22:44:00 DEBUG }, 22:44:00 DEBUG }, 22:44:00 DEBUG 1713303840, 22:44:00 DEBUG }, 22:44:00 DEBUG "value": [ 22:44:00 DEBUG "value": [ 22:44:00 DEBUG }, 22:44:00 DEBUG "instance": "10.142.0.43:9100" 22:44:00 DEBUG 1713303840, 22:44:00 DEBUG }, 22:44:00 DEBUG 1713303840, 22:44:00 DEBUG } 22:44:00 DEBUG "0" 22:44:00 DEBUG 1713303840, 22:44:00 DEBUG "1995.7" 22:44:00 DEBUG 1713303840, 22:44:00 DEBUG } 22:44:00 DEBUG "0" 22:44:00 DEBUG "value": [ 22:44:00 DEBUG }, 22:44:00 DEBUG 1713303840, 22:44:00 DEBUG 1713303840, 22:44:00 DEBUG 1713303840, 22:44:00 DEBUG 1713303840, 22:44:00 DEBUG 1713303840, 22:44:00 DEBUG "value": [ 22:44:00 DEBUG "value": [ 22:44:00 DEBUG "2173.4" 22:44:00 DEBUG "value": [ 22:44:00 DEBUG 1713303840, 22:44:00 DEBUG 1713303840, 22:44:00 DEBUG "value": [ 22:44:00 DEBUG }, 22:44:00 DEBUG "3492597.5" 22:44:00 DEBUG "value": [ 22:44:00 DEBUG "0" 22:44:00 DEBUG } 22:44:00 DEBUG ] 22:44:00 DEBUG "734928896" 22:44:00 DEBUG ] 22:44:00 DEBUG "0" 22:44:00 DEBUG } 22:44:00 DEBUG ] 22:44:00 DEBUG 1713303840, 22:44:00 DEBUG "value": [ 22:44:00 DEBUG "19968779264" 22:44:00 DEBUG "0.0004999999999995453" 22:44:00 DEBUG "0" 22:44:00 DEBUG "0" 22:44:00 DEBUG "6.799798124834309" 22:44:00 DEBUG 1713303840, 22:44:00 DEBUG 1713303840, 22:44:00 DEBUG ] 22:44:00 DEBUG 1713303840, 22:44:00 DEBUG "16293402.252595428" 22:44:00 DEBUG "49506564.699999996" 22:44:00 DEBUG 1713303840, 22:44:00 DEBUG "value": [ 22:44:00 DEBUG ] 22:44:00 DEBUG 1713303840, 22:44:00 DEBUG ] 22:44:00 DEBUG } 22:44:00 DEBUG ] 22:44:00 DEBUG } 22:44:00 DEBUG ] 22:44:00 DEBUG } 22:44:00 DEBUG "2172.866666666667" 22:44:00 DEBUG 1713303840, 22:44:00 DEBUG ] 22:44:00 DEBUG ] 22:44:00 DEBUG ] 22:44:00 DEBUG ] 22:44:00 DEBUG ] 22:44:00 DEBUG "49497120.266666666" 22:44:00 DEBUG "0" 22:44:00 DEBUG } 22:44:00 DEBUG "2.6123380847078552" 22:44:00 DEBUG ] 22:44:00 DEBUG ] 22:44:00 DEBUG "2173.5" 22:44:00 DEBUG 1713303840, 22:44:00 DEBUG } 22:44:00 DEBUG "0" 22:44:00 DEBUG } 22:44:00 DEBUG ] 22:44:00 DEBUG } 22:44:00 DEBUG ] 22:44:00 DEBUG } 22:44:00 DEBUG ] 22:44:00 DEBUG ] 22:44:00 DEBUG "0.026" 22:44:00 DEBUG } 22:44:00 DEBUG } 22:44:00 DEBUG } 22:44:00 DEBUG } 22:44:00 DEBUG } 22:44:00 DEBUG ] 22:44:00 DEBUG ] 22:44:00 DEBUG ] 22:44:00 DEBUG ] 22:44:00 DEBUG } 22:44:00 DEBUG } 22:44:00 DEBUG ] 22:44:00 DEBUG "0" 22:44:00 DEBUG ] 22:44:00 DEBUG ] 22:44:00 DEBUG ] 22:44:00 DEBUG } 22:44:00 DEBUG } 22:44:00 DEBUG } 22:44:00 DEBUG ] 22:44:00 DEBUG } 22:44:00 DEBUG } 22:44:00 DEBUG ] 22:44:00 DEBUG ] 22:44:00 DEBUG ] 22:44:00 DEBUG ] 22:44:00 DEBUG ] 22:44:00 DEBUG ] 22:44:00 DEBUG } 22:44:00 DEBUG } 22:44:00 DEBUG } 22:44:00 DEBUG } 22:44:00 DEBUG ] 22:44:00 DEBUG ] 22:44:00 DEBUG } 22:44:00 DEBUG ] 22:44:00 DEBUG } 22:44:00 DEBUG } 22:44:00 DEBUG } 22:44:00 DEBUG ] 22:44:00 DEBUG } 22:44:00 DEBUG } 22:44:00 DEBUG } 22:44:00 DEBUG ] 22:44:00 DEBUG } 22:44:00 DEBUG } 22:44:00 DEBUG } 22:44:00 DEBUG } 22:44:00 DEBUG } 22:44:00 DEBUG } 22:44:00 DEBUG ] 22:44:00 DEBUG ] 22:44:00 DEBUG } 22:44:00 DEBUG ] 22:44:00 DEBUG } 22:44:00 DEBUG } 22:44:00 DEBUG ] 22:44:00 DEBUG } 22:44:00 DEBUG } 22:44:00 DEBUG ] 22:44:00 DEBUG } 22:44:00 DEBUG } 22:44:00 DEBUG } 22:44:00 DEBUG } 22:44:00 DEBUG ] 22:44:00 DEBUG } 22:44:00 DEBUG } 22:44:00 DEBUG } 22:44:00 DEBUG } 22:44:00 DEBUG } 22:44:00 DEBUG } 22:44:00 DEBUG } 22:44:00 DEBUG } 22:44:00 DEBUG } 22:44:00 DEBUG } 22:44:00 DEBUG } 22:44:00 DEBUG ] 22:44:00 DEBUG } 22:44:00 DEBUG } 22:44:00 DEBUG } 22:44:00 DEBUG } 22:44:00 DEBUG } 22:44:00 DEBUG } 22:44:00 DEBUG } 22:44:00 DEBUG } 22:44:00 DEBUG } 22:44:00 DEBUG } 22:44:00 DEBUG } 22:44:00 DEBUG } 22:44:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713303897" 22:44:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303897" 22:44:57 INFO [http_cmd]: http status code OK 22:44:57 ERROR [http_cmd]: ERROR 22:44:57 DEBUG --- status code --- 22:44:57 ERROR --- status code --- 22:44:57 ERROR http status code is 400 (expected [200, 401, 403, 404]) 22:44:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:44:57 ERROR --- http response --- 22:44:57 DEBUG --- http response --- 22:44:57 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 22:44:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 22:44:57 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 22:44:59 INFO 22:44:59 INFO [loop_until]: kubectl --namespace=xlou top pods 22:44:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:44:59 INFO [loop_until]: OK (rc = 0) 22:44:59 DEBUG --- stdout --- 22:44:59 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 9Mi am-555f4db595-4lbss 13171m 8670Mi am-555f4db595-fz8sn 13575m 8555Mi am-555f4db595-kzxsp 13485m 8662Mi ds-cts-0 19834m 12817Mi ds-cts-1 18319m 12754Mi ds-cts-2 17737m 12823Mi ds-idrepo-0 5916m 5028Mi ds-idrepo-1 5573m 2875Mi ds-idrepo-2 5485m 3160Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 4m 1207Mi idm-b85f9d978-rl76f 4m 1226Mi lodemon-bc756c647-cr7x6 2m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-7c9b667974-9mr2s 6615m 1378Mi 22:44:59 DEBUG --- stderr --- 22:44:59 DEBUG 22:45:00 INFO 22:45:00 INFO [loop_until]: kubectl --namespace=xlou top node 22:45:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:45:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303900" 22:45:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303900" 22:45:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713303900" 22:45:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303900" 22:45:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303900" 22:45:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303900" 22:45:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303900" 22:45:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303900" 22:45:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303900" 22:45:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303900" 22:45:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303900" 22:45:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303900" 22:45:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303900" 22:45:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713303900" 22:45:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303900" 22:45:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303900" 22:45:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303900" 22:45:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303900" 22:45:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303900" 22:45:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303900" 22:45:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713303900" 22:45:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303900" 22:45:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303900" 22:45:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713303900" 22:45:00 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303900" 22:45:00 INFO [http_cmd]: http status code OK 22:45:00 INFO [http_cmd]: http status code OK 22:45:00 INFO [http_cmd]: http status code OK 22:45:00 INFO [http_cmd]: http status code OK 22:45:00 INFO [http_cmd]: http status code OK 22:45:00 INFO [http_cmd]: http status code OK 22:45:00 INFO [http_cmd]: http status code OK 22:45:00 INFO [http_cmd]: http status code OK 22:45:00 INFO [http_cmd]: http status code OK 22:45:00 INFO [http_cmd]: http status code OK 22:45:00 INFO [http_cmd]: http status code OK 22:45:00 INFO [http_cmd]: http status code OK 22:45:00 DEBUG --- status code --- 22:45:00 INFO [http_cmd]: http status code OK 22:45:00 DEBUG --- status code --- 22:45:00 DEBUG --- status code --- 22:45:00 INFO [http_cmd]: http status code OK 22:45:00 DEBUG --- status code --- 22:45:00 INFO [http_cmd]: http status code OK 22:45:00 INFO [http_cmd]: http status code OK 22:45:00 DEBUG --- status code --- 22:45:00 INFO [loop_until]: OK (rc = 0) 22:45:00 INFO [http_cmd]: http status code OK 22:45:00 INFO [http_cmd]: http status code OK 22:45:00 INFO [http_cmd]: http status code OK 22:45:00 DEBUG --- status code --- 22:45:00 INFO [http_cmd]: http status code OK 22:45:00 INFO [http_cmd]: http status code OK 22:45:00 INFO [http_cmd]: http status code OK 22:45:00 DEBUG --- status code --- 22:45:00 DEBUG --- status code --- 22:45:00 DEBUG --- status code --- 22:45:00 DEBUG --- status code --- 22:45:00 DEBUG --- status code --- 22:45:00 DEBUG --- status code --- 22:45:00 INFO [http_cmd]: http status code OK 22:45:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:45:00 DEBUG --- status code --- 22:45:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:45:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:45:00 DEBUG --- status code --- 22:45:00 INFO [http_cmd]: http status code OK 22:45:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:45:00 DEBUG --- status code --- 22:45:00 DEBUG --- status code --- 22:45:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:45:00 DEBUG --- stdout --- 22:45:00 DEBUG --- status code --- 22:45:00 DEBUG --- status code --- 22:45:00 DEBUG --- status code --- 22:45:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:45:00 DEBUG --- status code --- 22:45:00 DEBUG --- status code --- 22:45:00 DEBUG --- status code --- 22:45:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:45:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:45:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:45:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:45:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:45:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:45:00 DEBUG --- status code --- 22:45:00 DEBUG --- http response --- 22:45:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:45:00 DEBUG --- http response --- 22:45:00 DEBUG --- http response --- 22:45:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:45:00 DEBUG --- status code --- 22:45:00 DEBUG --- http response --- 22:45:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:45:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:45:00 DEBUG --- http response --- 22:45:00 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 5090Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 18201m 60% 15423Mi 13% gke-xlou-cdm-default-pool-1b0769c8-nhlf 13250m 44% 10779Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 82m 0% 2655Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 5537m 18% 6503Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 20960m 70% 17208Mi 15% gke-xlou-cdm-default-pool-1b0769c8-xnmp 60m 0% 3151Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 13495m 45% 10448Mi 9% gke-xlou-cdm-ds-0f601751-hmm7 6097m 20% 5615Mi 5% gke-xlou-cdm-ds-0f601751-j488 18178m 60% 14799Mi 13% gke-xlou-cdm-ds-0f601751-mc5d 11633m 38% 3314Mi 2% gke-xlou-cdm-ds-0f601751-nj6q 13727m 45% 10530Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 6058m 20% 7355Mi 6% gke-xlou-cdm-frontend-a8771548-i82n 6916m 43% 6814Mi 11% 22:45:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:45:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:45:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:45:00 DEBUG --- http response --- 22:45:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:45:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:45:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:45:00 INFO [http_cmd]: http status code OK 22:45:00 DEBUG --- http response --- 22:45:00 DEBUG --- http response --- 22:45:00 DEBUG --- http response --- 22:45:00 DEBUG --- http response --- 22:45:00 DEBUG --- http response --- 22:45:00 DEBUG --- http response --- 22:45:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:45:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:45:00 DEBUG --- http response --- 22:45:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:45:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:45:00 DEBUG --- http response --- 22:45:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:45:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:45:00 DEBUG --- http response --- 22:45:00 DEBUG --- http response --- 22:45:00 DEBUG --- stderr --- 22:45:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:45:00 DEBUG --- http response --- 22:45:00 DEBUG --- http response --- 22:45:00 DEBUG --- http response --- 22:45:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:45:00 DEBUG --- http response --- 22:45:00 DEBUG --- http response --- 22:45:00 DEBUG --- http response --- 22:45:00 DEBUG --- status code --- 22:45:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:45:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:45:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:45:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:45:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:45:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:45:00 DEBUG --- http response --- 22:45:00 DEBUG ----- output ----- 22:45:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:45:00 DEBUG ----- output ----- 22:45:00 DEBUG ----- output ----- 22:45:00 DEBUG --- http response --- 22:45:00 DEBUG ----- output ----- 22:45:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:45:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:45:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:45:00 DEBUG 22:45:00 DEBUG ----- output ----- 22:45:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:45:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:45:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:45:00 DEBUG ----- output ----- 22:45:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:45:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:45:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:45:00 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:45:00 DEBUG ----- output ----- 22:45:00 DEBUG ----- output ----- 22:45:00 DEBUG ----- output ----- 22:45:00 DEBUG ----- output ----- 22:45:00 DEBUG ----- output ----- 22:45:00 DEBUG ----- output ----- 22:45:00 DEBUG { 22:45:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:45:00 DEBUG ----- output ----- 22:45:00 DEBUG { 22:45:00 DEBUG { 22:45:00 DEBUG { 22:45:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:45:00 DEBUG ----- output ----- 22:45:00 DEBUG ----- output ----- 22:45:00 DEBUG ----- output ----- 22:45:00 DEBUG { 22:45:00 DEBUG ----- output ----- 22:45:00 DEBUG ----- output ----- 22:45:00 DEBUG ----- output ----- 22:45:00 DEBUG { 22:45:00 DEBUG ----- output ----- 22:45:00 DEBUG ----- output ----- 22:45:00 DEBUG ----- output ----- 22:45:00 DEBUG --- http response --- 22:45:00 DEBUG { 22:45:00 DEBUG "status": "success", 22:45:00 DEBUG { 22:45:00 DEBUG { 22:45:00 DEBUG { 22:45:00 DEBUG { 22:45:00 DEBUG "status": "success", 22:45:00 DEBUG ----- output ----- 22:45:00 DEBUG { 22:45:00 DEBUG "status": "success", 22:45:00 DEBUG "status": "success", 22:45:00 DEBUG "status": "success", 22:45:00 DEBUG ----- output ----- 22:45:00 DEBUG { 22:45:00 DEBUG { 22:45:00 DEBUG { 22:45:00 DEBUG "status": "success", 22:45:00 DEBUG { 22:45:00 DEBUG { 22:45:00 DEBUG { 22:45:00 DEBUG "status": "success", 22:45:00 DEBUG { 22:45:00 DEBUG { 22:45:00 DEBUG { 22:45:00 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:45:00 DEBUG "data": { 22:45:00 DEBUG "status": "success", 22:45:00 DEBUG { 22:45:00 DEBUG "status": "success", 22:45:00 DEBUG "status": "success", 22:45:00 DEBUG "status": "success", 22:45:00 DEBUG "data": { 22:45:00 DEBUG { 22:45:00 DEBUG "status": "success", 22:45:00 DEBUG "data": { 22:45:00 DEBUG "data": { 22:45:00 DEBUG "data": { 22:45:00 DEBUG { 22:45:00 DEBUG "status": "success", 22:45:00 DEBUG "status": "success", 22:45:00 DEBUG "status": "success", 22:45:00 DEBUG "data": { 22:45:00 DEBUG "status": "success", 22:45:00 DEBUG "status": "success", 22:45:00 DEBUG "status": "success", 22:45:00 DEBUG "data": { 22:45:00 DEBUG "status": "success", 22:45:00 DEBUG "status": "success", 22:45:00 DEBUG "status": "success", 22:45:00 DEBUG ----- output ----- 22:45:00 DEBUG "resultType": "vector", 22:45:00 DEBUG "data": { 22:45:00 DEBUG "status": "success", 22:45:00 DEBUG "data": { 22:45:00 DEBUG "data": { 22:45:00 DEBUG "data": { 22:45:00 DEBUG "resultType": "vector", 22:45:00 DEBUG "status": "success", 22:45:00 DEBUG "data": { 22:45:00 DEBUG "resultType": "vector", 22:45:00 DEBUG "resultType": "vector", 22:45:00 DEBUG "resultType": "vector", 22:45:00 DEBUG "status": "success", 22:45:00 DEBUG "data": { 22:45:00 DEBUG "data": { 22:45:00 DEBUG "data": { 22:45:00 DEBUG "resultType": "vector", 22:45:00 DEBUG "data": { 22:45:00 DEBUG "data": { 22:45:00 DEBUG "data": { 22:45:00 DEBUG "resultType": "vector", 22:45:00 DEBUG "data": { 22:45:00 DEBUG "data": { 22:45:00 DEBUG "data": { 22:45:00 DEBUG { 22:45:00 DEBUG "result": [ 22:45:00 DEBUG "resultType": "vector", 22:45:00 DEBUG "data": { 22:45:00 DEBUG "resultType": "vector", 22:45:00 DEBUG "resultType": "vector", 22:45:00 DEBUG "resultType": "vector", 22:45:00 DEBUG "result": [ 22:45:00 DEBUG "data": { 22:45:00 DEBUG "resultType": "vector", 22:45:00 DEBUG "result": [ 22:45:00 DEBUG "result": [ 22:45:00 DEBUG "result": [ 22:45:00 DEBUG "data": { 22:45:00 DEBUG "resultType": "vector", 22:45:00 DEBUG "resultType": "vector", 22:45:00 DEBUG "resultType": "vector", 22:45:00 DEBUG "result": [ 22:45:00 DEBUG "resultType": "vector", 22:45:00 DEBUG "resultType": "vector", 22:45:00 DEBUG "resultType": "vector", 22:45:00 DEBUG "result": [ 22:45:00 DEBUG "resultType": "vector", 22:45:00 DEBUG "resultType": "vector", 22:45:00 DEBUG "resultType": "vector", 22:45:00 DEBUG "status": "success", 22:45:00 DEBUG { 22:45:00 DEBUG "result": [ 22:45:00 DEBUG "resultType": "vector", 22:45:00 DEBUG "result": [ 22:45:00 DEBUG "result": [ 22:45:00 DEBUG "result": [ 22:45:00 DEBUG { 22:45:00 DEBUG "resultType": "vector", 22:45:00 DEBUG "result": [ 22:45:00 DEBUG { 22:45:00 DEBUG { 22:45:00 DEBUG { 22:45:00 DEBUG "resultType": "vector", 22:45:00 DEBUG "result": [ 22:45:00 DEBUG "result": [ 22:45:00 DEBUG "result": [ 22:45:00 DEBUG { 22:45:00 DEBUG "result": [ 22:45:00 DEBUG "result": [ 22:45:00 DEBUG "result": [ 22:45:00 DEBUG { 22:45:00 DEBUG "result": [ 22:45:00 DEBUG "result": [ 22:45:00 DEBUG "result": [ 22:45:00 DEBUG "data": { 22:45:00 DEBUG "metric": { 22:45:00 DEBUG { 22:45:00 DEBUG "result": [ 22:45:00 DEBUG { 22:45:00 DEBUG { 22:45:00 DEBUG { 22:45:00 DEBUG "metric": { 22:45:00 DEBUG "result": [ 22:45:00 DEBUG { 22:45:00 DEBUG "metric": { 22:45:00 DEBUG "metric": { 22:45:00 DEBUG "metric": { 22:45:00 DEBUG "result": [ 22:45:00 DEBUG { 22:45:00 DEBUG { 22:45:00 DEBUG { 22:45:00 DEBUG "metric": { 22:45:00 DEBUG { 22:45:00 DEBUG { 22:45:00 DEBUG { 22:45:00 DEBUG "metric": { 22:45:00 DEBUG { 22:45:00 DEBUG { 22:45:00 DEBUG { 22:45:00 DEBUG "resultType": "vector", 22:45:00 DEBUG "pod": "ds-idrepo-2" 22:45:00 DEBUG "metric": { 22:45:00 DEBUG { 22:45:00 DEBUG "metric": { 22:45:00 DEBUG "metric": { 22:45:00 DEBUG "metric": { 22:45:00 DEBUG "pod": "ds-idrepo-2" 22:45:00 DEBUG { 22:45:00 DEBUG "metric": { 22:45:00 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:45:00 DEBUG "pod": "am-555f4db595-fz8sn" 22:45:00 DEBUG "pod": "am-555f4db595-fz8sn" 22:45:00 DEBUG { 22:45:00 DEBUG "metric": { 22:45:00 DEBUG "metric": { 22:45:00 DEBUG "metric": { 22:45:00 DEBUG "pod": "ds-idrepo-2" 22:45:00 DEBUG "metric": { 22:45:00 DEBUG "metric": { 22:45:00 DEBUG "metric": { 22:45:00 DEBUG "pod": "am-555f4db595-fz8sn" 22:45:00 DEBUG "metric": { 22:45:00 DEBUG "metric": { 22:45:00 DEBUG "metric": { 22:45:00 DEBUG "result": [ 22:45:00 DEBUG }, 22:45:00 DEBUG "instance": "10.142.0.107:9100" 22:45:00 DEBUG "metric": { 22:45:00 DEBUG "pod": "ds-idrepo-2" 22:45:00 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:45:00 DEBUG "pod": "ds-idrepo-2" 22:45:00 DEBUG }, 22:45:00 DEBUG "metric": { 22:45:00 DEBUG "instance": "10.142.0.107:9100" 22:45:00 DEBUG }, 22:45:00 DEBUG }, 22:45:00 DEBUG }, 22:45:00 DEBUG "metric": { 22:45:00 DEBUG "pod": "am-555f4db595-fz8sn" 22:45:00 DEBUG "pod": "am-555f4db595-fz8sn" 22:45:00 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:45:00 DEBUG }, 22:45:00 DEBUG "pod": "ds-idrepo-2" 22:45:00 DEBUG "instance": "10.142.0.107:9100" 22:45:00 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:45:00 DEBUG }, 22:45:00 DEBUG "instance": "10.142.0.107:9100" 22:45:00 DEBUG "pod": "am-555f4db595-fz8sn" 22:45:00 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:45:00 DEBUG { 22:45:00 DEBUG "value": [ 22:45:00 DEBUG }, 22:45:00 DEBUG "pod": "ds-idrepo-2" 22:45:00 DEBUG }, 22:45:00 DEBUG }, 22:45:00 DEBUG }, 22:45:00 DEBUG "value": [ 22:45:00 DEBUG "pod": "am-555f4db595-fz8sn" 22:45:00 DEBUG "value": [ 22:45:00 DEBUG "value": [ 22:45:00 DEBUG "value": [ 22:45:00 DEBUG "value": [ 22:45:00 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:45:00 DEBUG }, 22:45:00 DEBUG }, 22:45:00 DEBUG }, 22:45:00 DEBUG "value": [ 22:45:00 DEBUG }, 22:45:00 DEBUG }, 22:45:00 DEBUG }, 22:45:00 DEBUG "value": [ 22:45:00 DEBUG }, 22:45:00 DEBUG }, 22:45:00 DEBUG }, 22:45:00 DEBUG "metric": { 22:45:00 DEBUG 22:45:00 DEBUG "value": [ 22:45:00 DEBUG }, 22:45:00 DEBUG "value": [ 22:45:00 DEBUG "value": [ 22:45:00 DEBUG 22:45:00 DEBUG }, 22:45:00 DEBUG }, 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG }, 22:45:00 DEBUG "value": [ 22:45:00 DEBUG "value": [ 22:45:00 DEBUG "value": [ 22:45:00 DEBUG 22:45:00 DEBUG "value": [ 22:45:00 DEBUG "value": [ 22:45:00 DEBUG "value": [ 22:45:00 DEBUG 22:45:00 DEBUG "value": [ 22:45:00 DEBUG "value": [ 22:45:00 DEBUG "value": [ 22:45:00 DEBUG "instance": "10.142.0.107:9100" 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG "value": [ 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG "value": [ 22:45:00 DEBUG "value": [ 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG "value": [ 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG }, 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG "value": [ 22:45:00 DEBUG [...] 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG [...] 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG [...] 22:45:00 DEBUG [...] 22:45:00 DEBUG [...] 22:45:00 DEBUG [...] 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG [...] 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG [...] 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG [...] 22:45:00 DEBUG 22:45:00 DEBUG [...] 22:45:00 DEBUG [...] 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG [...] 22:45:00 DEBUG [...] 22:45:00 DEBUG [...] 22:45:00 DEBUG 22:45:00 DEBUG [...] 22:45:00 DEBUG [...] 22:45:00 DEBUG [...] 22:45:00 DEBUG 22:45:00 DEBUG [...] 22:45:00 DEBUG [...] 22:45:00 DEBUG [...] 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG [...] 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG [...] 22:45:00 DEBUG [...] 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG [...] 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG [...] 22:45:00 DEBUG "pod": "ds-idrepo-0" 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG "pod": "idm-b85f9d978-j5pbm" 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG "pod": "ds-idrepo-0" 22:45:00 DEBUG "pod": "idm-b85f9d978-j5pbm" 22:45:00 DEBUG "pod": "am-555f4db595-kzxsp" 22:45:00 DEBUG "pod": "am-555f4db595-kzxsp" 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG "pod": "ds-cts-2" 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG "pod": "am-555f4db595-kzxsp" 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG 22:45:00 DEBUG }, 22:45:00 DEBUG "instance": "10.142.0.43:9100" 22:45:00 DEBUG 22:45:00 DEBUG "pod": "ds-idrepo-0" 22:45:00 DEBUG "pod": "ds-idrepo-0" 22:45:00 DEBUG }, 22:45:00 DEBUG 22:45:00 DEBUG "instance": "10.142.0.43:9100" 22:45:00 DEBUG }, 22:45:00 DEBUG }, 22:45:00 DEBUG }, 22:45:00 DEBUG }, 22:45:00 DEBUG 22:45:00 DEBUG "pod": "am-555f4db595-kzxsp" 22:45:00 DEBUG "pod": "am-555f4db595-kzxsp" 22:45:00 DEBUG "pod": "idm-b85f9d978-j5pbm" 22:45:00 DEBUG }, 22:45:00 DEBUG "pod": "ds-idrepo-0" 22:45:00 DEBUG "instance": "10.142.0.43:9100" 22:45:00 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:45:00 DEBUG }, 22:45:00 DEBUG "instance": "10.142.0.43:9100" 22:45:00 DEBUG "pod": "am-555f4db595-kzxsp" 22:45:00 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:45:00 DEBUG 22:45:00 DEBUG "value": [ 22:45:00 DEBUG }, 22:45:00 DEBUG "pod": "ds-idrepo-0" 22:45:00 DEBUG }, 22:45:00 DEBUG }, 22:45:00 DEBUG "value": [ 22:45:00 DEBUG 22:45:00 DEBUG }, 22:45:00 DEBUG "value": [ 22:45:00 DEBUG "value": [ 22:45:00 DEBUG "value": [ 22:45:00 DEBUG "value": [ 22:45:00 DEBUG "pod": "idm-b85f9d978-j5pbm" 22:45:00 DEBUG }, 22:45:00 DEBUG }, 22:45:00 DEBUG }, 22:45:00 DEBUG "value": [ 22:45:00 DEBUG }, 22:45:00 DEBUG }, 22:45:00 DEBUG }, 22:45:00 DEBUG "value": [ 22:45:00 DEBUG }, 22:45:00 DEBUG }, 22:45:00 DEBUG }, 22:45:00 DEBUG 22:45:00 DEBUG 1713303900, 22:45:00 DEBUG "value": [ 22:45:00 DEBUG }, 22:45:00 DEBUG "value": [ 22:45:00 DEBUG "value": [ 22:45:00 DEBUG 1713303900, 22:45:00 DEBUG "pod": "am-555f4db595-kzxsp" 22:45:00 DEBUG "value": [ 22:45:00 DEBUG 1713303900, 22:45:00 DEBUG 1713303900, 22:45:00 DEBUG 1713303900, 22:45:00 DEBUG 1713303900, 22:45:00 DEBUG }, 22:45:00 DEBUG "value": [ 22:45:00 DEBUG "value": [ 22:45:00 DEBUG "value": [ 22:45:00 DEBUG 1713303900, 22:45:00 DEBUG "value": [ 22:45:00 DEBUG "value": [ 22:45:00 DEBUG "value": [ 22:45:00 DEBUG 1713303900, 22:45:00 DEBUG "value": [ 22:45:00 DEBUG "value": [ 22:45:00 DEBUG "value": [ 22:45:00 DEBUG "instance": "10.142.0.43:9100" 22:45:00 DEBUG "0" 22:45:00 DEBUG 1713303900, 22:45:00 DEBUG "value": [ 22:45:00 DEBUG 1713303900, 22:45:00 DEBUG 1713303900, 22:45:00 DEBUG "0" 22:45:00 DEBUG }, 22:45:00 DEBUG 1713303900, 22:45:00 DEBUG "0" 22:45:00 DEBUG "538.5753262548898" 22:45:00 DEBUG "1992.4666666666667" 22:45:00 DEBUG "3502413.5" 22:45:00 DEBUG ] 22:45:00 DEBUG 1713303900, 22:45:00 DEBUG 1713303900, 22:45:00 DEBUG 1713303900, 22:45:00 DEBUG "0" 22:45:00 DEBUG 1713303900, 22:45:00 DEBUG 1713303900, 22:45:00 DEBUG 1713303900, 22:45:00 DEBUG "2169.7666666666664" 22:45:00 DEBUG 1713303900, 22:45:00 DEBUG 1713303900, 22:45:00 DEBUG 1713303900, 22:45:00 DEBUG }, 22:45:00 DEBUG ] 22:45:00 DEBUG "19969338368" 22:45:00 DEBUG 1713303900, 22:45:00 DEBUG "0" 22:45:00 DEBUG "0" 22:45:00 DEBUG ] 22:45:00 DEBUG "value": [ 22:45:00 DEBUG "0.0005000000000004927" 22:45:00 DEBUG ] 22:45:00 DEBUG ] 22:45:00 DEBUG ] 22:45:00 DEBUG "value": [ 22:45:00 DEBUG } 22:45:00 DEBUG "0" 22:45:00 DEBUG "2170.1" 22:45:00 DEBUG "41814584.39893495" 22:45:00 DEBUG ] 22:45:00 DEBUG "0" 22:45:00 DEBUG "49355476.06666666" 22:45:00 DEBUG "6.889016631339636" 22:45:00 DEBUG ] 22:45:00 DEBUG "49346768.53333333" 22:45:00 DEBUG "2173.4" 22:45:00 DEBUG "1543274496" 22:45:00 DEBUG "value": [ 22:45:00 DEBUG } 22:45:00 DEBUG ] 22:45:00 DEBUG "0" 22:45:00 DEBUG ] 22:45:00 DEBUG ] 22:45:00 DEBUG } 22:45:00 DEBUG 1713303900, 22:45:00 DEBUG ] 22:45:00 DEBUG } 22:45:00 DEBUG } 22:45:00 DEBUG } 22:45:00 DEBUG 1713303900, 22:45:00 DEBUG ] 22:45:00 DEBUG ] 22:45:00 DEBUG ] 22:45:00 DEBUG } 22:45:00 DEBUG } 22:45:00 DEBUG ] 22:45:00 DEBUG ] 22:45:00 DEBUG ] 22:45:00 DEBUG } 22:45:00 DEBUG ] 22:45:00 DEBUG ] 22:45:00 DEBUG ] 22:45:00 DEBUG 1713303900, 22:45:00 DEBUG ] 22:45:00 DEBUG } 22:45:00 DEBUG ] 22:45:00 DEBUG } 22:45:00 DEBUG } 22:45:00 DEBUG ] 22:45:00 DEBUG "0.0265" 22:45:00 DEBUG } 22:45:00 DEBUG ] 22:45:00 DEBUG ] 22:45:00 DEBUG ] 22:45:00 DEBUG "0.003051116005389884" 22:45:00 DEBUG } 22:45:00 DEBUG } 22:45:00 DEBUG ] 22:45:00 DEBUG ] 22:45:00 DEBUG ] 22:45:00 DEBUG } 22:45:00 DEBUG } 22:45:00 DEBUG } 22:45:00 DEBUG ] 22:45:00 DEBUG } 22:45:00 DEBUG } 22:45:00 DEBUG } 22:45:00 DEBUG "0.0006666666666666672" 22:45:00 DEBUG } 22:45:00 DEBUG ] 22:45:00 DEBUG } 22:45:00 DEBUG ] 22:45:00 DEBUG ] 22:45:00 DEBUG } 22:45:00 DEBUG ] 22:45:00 DEBUG ] 22:45:00 DEBUG } 22:45:00 DEBUG } 22:45:00 DEBUG } 22:45:00 DEBUG ] 22:45:00 DEBUG } 22:45:00 DEBUG ] 22:45:00 DEBUG } 22:45:00 DEBUG } 22:45:00 DEBUG } 22:45:00 DEBUG ] 22:45:00 DEBUG ] 22:45:00 DEBUG ] 22:45:00 DEBUG } 22:45:00 DEBUG ] 22:45:00 DEBUG ] 22:45:00 DEBUG ] 22:45:00 DEBUG ] 22:45:00 DEBUG } 22:45:00 DEBUG } 22:45:00 DEBUG ] 22:45:00 DEBUG } 22:45:00 DEBUG } 22:45:00 DEBUG } 22:45:00 DEBUG } 22:45:00 DEBUG } 22:45:00 DEBUG } 22:45:00 DEBUG } 22:45:00 DEBUG } 22:45:00 DEBUG } 22:45:00 DEBUG } 22:45:00 DEBUG } 22:45:00 DEBUG } 22:45:00 DEBUG } 22:45:00 DEBUG } 22:45:00 DEBUG } 22:45:00 DEBUG } 22:45:00 DEBUG } 22:45:00 DEBUG } 22:45:00 DEBUG } 22:45:00 DEBUG } 22:45:00 DEBUG } 22:45:00 DEBUG } 22:45:00 DEBUG } 22:45:00 DEBUG } 22:45:00 DEBUG } 22:45:00 DEBUG ] 22:45:00 DEBUG } 22:45:00 DEBUG ] 22:45:00 DEBUG ] 22:45:00 DEBUG } 22:45:00 DEBUG } 22:45:00 DEBUG } 22:45:00 DEBUG } 22:45:00 DEBUG } 22:45:00 DEBUG } 22:45:00 DEBUG ] 22:45:00 DEBUG } 22:45:00 DEBUG } 22:45:00 DEBUG } 22:45:00 DEBUG } 22:45:00 DEBUG } 22:45:00 DEBUG } 22:45:00 DEBUG } 22:45:00 DEBUG } 22:45:00 DEBUG } 22:45:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713303957" 22:45:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303957" 22:45:57 ERROR [http_cmd]: ERROR 22:45:57 INFO [http_cmd]: http status code OK 22:45:57 ERROR --- status code --- 22:45:57 DEBUG --- status code --- 22:45:57 ERROR http status code is 400 (expected [200, 401, 403, 404]) 22:45:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:45:57 ERROR --- http response --- 22:45:57 DEBUG --- http response --- 22:45:57 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 22:45:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 22:45:57 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 22:45:59 INFO 22:45:59 INFO [loop_until]: kubectl --namespace=xlou top pods 22:45:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:45:59 INFO [loop_until]: OK (rc = 0) 22:45:59 DEBUG --- stdout --- 22:45:59 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 9Mi am-555f4db595-4lbss 12911m 8670Mi am-555f4db595-fz8sn 13002m 8555Mi am-555f4db595-kzxsp 13270m 8665Mi ds-cts-0 19584m 13002Mi ds-cts-1 18600m 13083Mi ds-cts-2 18381m 13015Mi ds-idrepo-0 5801m 5028Mi ds-idrepo-1 5450m 2875Mi ds-idrepo-2 5472m 3163Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 4m 1208Mi idm-b85f9d978-rl76f 3m 1226Mi lodemon-bc756c647-cr7x6 1m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-7c9b667974-9mr2s 6485m 1378Mi 22:45:59 DEBUG --- stderr --- 22:45:59 DEBUG 22:46:00 INFO 22:46:00 INFO [loop_until]: kubectl --namespace=xlou top node 22:46:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:46:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303961" 22:46:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303961" 22:46:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303961" 22:46:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303961" 22:46:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303961" 22:46:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303961" 22:46:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303961" 22:46:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303961" 22:46:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713303961" 22:46:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303961" 22:46:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303961" 22:46:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303961" 22:46:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303961" 22:46:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713303961" 22:46:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303961" 22:46:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303961" 22:46:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303961" 22:46:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303961" 22:46:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303961" 22:46:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713303961" 22:46:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303961" 22:46:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713303961" 22:46:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303961" 22:46:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713303961" 22:46:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713303961" 22:46:01 INFO [http_cmd]: http status code OK 22:46:01 INFO [http_cmd]: http status code OK 22:46:01 INFO [http_cmd]: http status code OK 22:46:01 INFO [loop_until]: OK (rc = 0) 22:46:01 INFO [http_cmd]: http status code OK 22:46:01 INFO [http_cmd]: http status code OK 22:46:01 INFO [http_cmd]: http status code OK 22:46:01 INFO [http_cmd]: http status code OK 22:46:01 INFO [http_cmd]: http status code OK 22:46:01 INFO [http_cmd]: http status code OK 22:46:01 INFO [http_cmd]: http status code OK 22:46:01 INFO [http_cmd]: http status code OK 22:46:01 INFO [http_cmd]: http status code OK 22:46:01 INFO [http_cmd]: http status code OK 22:46:01 INFO [http_cmd]: http status code OK 22:46:01 INFO [http_cmd]: http status code OK 22:46:01 DEBUG --- status code --- 22:46:01 INFO [http_cmd]: http status code OK 22:46:01 INFO [http_cmd]: http status code OK 22:46:01 INFO [http_cmd]: http status code OK 22:46:01 DEBUG --- status code --- 22:46:01 INFO [http_cmd]: http status code OK 22:46:01 DEBUG --- status code --- 22:46:01 DEBUG --- stdout --- 22:46:01 DEBUG --- status code --- 22:46:01 DEBUG --- status code --- 22:46:01 DEBUG --- status code --- 22:46:01 DEBUG --- status code --- 22:46:01 DEBUG --- status code --- 22:46:01 DEBUG --- status code --- 22:46:01 DEBUG --- status code --- 22:46:01 DEBUG --- status code --- 22:46:01 DEBUG --- status code --- 22:46:01 DEBUG --- status code --- 22:46:01 DEBUG --- status code --- 22:46:01 DEBUG --- status code --- 22:46:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:46:01 DEBUG --- status code --- 22:46:01 DEBUG --- status code --- 22:46:01 INFO [http_cmd]: http status code OK 22:46:01 DEBUG --- status code --- 22:46:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:46:01 DEBUG --- status code --- 22:46:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:46:01 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 63m 0% 5096Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 18379m 61% 15640Mi 14% gke-xlou-cdm-default-pool-1b0769c8-nhlf 13147m 44% 10783Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 78m 0% 2653Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 5561m 18% 6516Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 21040m 70% 17375Mi 15% gke-xlou-cdm-default-pool-1b0769c8-xnmp 59m 0% 3145Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 13185m 44% 10442Mi 9% gke-xlou-cdm-ds-0f601751-hmm7 6120m 20% 5624Mi 5% gke-xlou-cdm-ds-0f601751-j488 18272m 61% 14979Mi 13% gke-xlou-cdm-ds-0f601751-mc5d 11555m 38% 3301Mi 2% gke-xlou-cdm-ds-0f601751-nj6q 13703m 45% 10547Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 6075m 20% 7366Mi 6% gke-xlou-cdm-frontend-a8771548-i82n 6912m 43% 6816Mi 11% 22:46:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:46:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:46:01 INFO [http_cmd]: http status code OK 22:46:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:46:01 INFO [http_cmd]: http status code OK 22:46:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:46:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:46:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:46:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:46:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:46:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:46:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:46:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:46:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:46:01 DEBUG --- http response --- 22:46:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:46:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:46:01 DEBUG --- status code --- 22:46:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:46:01 INFO [http_cmd]: http status code OK 22:46:01 DEBUG --- http response --- 22:46:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:46:01 DEBUG --- http response --- 22:46:01 INFO [http_cmd]: http status code OK 22:46:01 DEBUG --- stderr --- 22:46:01 DEBUG --- http response --- 22:46:01 DEBUG --- http response --- 22:46:01 INFO [http_cmd]: http status code OK 22:46:01 DEBUG --- status code --- 22:46:01 DEBUG --- http response --- 22:46:01 DEBUG --- status code --- 22:46:01 DEBUG --- http response --- 22:46:01 DEBUG --- http response --- 22:46:01 DEBUG --- http response --- 22:46:01 DEBUG --- http response --- 22:46:01 DEBUG --- http response --- 22:46:01 DEBUG --- http response --- 22:46:01 DEBUG --- http response --- 22:46:01 DEBUG --- http response --- 22:46:01 DEBUG --- http response --- 22:46:01 DEBUG --- http response --- 22:46:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:46:01 DEBUG --- http response --- 22:46:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:46:01 DEBUG --- http response --- 22:46:01 DEBUG --- status code --- 22:46:01 DEBUG --- http response --- 22:46:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:46:01 DEBUG --- status code --- 22:46:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:46:01 DEBUG 22:46:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:46:01 DEBUG --- status code --- 22:46:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:46:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:46:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:46:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:46:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:46:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:46:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:46:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:46:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:46:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:46:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:46:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:46:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:46:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:46:01 DEBUG ----- output ----- 22:46:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:46:01 DEBUG --- http response --- 22:46:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:46:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:46:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:46:01 DEBUG ----- output ----- 22:46:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:46:01 DEBUG ----- output ----- 22:46:01 DEBUG ----- output ----- 22:46:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:46:01 DEBUG ----- output ----- 22:46:01 DEBUG --- http response --- 22:46:01 DEBUG --- http response --- 22:46:01 DEBUG ----- output ----- 22:46:01 DEBUG ----- output ----- 22:46:01 DEBUG ----- output ----- 22:46:01 DEBUG ----- output ----- 22:46:01 DEBUG ----- output ----- 22:46:01 DEBUG ----- output ----- 22:46:01 DEBUG ----- output ----- 22:46:01 DEBUG ----- output ----- 22:46:01 DEBUG ----- output ----- 22:46:01 DEBUG ----- output ----- 22:46:01 DEBUG ----- output ----- 22:46:01 DEBUG { 22:46:01 DEBUG ----- output ----- 22:46:01 DEBUG ----- output ----- 22:46:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:46:01 DEBUG --- http response --- 22:46:01 DEBUG ----- output ----- 22:46:01 DEBUG { 22:46:01 DEBUG --- http response --- 22:46:01 DEBUG { 22:46:01 DEBUG { 22:46:01 DEBUG --- http response --- 22:46:01 DEBUG { 22:46:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 22:46:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:46:01 DEBUG { 22:46:01 DEBUG { 22:46:01 DEBUG { 22:46:01 DEBUG { 22:46:01 DEBUG { 22:46:01 DEBUG { 22:46:01 DEBUG { 22:46:01 DEBUG { 22:46:01 DEBUG { 22:46:01 DEBUG { 22:46:01 DEBUG { 22:46:01 DEBUG "status": "success", 22:46:01 DEBUG { 22:46:01 DEBUG { 22:46:01 DEBUG ----- output ----- 22:46:01 DEBUG { 22:46:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:46:01 DEBUG ----- output ----- 22:46:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:46:01 DEBUG "status": "success", 22:46:01 DEBUG "status": "success", 22:46:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 22:46:01 DEBUG "status": "success", 22:46:01 DEBUG ----- output ----- 22:46:01 DEBUG ----- output ----- 22:46:01 DEBUG "status": "success", 22:46:01 DEBUG "status": "success", 22:46:01 DEBUG "status": "success", 22:46:01 DEBUG "status": "success", 22:46:01 DEBUG "status": "success", 22:46:01 DEBUG "status": "success", 22:46:01 DEBUG "status": "success", 22:46:01 DEBUG "status": "success", 22:46:01 DEBUG "status": "success", 22:46:01 DEBUG "status": "success", 22:46:01 DEBUG "status": "success", 22:46:01 DEBUG "data": { 22:46:01 DEBUG "status": "success", 22:46:01 DEBUG "status": "success", 22:46:01 DEBUG { 22:46:01 DEBUG "status": "success", 22:46:01 DEBUG "status": "success", 22:46:01 DEBUG { 22:46:01 DEBUG ----- output ----- 22:46:01 DEBUG "data": { 22:46:01 DEBUG "data": { 22:46:01 DEBUG ----- output ----- 22:46:01 DEBUG "data": { 22:46:01 DEBUG { 22:46:01 DEBUG { 22:46:01 DEBUG "data": { 22:46:01 DEBUG "data": { 22:46:01 DEBUG "data": { 22:46:01 DEBUG "data": { 22:46:01 DEBUG "data": { 22:46:01 DEBUG "data": { 22:46:01 DEBUG "data": { 22:46:01 DEBUG "data": { 22:46:01 DEBUG "resultType": "vector", 22:46:01 DEBUG "data": { 22:46:01 DEBUG "data": { 22:46:01 DEBUG "resultType": "vector", 22:46:01 DEBUG "data": { 22:46:01 DEBUG "data": { 22:46:01 DEBUG "status": "success", 22:46:01 DEBUG "data": { 22:46:01 DEBUG "data": { 22:46:01 DEBUG "status": "success", 22:46:01 DEBUG { 22:46:01 DEBUG "resultType": "vector", 22:46:01 DEBUG "resultType": "vector", 22:46:01 DEBUG { 22:46:01 DEBUG "resultType": "vector", 22:46:01 DEBUG "status": "success", 22:46:01 DEBUG "status": "success", 22:46:01 DEBUG "resultType": "vector", 22:46:01 DEBUG "resultType": "vector", 22:46:01 DEBUG "resultType": "vector", 22:46:01 DEBUG "resultType": "vector", 22:46:01 DEBUG "resultType": "vector", 22:46:01 DEBUG "resultType": "vector", 22:46:01 DEBUG "resultType": "vector", 22:46:01 DEBUG "data": { 22:46:01 DEBUG "result": [ 22:46:01 DEBUG "resultType": "vector", 22:46:01 DEBUG "resultType": "vector", 22:46:01 DEBUG "result": [ 22:46:01 DEBUG "resultType": "vector", 22:46:01 DEBUG "resultType": "vector", 22:46:01 DEBUG "data": { 22:46:01 DEBUG "resultType": "vector", 22:46:01 DEBUG "resultType": "vector", 22:46:01 DEBUG "data": { 22:46:01 DEBUG "status": "success", 22:46:01 DEBUG "result": [ 22:46:01 DEBUG "result": [ 22:46:01 DEBUG "status": "success", 22:46:01 DEBUG "result": [ 22:46:01 DEBUG "data": { 22:46:01 DEBUG "data": { 22:46:01 DEBUG "result": [ 22:46:01 DEBUG "result": [ 22:46:01 DEBUG "result": [ 22:46:01 DEBUG "result": [ 22:46:01 DEBUG "result": [ 22:46:01 DEBUG "result": [ 22:46:01 DEBUG "result": [ 22:46:01 DEBUG "resultType": "vector", 22:46:01 DEBUG { 22:46:01 DEBUG "result": [ 22:46:01 DEBUG "result": [ 22:46:01 DEBUG { 22:46:01 DEBUG "result": [ 22:46:01 DEBUG "result": [ 22:46:01 DEBUG "resultType": "vector", 22:46:01 DEBUG "result": [ 22:46:01 DEBUG "result": [ 22:46:01 DEBUG "resultType": "vector", 22:46:01 DEBUG "data": { 22:46:01 DEBUG { 22:46:01 DEBUG { 22:46:01 DEBUG "data": { 22:46:01 DEBUG { 22:46:01 DEBUG "resultType": "vector", 22:46:01 DEBUG "resultType": "vector", 22:46:01 DEBUG { 22:46:01 DEBUG { 22:46:01 DEBUG { 22:46:01 DEBUG { 22:46:01 DEBUG { 22:46:01 DEBUG { 22:46:01 DEBUG { 22:46:01 DEBUG "result": [ 22:46:01 DEBUG "metric": { 22:46:01 DEBUG { 22:46:01 DEBUG { 22:46:01 DEBUG "metric": { 22:46:01 DEBUG { 22:46:01 DEBUG { 22:46:01 DEBUG "result": [ 22:46:01 DEBUG { 22:46:01 DEBUG { 22:46:01 DEBUG "result": [ 22:46:01 DEBUG "resultType": "vector", 22:46:01 DEBUG "metric": { 22:46:01 DEBUG "metric": { 22:46:01 DEBUG "resultType": "vector", 22:46:01 DEBUG "metric": { 22:46:01 DEBUG "result": [ 22:46:01 DEBUG "result": [ 22:46:01 DEBUG "metric": { 22:46:01 DEBUG "metric": { 22:46:01 DEBUG "metric": { 22:46:01 DEBUG "metric": { 22:46:01 DEBUG "metric": { 22:46:01 DEBUG "metric": { 22:46:01 DEBUG "metric": { 22:46:01 DEBUG { 22:46:01 DEBUG "pod": "am-555f4db595-fz8sn" 22:46:01 DEBUG "metric": { 22:46:01 DEBUG "metric": { 22:46:01 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:46:01 DEBUG "metric": { 22:46:01 DEBUG "metric": { 22:46:01 DEBUG { 22:46:01 DEBUG "metric": { 22:46:01 DEBUG "metric": { 22:46:01 DEBUG { 22:46:01 DEBUG "result": [ 22:46:01 DEBUG "pod": "am-555f4db595-fz8sn" 22:46:01 DEBUG "pod": "am-555f4db595-fz8sn" 22:46:01 DEBUG "result": [ 22:46:01 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:46:01 DEBUG { 22:46:01 DEBUG { 22:46:01 DEBUG "pod": "ds-idrepo-2" 22:46:01 DEBUG "pod": "am-555f4db595-fz8sn" 22:46:01 DEBUG "pod": "am-555f4db595-fz8sn" 22:46:01 DEBUG "pod": "am-555f4db595-fz8sn" 22:46:01 DEBUG "pod": "ds-idrepo-2" 22:46:01 DEBUG "instance": "10.142.0.107:9100" 22:46:01 DEBUG "pod": "ds-idrepo-2" 22:46:01 DEBUG "metric": { 22:46:01 DEBUG }, 22:46:01 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:46:01 DEBUG "pod": "ds-idrepo-2" 22:46:01 DEBUG }, 22:46:01 DEBUG "instance": "10.142.0.107:9100" 22:46:01 DEBUG "instance": "10.142.0.107:9100" 22:46:01 DEBUG "metric": { 22:46:01 DEBUG "pod": "ds-idrepo-2" 22:46:01 DEBUG "pod": "ds-idrepo-2" 22:46:01 DEBUG "metric": { 22:46:01 DEBUG { 22:46:01 DEBUG }, 22:46:01 DEBUG }, 22:46:01 DEBUG { 22:46:01 DEBUG }, 22:46:01 DEBUG "metric": { 22:46:01 DEBUG "metric": { 22:46:01 DEBUG }, 22:46:01 DEBUG }, 22:46:01 DEBUG }, 22:46:01 DEBUG }, 22:46:01 DEBUG }, 22:46:01 DEBUG }, 22:46:01 DEBUG }, 22:46:01 DEBUG "pod": "ds-idrepo-2" 22:46:01 DEBUG "value": [ 22:46:01 DEBUG }, 22:46:01 DEBUG }, 22:46:01 DEBUG "value": [ 22:46:01 DEBUG }, 22:46:01 DEBUG }, 22:46:01 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:46:01 DEBUG }, 22:46:01 DEBUG }, 22:46:01 DEBUG "instance": "10.142.0.107:9100" 22:46:01 DEBUG "metric": { 22:46:01 DEBUG "value": [ 22:46:01 DEBUG "value": [ 22:46:01 DEBUG "metric": { 22:46:01 DEBUG "value": [ 22:46:01 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:46:01 DEBUG "instance": "10.142.0.107:9100" 22:46:01 DEBUG "value": [ 22:46:01 DEBUG "value": [ 22:46:01 DEBUG "value": [ 22:46:01 DEBUG "value": [ 22:46:01 DEBUG "value": [ 22:46:01 DEBUG "value": [ 22:46:01 DEBUG "value": [ 22:46:01 DEBUG }, 22:46:01 DEBUG 22:46:01 DEBUG "value": [ 22:46:01 DEBUG "value": [ 22:46:01 DEBUG 22:46:01 DEBUG "value": [ 22:46:01 DEBUG "value": [ 22:46:01 DEBUG }, 22:46:01 DEBUG "value": [ 22:46:01 DEBUG "value": [ 22:46:01 DEBUG }, 22:46:01 DEBUG "pod": "am-555f4db595-fz8sn" 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:46:01 DEBUG 22:46:01 DEBUG }, 22:46:01 DEBUG }, 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG "value": [ 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG "value": [ 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG "value": [ 22:46:01 DEBUG }, 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG "value": [ 22:46:01 DEBUG "value": [ 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG "value": [ 22:46:01 DEBUG 22:46:01 DEBUG }, 22:46:01 DEBUG [...] 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG [...] 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG [...] 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG [...] 22:46:01 DEBUG "value": [ 22:46:01 DEBUG 22:46:01 DEBUG [...] 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG [...] 22:46:01 DEBUG [...] 22:46:01 DEBUG [...] 22:46:01 DEBUG [...] 22:46:01 DEBUG [...] 22:46:01 DEBUG [...] 22:46:01 DEBUG [...] 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG [...] 22:46:01 DEBUG [...] 22:46:01 DEBUG 22:46:01 DEBUG [...] 22:46:01 DEBUG [...] 22:46:01 DEBUG 22:46:01 DEBUG [...] 22:46:01 DEBUG [...] 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG [...] 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG [...] 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG [...] 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG [...] 22:46:01 DEBUG [...] 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG [...] 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG "pod": "am-555f4db595-kzxsp" 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG "pod": "am-555f4db595-kzxsp" 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG "pod": "am-555f4db595-kzxsp" 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG "pod": "ds-idrepo-0" 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG "pod": "am-555f4db595-kzxsp" 22:46:01 DEBUG [...] 22:46:01 DEBUG }, 22:46:01 DEBUG "pod": "am-555f4db595-kzxsp" 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG "pod": "ds-idrepo-0" 22:46:01 DEBUG "pod": "am-555f4db595-kzxsp" 22:46:01 DEBUG "pod": "am-555f4db595-kzxsp" 22:46:01 DEBUG "pod": "am-555f4db595-kzxsp" 22:46:01 DEBUG "pod": "ds-idrepo-0" 22:46:01 DEBUG "instance": "10.142.0.43:9100" 22:46:01 DEBUG "pod": "ds-cts-2" 22:46:01 DEBUG 22:46:01 DEBUG }, 22:46:01 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:46:01 DEBUG "pod": "ds-idrepo-0" 22:46:01 DEBUG }, 22:46:01 DEBUG "instance": "10.142.0.43:9100" 22:46:01 DEBUG "instance": "10.142.0.43:9100" 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG }, 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG }, 22:46:01 DEBUG "value": [ 22:46:01 DEBUG "value": [ 22:46:01 DEBUG }, 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG }, 22:46:01 DEBUG }, 22:46:01 DEBUG }, 22:46:01 DEBUG }, 22:46:01 DEBUG }, 22:46:01 DEBUG }, 22:46:01 DEBUG }, 22:46:01 DEBUG "pod": "ds-idrepo-0" 22:46:01 DEBUG "value": [ 22:46:01 DEBUG }, 22:46:01 DEBUG }, 22:46:01 DEBUG "value": [ 22:46:01 DEBUG }, 22:46:01 DEBUG }, 22:46:01 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:46:01 DEBUG "pod": "ds-idrepo-0" 22:46:01 DEBUG "value": [ 22:46:01 DEBUG "instance": "10.142.0.43:9100" 22:46:01 DEBUG 22:46:01 DEBUG 22:46:01 DEBUG 1713303961, 22:46:01 DEBUG 1713303961, 22:46:01 DEBUG "value": [ 22:46:01 DEBUG "pod": "am-555f4db595-kzxsp" 22:46:01 DEBUG "instance": "10.142.0.43:9100" 22:46:01 DEBUG "value": [ 22:46:01 DEBUG "value": [ 22:46:01 DEBUG "value": [ 22:46:01 DEBUG "value": [ 22:46:01 DEBUG "value": [ 22:46:01 DEBUG "value": [ 22:46:01 DEBUG "value": [ 22:46:01 DEBUG }, 22:46:01 DEBUG 1713303961, 22:46:01 DEBUG "value": [ 22:46:01 DEBUG "value": [ 22:46:01 DEBUG 1713303961, 22:46:01 DEBUG "value": [ 22:46:01 DEBUG "value": [ 22:46:01 DEBUG }, 22:46:01 DEBUG }, 22:46:01 DEBUG 1713303961, 22:46:01 DEBUG }, 22:46:01 DEBUG "pod": "am-555f4db595-kzxsp" 22:46:01 DEBUG 22:46:01 DEBUG "3512882" 22:46:01 DEBUG "0" 22:46:01 DEBUG 1713303961, 22:46:01 DEBUG }, 22:46:01 DEBUG }, 22:46:01 DEBUG 1713303961, 22:46:01 DEBUG 1713303961, 22:46:01 DEBUG 1713303961, 22:46:01 DEBUG 1713303961, 22:46:01 DEBUG 1713303961, 22:46:01 DEBUG "0" 22:46:01 DEBUG 1713303961, 22:46:01 DEBUG "value": [ 22:46:01 DEBUG "2165.8333333333335" 22:46:01 DEBUG 1713303961, 22:46:01 DEBUG "6.773011697417551" 22:46:01 DEBUG "0" 22:46:01 DEBUG 1713303961, 22:46:01 DEBUG 1713303961, 22:46:01 DEBUG "value": [ 22:46:01 DEBUG "value": [ 22:46:01 DEBUG "0" 22:46:01 DEBUG "value": [ 22:46:01 DEBUG }, 22:46:01 DEBUG 22:46:01 DEBUG ] 22:46:01 DEBUG ] 22:46:01 DEBUG "873.239807407969" 22:46:01 DEBUG "value": [ 22:46:01 DEBUG "value": [ 22:46:01 DEBUG "0" 22:46:01 DEBUG "2164.9333333333334" 22:46:01 DEBUG "2164.633333333333" 22:46:01 DEBUG "2002.9666666666667" 22:46:01 DEBUG ] 22:46:01 DEBUG 1713303961, 22:46:01 DEBUG "0" 22:46:01 DEBUG 1713303961, 22:46:01 DEBUG ] 22:46:01 DEBUG 1713303961, 22:46:01 DEBUG ] 22:46:01 DEBUG ] 22:46:01 DEBUG "0.0004999999999995453" 22:46:01 DEBUG "19970098176" 22:46:01 DEBUG 1713303961, 22:46:01 DEBUG 1713303961, 22:46:01 DEBUG ] 22:46:01 DEBUG 1713303961, 22:46:01 DEBUG "value": [ 22:46:01 DEBUG "pod": "am-555f4db595-kzxsp" 22:46:01 DEBUG } 22:46:01 DEBUG } 22:46:01 DEBUG ] 22:46:01 DEBUG 1713303961, 22:46:01 DEBUG 1713303961, 22:46:01 DEBUG ] 22:46:01 DEBUG ] 22:46:01 DEBUG ] 22:46:01 DEBUG ] 22:46:01 DEBUG } 22:46:01 DEBUG "49262366.99999999" 22:46:01 DEBUG ] 22:46:01 DEBUG "0" 22:46:01 DEBUG } 22:46:01 DEBUG "0" 22:46:01 DEBUG } 22:46:01 DEBUG } 22:46:01 DEBUG ] 22:46:01 DEBUG ] 22:46:01 DEBUG "1541722112" 22:46:01 DEBUG "0" 22:46:01 DEBUG } 22:46:01 DEBUG "0" 22:46:01 DEBUG 1713303961, 22:46:01 DEBUG }, 22:46:01 DEBUG ] 22:46:01 DEBUG ] 22:46:01 DEBUG } 22:46:01 DEBUG "9.332604191060389" 22:46:01 DEBUG "49253042.766666666" 22:46:01 DEBUG } 22:46:01 DEBUG } 22:46:01 DEBUG } 22:46:01 DEBUG } 22:46:01 DEBUG ] 22:46:01 DEBUG ] 22:46:01 DEBUG } 22:46:01 DEBUG ] 22:46:01 DEBUG ] 22:46:01 DEBUG ] 22:46:01 DEBUG ] 22:46:01 DEBUG ] 22:46:01 DEBUG } 22:46:01 DEBUG } 22:46:01 DEBUG ] 22:46:01 DEBUG ] 22:46:01 DEBUG ] 22:46:01 DEBUG ] 22:46:01 DEBUG "0.027" 22:46:01 DEBUG "value": [ 22:46:01 DEBUG } 22:46:01 DEBUG } 22:46:01 DEBUG ] 22:46:01 DEBUG ] 22:46:01 DEBUG ] 22:46:01 DEBUG ] 22:46:01 DEBUG ] 22:46:01 DEBUG ] 22:46:01 DEBUG ] 22:46:01 DEBUG } 22:46:01 DEBUG } 22:46:01 DEBUG ] 22:46:01 DEBUG } 22:46:01 DEBUG } 22:46:01 DEBUG } 22:46:01 DEBUG } 22:46:01 DEBUG } 22:46:01 DEBUG ] 22:46:01 DEBUG ] 22:46:01 DEBUG } 22:46:01 DEBUG } 22:46:01 DEBUG } 22:46:01 DEBUG } 22:46:01 DEBUG ] 22:46:01 DEBUG 1713303961, 22:46:01 DEBUG } 22:46:01 DEBUG } 22:46:01 DEBUG } 22:46:01 DEBUG } 22:46:01 DEBUG } 22:46:01 DEBUG } 22:46:01 DEBUG } 22:46:01 DEBUG } 22:46:01 DEBUG } 22:46:01 DEBUG } 22:46:01 DEBUG ] 22:46:01 DEBUG } 22:46:01 DEBUG ] 22:46:01 DEBUG } 22:46:01 DEBUG ] 22:46:01 DEBUG } 22:46:01 DEBUG } 22:46:01 DEBUG } 22:46:01 DEBUG } 22:46:01 DEBUG ] 22:46:01 DEBUG ] 22:46:01 DEBUG } 22:46:01 DEBUG ] 22:46:01 DEBUG } 22:46:01 DEBUG "49665.51404632824" 22:46:01 DEBUG } 22:46:01 DEBUG ] 22:46:01 DEBUG ] 22:46:01 DEBUG } 22:46:01 DEBUG } 22:46:01 DEBUG } 22:46:01 DEBUG } 22:46:01 DEBUG } 22:46:01 DEBUG } 22:46:01 DEBUG } 22:46:01 DEBUG } 22:46:01 DEBUG } 22:46:01 DEBUG } 22:46:01 DEBUG } 22:46:01 DEBUG } 22:46:01 DEBUG } 22:46:01 DEBUG ] 22:46:01 DEBUG ] 22:46:01 DEBUG } 22:46:01 DEBUG } 22:46:01 DEBUG } 22:46:01 DEBUG } 22:46:01 DEBUG } 22:46:01 DEBUG } 22:46:01 DEBUG } 22:46:01 DEBUG } 22:46:01 DEBUG } 22:46:01 DEBUG } 22:46:01 DEBUG } 22:46:01 DEBUG } 22:46:01 DEBUG } 22:46:01 DEBUG ] 22:46:01 DEBUG } 22:46:01 DEBUG } 22:46:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304017" 22:46:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713304017" 22:46:57 INFO [http_cmd]: http status code OK 22:46:57 DEBUG --- status code --- 22:46:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:46:57 DEBUG --- http response --- 22:46:57 ERROR [http_cmd]: ERROR 22:46:57 ERROR --- status code --- 22:46:57 ERROR http status code is 400 (expected [200, 401, 403, 404]) 22:46:57 ERROR --- http response --- 22:46:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 22:46:57 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 22:46:57 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 22:46:59 INFO 22:46:59 INFO [loop_until]: kubectl --namespace=xlou top pods 22:46:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:46:59 INFO [loop_until]: OK (rc = 0) 22:46:59 DEBUG --- stdout --- 22:46:59 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 9Mi am-555f4db595-4lbss 13073m 8672Mi am-555f4db595-fz8sn 13698m 8602Mi am-555f4db595-kzxsp 13147m 8668Mi ds-cts-0 20320m 13018Mi ds-cts-1 18309m 12631Mi ds-cts-2 17999m 12630Mi ds-idrepo-0 5828m 5028Mi ds-idrepo-1 5573m 2908Mi ds-idrepo-2 5546m 3163Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 4m 1207Mi idm-b85f9d978-rl76f 4m 1226Mi lodemon-bc756c647-cr7x6 2m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-7c9b667974-9mr2s 6870m 1635Mi 22:46:59 DEBUG --- stderr --- 22:46:59 DEBUG 22:47:01 INFO 22:47:01 INFO [loop_until]: kubectl --namespace=xlou top node 22:47:01 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:47:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304021" 22:47:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304021" 22:47:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304021" 22:47:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304021" 22:47:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713304021" 22:47:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304021" 22:47:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304021" 22:47:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304021" 22:47:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304021" 22:47:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304021" 22:47:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713304021" 22:47:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304021" 22:47:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304021" 22:47:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304021" 22:47:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713304021" 22:47:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304021" 22:47:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304021" 22:47:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304021" 22:47:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304021" 22:47:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304021" 22:47:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304021" 22:47:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713304021" 22:47:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304021" 22:47:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304021" 22:47:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304021" 22:47:01 INFO [loop_until]: OK (rc = 0) 22:47:01 INFO [http_cmd]: http status code OK 22:47:01 INFO [http_cmd]: http status code OK 22:47:01 INFO [http_cmd]: http status code OK 22:47:01 INFO [http_cmd]: http status code OK 22:47:01 INFO [http_cmd]: http status code OK 22:47:01 INFO [http_cmd]: http status code OK 22:47:01 INFO [http_cmd]: http status code OK 22:47:01 INFO [http_cmd]: http status code OK 22:47:01 INFO [http_cmd]: http status code OK 22:47:01 INFO [http_cmd]: http status code OK 22:47:01 INFO [http_cmd]: http status code OK 22:47:01 INFO [http_cmd]: http status code OK 22:47:01 INFO [http_cmd]: http status code OK 22:47:01 INFO [http_cmd]: http status code OK 22:47:01 INFO [http_cmd]: http status code OK 22:47:01 DEBUG --- stdout --- 22:47:01 DEBUG --- status code --- 22:47:01 INFO [http_cmd]: http status code OK 22:47:01 INFO [http_cmd]: http status code OK 22:47:01 DEBUG --- status code --- 22:47:01 INFO [http_cmd]: http status code OK 22:47:01 DEBUG --- status code --- 22:47:01 INFO [http_cmd]: http status code OK 22:47:01 INFO [http_cmd]: http status code OK 22:47:01 DEBUG --- status code --- 22:47:01 INFO [http_cmd]: http status code OK 22:47:01 DEBUG --- status code --- 22:47:01 DEBUG --- status code --- 22:47:01 DEBUG --- status code --- 22:47:01 DEBUG --- status code --- 22:47:01 DEBUG --- status code --- 22:47:01 DEBUG --- status code --- 22:47:01 DEBUG --- status code --- 22:47:01 DEBUG --- status code --- 22:47:01 DEBUG --- status code --- 22:47:01 DEBUG --- status code --- 22:47:01 DEBUG --- status code --- 22:47:01 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 5077Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 18474m 61% 15162Mi 13% gke-xlou-cdm-default-pool-1b0769c8-nhlf 13207m 44% 10826Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 82m 0% 2639Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 5647m 18% 6513Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 21065m 70% 17616Mi 15% gke-xlou-cdm-default-pool-1b0769c8-xnmp 59m 0% 3143Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 13616m 45% 10454Mi 9% gke-xlou-cdm-ds-0f601751-hmm7 6089m 20% 5660Mi 5% gke-xlou-cdm-ds-0f601751-j488 18235m 61% 14497Mi 12% gke-xlou-cdm-ds-0f601751-mc5d 11644m 39% 3321Mi 2% gke-xlou-cdm-ds-0f601751-nj6q 14077m 47% 10543Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 6110m 20% 7351Mi 6% gke-xlou-cdm-frontend-a8771548-i82n 6968m 43% 7071Mi 12% 22:47:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:47:01 DEBUG --- status code --- 22:47:01 DEBUG --- status code --- 22:47:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:47:01 DEBUG --- status code --- 22:47:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:47:01 DEBUG --- status code --- 22:47:01 INFO [http_cmd]: http status code OK 22:47:01 DEBUG --- status code --- 22:47:01 INFO [http_cmd]: http status code OK 22:47:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:47:01 DEBUG --- status code --- 22:47:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:47:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:47:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:47:01 DEBUG --- http response --- 22:47:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:47:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:47:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:47:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:47:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:47:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:47:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:47:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:47:01 DEBUG --- stderr --- 22:47:01 DEBUG --- http response --- 22:47:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:47:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:47:01 DEBUG --- http response --- 22:47:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:47:01 DEBUG --- http response --- 22:47:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:47:01 DEBUG --- status code --- 22:47:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:47:01 DEBUG --- status code --- 22:47:01 DEBUG --- http response --- 22:47:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:47:01 DEBUG --- http response --- 22:47:01 DEBUG --- http response --- 22:47:01 INFO [http_cmd]: http status code OK 22:47:01 INFO [http_cmd]: http status code OK 22:47:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 22:47:01 DEBUG --- http response --- 22:47:01 DEBUG --- http response --- 22:47:01 DEBUG --- http response --- 22:47:01 DEBUG --- http response --- 22:47:01 DEBUG --- http response --- 22:47:01 DEBUG --- http response --- 22:47:01 DEBUG --- http response --- 22:47:01 DEBUG --- http response --- 22:47:01 DEBUG 22:47:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:47:01 DEBUG --- http response --- 22:47:01 DEBUG --- http response --- 22:47:01 DEBUG --- http response --- 22:47:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 22:47:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:47:01 DEBUG --- http response --- 22:47:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:47:01 DEBUG --- http response --- 22:47:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:47:01 DEBUG --- http response --- 22:47:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:47:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:47:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:47:01 DEBUG --- status code --- 22:47:01 DEBUG --- status code --- 22:47:01 DEBUG ----- output ----- 22:47:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:47:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:47:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:47:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:47:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:47:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:47:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:47:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:47:01 DEBUG ----- output ----- 22:47:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:47:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:47:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:47:01 DEBUG ----- output ----- 22:47:01 DEBUG ----- output ----- 22:47:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:47:01 DEBUG --- http response --- 22:47:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:47:01 DEBUG --- http response --- 22:47:01 DEBUG ----- output ----- 22:47:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:47:01 DEBUG ----- output ----- 22:47:01 DEBUG ----- output ----- 22:47:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:47:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:47:01 DEBUG { 22:47:01 DEBUG ----- output ----- 22:47:01 DEBUG ----- output ----- 22:47:01 DEBUG ----- output ----- 22:47:01 DEBUG ----- output ----- 22:47:01 DEBUG ----- output ----- 22:47:01 DEBUG ----- output ----- 22:47:01 DEBUG ----- output ----- 22:47:01 DEBUG ----- output ----- 22:47:01 DEBUG { 22:47:01 DEBUG ----- output ----- 22:47:01 DEBUG ----- output ----- 22:47:01 DEBUG ----- output ----- 22:47:01 DEBUG { 22:47:01 DEBUG { 22:47:01 DEBUG ----- output ----- 22:47:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:47:01 DEBUG ----- output ----- 22:47:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:47:01 DEBUG { 22:47:01 DEBUG ----- output ----- 22:47:01 DEBUG { 22:47:01 DEBUG { 22:47:01 DEBUG --- http response --- 22:47:01 DEBUG --- http response --- 22:47:01 DEBUG "status": "success", 22:47:01 DEBUG { 22:47:01 DEBUG { 22:47:01 DEBUG { 22:47:01 DEBUG { 22:47:01 DEBUG "status": "success", 22:47:01 DEBUG { 22:47:01 DEBUG { 22:47:01 DEBUG { 22:47:01 DEBUG "status": "success", 22:47:01 DEBUG { 22:47:01 DEBUG { 22:47:01 DEBUG { 22:47:01 DEBUG "status": "success", 22:47:01 DEBUG "status": "success", 22:47:01 DEBUG { 22:47:01 DEBUG ----- output ----- 22:47:01 DEBUG { 22:47:01 DEBUG ----- output ----- 22:47:01 DEBUG "status": "success", 22:47:01 DEBUG { 22:47:01 DEBUG "status": "success", 22:47:01 DEBUG "status": "success", 22:47:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:47:01 DEBUG "data": { 22:47:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:47:01 DEBUG "status": "success", 22:47:01 DEBUG "status": "success", 22:47:01 DEBUG "status": "success", 22:47:01 DEBUG { 22:47:01 DEBUG "data": { 22:47:01 DEBUG "status": "success", 22:47:01 DEBUG "status": "success", 22:47:01 DEBUG "status": "success", 22:47:01 DEBUG "data": { 22:47:01 DEBUG "status": "success", 22:47:01 DEBUG "status": "success", 22:47:01 DEBUG "status": "success", 22:47:01 DEBUG "data": { 22:47:01 DEBUG "data": { 22:47:01 DEBUG "status": "success", 22:47:01 DEBUG { 22:47:01 DEBUG "status": "success", 22:47:01 DEBUG { 22:47:01 DEBUG "data": { 22:47:01 DEBUG "status": "success", 22:47:01 DEBUG "data": { 22:47:01 DEBUG "data": { 22:47:01 DEBUG ----- output ----- 22:47:01 DEBUG "resultType": "vector", 22:47:01 DEBUG ----- output ----- 22:47:01 DEBUG "data": { 22:47:01 DEBUG "data": { 22:47:01 DEBUG "data": { 22:47:01 DEBUG "status": "success", 22:47:01 DEBUG "resultType": "vector", 22:47:01 DEBUG "data": { 22:47:01 DEBUG "data": { 22:47:01 DEBUG "data": { 22:47:01 DEBUG "resultType": "vector", 22:47:01 DEBUG "data": { 22:47:01 DEBUG "data": { 22:47:01 DEBUG "data": { 22:47:01 DEBUG "resultType": "vector", 22:47:01 DEBUG "resultType": "vector", 22:47:01 DEBUG "data": { 22:47:01 DEBUG "resultType": "vector", 22:47:01 DEBUG "data": { 22:47:01 DEBUG "status": "success", 22:47:01 DEBUG "resultType": "vector", 22:47:01 DEBUG "data": { 22:47:01 DEBUG "resultType": "vector", 22:47:01 DEBUG "resultType": "vector", 22:47:01 DEBUG { 22:47:01 DEBUG "result": [ 22:47:01 DEBUG { 22:47:01 DEBUG "resultType": "vector", 22:47:01 DEBUG "resultType": "vector", 22:47:01 DEBUG "resultType": "vector", 22:47:01 DEBUG "data": { 22:47:01 DEBUG "result": [ 22:47:01 DEBUG "resultType": "vector", 22:47:01 DEBUG "resultType": "vector", 22:47:01 DEBUG "resultType": "vector", 22:47:01 DEBUG "result": [ 22:47:01 DEBUG "resultType": "vector", 22:47:01 DEBUG "resultType": "vector", 22:47:01 DEBUG "resultType": "vector", 22:47:01 DEBUG "result": [ 22:47:01 DEBUG "result": [ 22:47:01 DEBUG "status": "success", 22:47:01 DEBUG "result": [ 22:47:01 DEBUG "resultType": "vector", 22:47:01 DEBUG "data": { 22:47:01 DEBUG "result": [ 22:47:01 DEBUG "resultType": "vector", 22:47:01 DEBUG "result": [ 22:47:01 DEBUG "result": [ 22:47:01 DEBUG "status": "success", 22:47:01 DEBUG { 22:47:01 DEBUG "status": "success", 22:47:01 DEBUG "result": [ 22:47:01 DEBUG "result": [ 22:47:01 DEBUG "result": [ 22:47:01 DEBUG "resultType": "vector", 22:47:01 DEBUG { 22:47:01 DEBUG "result": [ 22:47:01 DEBUG "result": [ 22:47:01 DEBUG "result": [ 22:47:01 DEBUG { 22:47:01 DEBUG "result": [ 22:47:01 DEBUG "result": [ 22:47:01 DEBUG "result": [ 22:47:01 DEBUG { 22:47:01 DEBUG { 22:47:01 DEBUG "data": { 22:47:01 DEBUG { 22:47:01 DEBUG "result": [ 22:47:01 DEBUG "resultType": "vector", 22:47:01 DEBUG { 22:47:01 DEBUG "result": [ 22:47:01 DEBUG { 22:47:01 DEBUG { 22:47:01 DEBUG "data": { 22:47:01 DEBUG "metric": { 22:47:01 DEBUG "data": { 22:47:01 DEBUG { 22:47:01 DEBUG { 22:47:01 DEBUG { 22:47:01 DEBUG "result": [ 22:47:01 DEBUG "metric": { 22:47:01 DEBUG { 22:47:01 DEBUG { 22:47:01 DEBUG { 22:47:01 DEBUG "metric": { 22:47:01 DEBUG { 22:47:01 DEBUG { 22:47:01 DEBUG { 22:47:01 DEBUG "metric": { 22:47:01 DEBUG "metric": { 22:47:01 DEBUG "resultType": "vector", 22:47:01 DEBUG "metric": { 22:47:01 DEBUG { 22:47:01 DEBUG "result": [ 22:47:01 DEBUG "metric": { 22:47:01 DEBUG { 22:47:01 DEBUG "metric": { 22:47:01 DEBUG "metric": { 22:47:01 DEBUG "resultType": "vector", 22:47:01 DEBUG "pod": "ds-cts-1" 22:47:01 DEBUG "resultType": "vector", 22:47:01 DEBUG "metric": { 22:47:01 DEBUG "metric": { 22:47:01 DEBUG "metric": { 22:47:01 DEBUG { 22:47:01 DEBUG "pod": "am-555f4db595-fz8sn" 22:47:01 DEBUG "metric": { 22:47:01 DEBUG "metric": { 22:47:01 DEBUG "metric": { 22:47:01 DEBUG "pod": "am-555f4db595-fz8sn" 22:47:01 DEBUG "metric": { 22:47:01 DEBUG "metric": { 22:47:01 DEBUG "metric": { 22:47:01 DEBUG "pod": "ds-cts-1" 22:47:01 DEBUG "pod": "am-555f4db595-fz8sn" 22:47:01 DEBUG "result": [ 22:47:01 DEBUG "pod": "ds-idrepo-2" 22:47:01 DEBUG "metric": { 22:47:01 DEBUG { 22:47:01 DEBUG "metric": { 22:47:01 DEBUG "metric": { 22:47:01 DEBUG "pod": "ds-idrepo-2" 22:47:01 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:47:01 DEBUG "result": [ 22:47:01 DEBUG }, 22:47:01 DEBUG "result": [ 22:47:01 DEBUG "pod": "am-555f4db595-fz8sn" 22:47:01 DEBUG "pod": "ds-idrepo-2" 22:47:01 DEBUG "pod": "ds-idrepo-2" 22:47:01 DEBUG "metric": { 22:47:01 DEBUG }, 22:47:01 DEBUG "pod": "ds-idrepo-2" 22:47:01 DEBUG "instance": "10.142.0.107:9100" 22:47:01 DEBUG "instance": "10.142.0.107:9100" 22:47:01 DEBUG }, 22:47:01 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:47:01 DEBUG "pod": "am-555f4db595-fz8sn" 22:47:01 DEBUG "pod": "ds-idrepo-2" 22:47:01 DEBUG }, 22:47:01 DEBUG }, 22:47:01 DEBUG { 22:47:01 DEBUG }, 22:47:01 DEBUG "pod": "ds-idrepo-2" 22:47:01 DEBUG "pod": "ds-cts-1" 22:47:01 DEBUG "pod": "am-555f4db595-fz8sn" 22:47:01 DEBUG "pod": "am-555f4db595-fz8sn" 22:47:01 DEBUG }, 22:47:01 DEBUG }, 22:47:01 DEBUG { 22:47:01 DEBUG "value": [ 22:47:01 DEBUG { 22:47:01 DEBUG }, 22:47:01 DEBUG }, 22:47:01 DEBUG }, 22:47:01 DEBUG "instance": "10.142.0.107:9100" 22:47:01 DEBUG "value": [ 22:47:01 DEBUG }, 22:47:01 DEBUG }, 22:47:01 DEBUG }, 22:47:01 DEBUG "value": [ 22:47:01 DEBUG }, 22:47:01 DEBUG }, 22:47:01 DEBUG }, 22:47:01 DEBUG "value": [ 22:47:01 DEBUG "value": [ 22:47:01 DEBUG "metric": { 22:47:01 DEBUG "value": [ 22:47:01 DEBUG }, 22:47:01 DEBUG }, 22:47:01 DEBUG }, 22:47:01 DEBUG }, 22:47:01 DEBUG "value": [ 22:47:01 DEBUG "value": [ 22:47:01 DEBUG "metric": { 22:47:01 DEBUG 22:47:01 DEBUG "metric": { 22:47:01 DEBUG "value": [ 22:47:01 DEBUG "value": [ 22:47:01 DEBUG "value": [ 22:47:01 DEBUG }, 22:47:01 DEBUG "value": [ 22:47:01 DEBUG 22:47:01 DEBUG "value": [ 22:47:01 DEBUG "value": [ 22:47:01 DEBUG 22:47:01 DEBUG "value": [ 22:47:01 DEBUG "value": [ 22:47:01 DEBUG "value": [ 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG "instance": "10.142.0.107:9100" 22:47:01 DEBUG 22:47:01 DEBUG "value": [ 22:47:01 DEBUG "value": [ 22:47:01 DEBUG "value": [ 22:47:01 DEBUG 22:47:01 DEBUG "value": [ 22:47:01 DEBUG 22:47:01 DEBUG "pod": "ds-cts-1" 22:47:01 DEBUG 22:47:01 DEBUG "instance": "10.142.0.107:9100" 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG "value": [ 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG }, 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG }, 22:47:01 DEBUG 22:47:01 DEBUG }, 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG "value": [ 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG "value": [ 22:47:01 DEBUG [...] 22:47:01 DEBUG "value": [ 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG [...] 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG [...] 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG [...] 22:47:01 DEBUG [...] 22:47:01 DEBUG 22:47:01 DEBUG [...] 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG [...] 22:47:01 DEBUG 22:47:01 DEBUG [...] 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG [...] 22:47:01 DEBUG [...] 22:47:01 DEBUG [...] 22:47:01 DEBUG 22:47:01 DEBUG [...] 22:47:01 DEBUG 22:47:01 DEBUG [...] 22:47:01 DEBUG [...] 22:47:01 DEBUG 22:47:01 DEBUG [...] 22:47:01 DEBUG [...] 22:47:01 DEBUG [...] 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG [...] 22:47:01 DEBUG [...] 22:47:01 DEBUG 22:47:01 DEBUG [...] 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG [...] 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG [...] 22:47:01 DEBUG [...] 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG [...] 22:47:01 DEBUG "pod": "overseer-0-7c9b667974-9mr2s" 22:47:01 DEBUG [...] 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG "pod": "am-555f4db595-kzxsp" 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG "pod": "am-555f4db595-kzxsp" 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG "pod": "overseer-0-7c9b667974-9mr2s" 22:47:01 DEBUG "pod": "am-555f4db595-kzxsp" 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG "pod": "ds-idrepo-0" 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG "pod": "am-555f4db595-kzxsp" 22:47:01 DEBUG 22:47:01 DEBUG "pod": "am-555f4db595-kzxsp" 22:47:01 DEBUG 22:47:01 DEBUG }, 22:47:01 DEBUG 22:47:01 DEBUG "pod": "am-555f4db595-kzxsp" 22:47:01 DEBUG "pod": "ds-idrepo-0" 22:47:01 DEBUG "pod": "ds-cts-2" 22:47:01 DEBUG 22:47:01 DEBUG "instance": "10.142.0.43:9100" 22:47:01 DEBUG }, 22:47:01 DEBUG "instance": "10.142.0.43:9100" 22:47:01 DEBUG "instance": "10.142.0.43:9100" 22:47:01 DEBUG }, 22:47:01 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:47:01 DEBUG "pod": "am-555f4db595-kzxsp" 22:47:01 DEBUG "pod": "ds-idrepo-0" 22:47:01 DEBUG }, 22:47:01 DEBUG }, 22:47:01 DEBUG 22:47:01 DEBUG 22:47:01 DEBUG }, 22:47:01 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:47:01 DEBUG "pod": "ds-idrepo-0" 22:47:01 DEBUG }, 22:47:01 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:47:01 DEBUG }, 22:47:01 DEBUG 22:47:01 DEBUG "value": [ 22:47:01 DEBUG 22:47:01 DEBUG }, 22:47:01 DEBUG }, 22:47:01 DEBUG }, 22:47:01 DEBUG "pod": "ds-idrepo-0" 22:47:01 DEBUG }, 22:47:01 DEBUG "value": [ 22:47:01 DEBUG }, 22:47:01 DEBUG }, 22:47:01 DEBUG "value": [ 22:47:01 DEBUG }, 22:47:01 DEBUG }, 22:47:01 DEBUG }, 22:47:01 DEBUG "value": [ 22:47:01 DEBUG "value": [ 22:47:01 DEBUG "pod": "ds-idrepo-0" 22:47:01 DEBUG "instance": "10.142.0.43:9100" 22:47:01 DEBUG "value": [ 22:47:01 DEBUG }, 22:47:01 DEBUG }, 22:47:01 DEBUG "value": [ 22:47:01 DEBUG }, 22:47:01 DEBUG "value": [ 22:47:01 DEBUG 22:47:01 DEBUG 1713304021, 22:47:01 DEBUG 22:47:01 DEBUG "value": [ 22:47:01 DEBUG "value": [ 22:47:01 DEBUG "value": [ 22:47:01 DEBUG }, 22:47:01 DEBUG "value": [ 22:47:01 DEBUG 1713304021, 22:47:01 DEBUG "value": [ 22:47:01 DEBUG "value": [ 22:47:01 DEBUG 1713304021, 22:47:01 DEBUG "3523373.5" 22:47:01 DEBUG ] 22:47:01 DEBUG "value": [ 22:47:01 DEBUG 1713304021, 22:47:01 DEBUG 1713304021, 22:47:01 DEBUG }, 22:47:01 DEBUG }, 22:47:01 DEBUG 1713304021, 22:47:01 DEBUG "value": [ 22:47:01 DEBUG "value": [ 22:47:01 DEBUG 1713304021, 22:47:01 DEBUG "value": [ 22:47:01 DEBUG 1713304021, 22:47:01 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:47:01 DEBUG "0" 22:47:01 DEBUG "instance": "10.142.0.43:9100" 22:47:01 DEBUG 1713304021, 22:47:01 DEBUG 1713304021, 22:47:01 DEBUG 1713304021, 22:47:01 DEBUG "value": [ 22:47:01 DEBUG 1713304021, 22:47:01 DEBUG "1994.8666666666666" 22:47:01 DEBUG 1713304021, 22:47:01 DEBUG 1713304021, 22:47:01 DEBUG "value": [ 22:47:01 DEBUG "value": [ 22:47:01 DEBUG } 22:47:01 DEBUG 1713304021, 22:47:01 DEBUG "0" 22:47:01 DEBUG "0" 22:47:01 DEBUG "value": [ 22:47:01 DEBUG 1713304021, 22:47:01 DEBUG "0" 22:47:01 DEBUG 1713304021, 22:47:01 DEBUG 1713304021, 22:47:01 DEBUG "2173.4" 22:47:01 DEBUG 1713304021, 22:47:01 DEBUG "0.027" 22:47:01 DEBUG }, 22:47:01 DEBUG ] 22:47:01 DEBUG }, 22:47:01 DEBUG "2170.6666666666665" 22:47:01 DEBUG "0" 22:47:01 DEBUG "0" 22:47:01 DEBUG 1713304021, 22:47:01 DEBUG "0.00046666666666700016" 22:47:01 DEBUG ] 22:47:01 DEBUG "19970798592" 22:47:01 DEBUG "49560622.033333324" 22:47:01 DEBUG 1713304021, 22:47:01 DEBUG 1713304021, 22:47:01 DEBUG ] 22:47:01 DEBUG "0" 22:47:01 DEBUG ] 22:47:01 DEBUG ] 22:47:01 DEBUG "value": [ 22:47:01 DEBUG "0" 22:47:01 DEBUG ] 22:47:01 DEBUG "0" 22:47:01 DEBUG "0" 22:47:01 DEBUG ] 22:47:01 DEBUG "6.904047085320128" 22:47:01 DEBUG ] 22:47:01 DEBUG "value": [ 22:47:01 DEBUG } 22:47:01 DEBUG "value": [ 22:47:01 DEBUG ] 22:47:01 DEBUG ] 22:47:01 DEBUG ] 22:47:01 DEBUG "0" 22:47:01 DEBUG ] 22:47:01 DEBUG } 22:47:01 DEBUG ] 22:47:01 DEBUG ] 22:47:01 DEBUG "1810526208" 22:47:01 DEBUG "2170.2666666666664" 22:47:01 DEBUG } 22:47:01 DEBUG ] 22:47:01 DEBUG } 22:47:01 DEBUG } 22:47:01 DEBUG 1713304021, 22:47:01 DEBUG ] 22:47:01 DEBUG } 22:47:01 DEBUG ] 22:47:01 DEBUG ] 22:47:01 DEBUG } 22:47:01 DEBUG ] 22:47:01 DEBUG } 22:47:01 DEBUG 1713304021, 22:47:01 DEBUG ] 22:47:01 DEBUG 1713304021, 22:47:01 DEBUG } 22:47:01 DEBUG } 22:47:01 DEBUG } 22:47:01 DEBUG ] 22:47:01 DEBUG } 22:47:01 DEBUG ] 22:47:01 DEBUG } 22:47:01 DEBUG } 22:47:01 DEBUG ] 22:47:01 DEBUG ] 22:47:01 DEBUG } 22:47:01 DEBUG } 22:47:01 DEBUG ] 22:47:01 DEBUG ] 22:47:01 DEBUG "49551356.93333334" 22:47:01 DEBUG } 22:47:01 DEBUG ] 22:47:01 DEBUG } 22:47:01 DEBUG } 22:47:01 DEBUG ] 22:47:01 DEBUG } 22:47:01 DEBUG ] 22:47:01 DEBUG "0" 22:47:01 DEBUG } 22:47:01 DEBUG "0.0003333333333333262" 22:47:01 DEBUG ] 22:47:01 DEBUG ] 22:47:01 DEBUG ] 22:47:01 DEBUG } 22:47:01 DEBUG ] 22:47:01 DEBUG } 22:47:01 DEBUG ] 22:47:01 DEBUG ] 22:47:01 DEBUG } 22:47:01 DEBUG } 22:47:01 DEBUG ] 22:47:01 DEBUG } 22:47:01 DEBUG } 22:47:01 DEBUG ] 22:47:01 DEBUG ] 22:47:01 DEBUG } 22:47:01 DEBUG ] 22:47:01 DEBUG ] 22:47:01 DEBUG } 22:47:01 DEBUG ] 22:47:01 DEBUG } 22:47:01 DEBUG ] 22:47:01 DEBUG } 22:47:01 DEBUG ] 22:47:01 DEBUG } 22:47:01 DEBUG } 22:47:01 DEBUG } 22:47:01 DEBUG ] 22:47:01 DEBUG } 22:47:01 DEBUG } 22:47:01 DEBUG } 22:47:01 DEBUG } 22:47:01 DEBUG ] 22:47:01 DEBUG ] 22:47:01 DEBUG } 22:47:01 DEBUG } 22:47:01 DEBUG } 22:47:01 DEBUG } 22:47:01 DEBUG } 22:47:01 DEBUG } 22:47:01 DEBUG } 22:47:01 DEBUG } 22:47:01 DEBUG } 22:47:01 DEBUG } 22:47:01 DEBUG } 22:47:01 DEBUG } 22:47:01 DEBUG } 22:47:01 DEBUG } 22:47:01 DEBUG } 22:47:01 DEBUG } 22:47:01 DEBUG } 22:47:01 DEBUG } 22:47:01 DEBUG } 22:47:01 DEBUG } 22:47:01 DEBUG } 22:47:01 DEBUG } 22:47:01 DEBUG } 22:47:01 DEBUG ] 22:47:01 DEBUG } 22:47:01 DEBUG } 22:47:01 DEBUG } 22:47:01 DEBUG } 22:47:01 DEBUG ] 22:47:01 DEBUG ] 22:47:01 DEBUG } 22:47:01 DEBUG } 22:47:01 DEBUG } 22:47:01 DEBUG } 22:47:01 DEBUG } 22:47:01 DEBUG } 22:47:01 DEBUG } 22:47:01 DEBUG } 22:47:01 DEBUG } 22:47:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304077" 22:47:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713304077" 22:47:57 ERROR [http_cmd]: ERROR 22:47:57 INFO [http_cmd]: http status code OK 22:47:57 ERROR --- status code --- 22:47:57 DEBUG --- status code --- 22:47:57 ERROR http status code is 400 (expected [200, 401, 403, 404]) 22:47:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:47:57 DEBUG --- http response --- 22:47:57 ERROR --- http response --- 22:47:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 22:47:57 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 22:47:57 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 22:47:59 INFO 22:47:59 INFO [loop_until]: kubectl --namespace=xlou top pods 22:47:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:47:59 INFO [loop_until]: OK (rc = 0) 22:47:59 DEBUG --- stdout --- 22:47:59 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 9Mi am-555f4db595-4lbss 12579m 8687Mi am-555f4db595-fz8sn 13809m 8558Mi am-555f4db595-kzxsp 13113m 8670Mi ds-cts-0 19434m 12588Mi ds-cts-1 18180m 12695Mi ds-cts-2 18018m 12682Mi ds-idrepo-0 5971m 5028Mi ds-idrepo-1 5497m 2908Mi ds-idrepo-2 5477m 3220Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 3m 1208Mi idm-b85f9d978-rl76f 4m 1225Mi lodemon-bc756c647-cr7x6 1m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-7c9b667974-9mr2s 7063m 1518Mi 22:47:59 DEBUG --- stderr --- 22:47:59 DEBUG 22:48:01 INFO 22:48:01 INFO [loop_until]: kubectl --namespace=xlou top node 22:48:01 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:48:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304081" 22:48:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304081" 22:48:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304081" 22:48:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304081" 22:48:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304081" 22:48:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713304081" 22:48:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304081" 22:48:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304081" 22:48:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304081" 22:48:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304081" 22:48:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304081" 22:48:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304081" 22:48:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304081" 22:48:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304081" 22:48:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304081" 22:48:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713304081" 22:48:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713304081" 22:48:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713304081" 22:48:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304081" 22:48:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304081" 22:48:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304081" 22:48:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304081" 22:48:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304081" 22:48:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304081" 22:48:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304081" 22:48:01 INFO [http_cmd]: http status code OK 22:48:01 INFO [http_cmd]: http status code OK 22:48:01 INFO [http_cmd]: http status code OK 22:48:01 INFO [http_cmd]: http status code OK 22:48:01 INFO [http_cmd]: http status code OK 22:48:01 INFO [http_cmd]: http status code OK 22:48:01 INFO [loop_until]: OK (rc = 0) 22:48:01 INFO [http_cmd]: http status code OK 22:48:01 INFO [http_cmd]: http status code OK 22:48:01 INFO [http_cmd]: http status code OK 22:48:01 INFO [http_cmd]: http status code OK 22:48:01 INFO [http_cmd]: http status code OK 22:48:01 INFO [http_cmd]: http status code OK 22:48:01 INFO [http_cmd]: http status code OK 22:48:01 INFO [http_cmd]: http status code OK 22:48:01 DEBUG --- status code --- 22:48:01 DEBUG --- status code --- 22:48:01 INFO [http_cmd]: http status code OK 22:48:01 DEBUG --- status code --- 22:48:01 INFO [http_cmd]: http status code OK 22:48:01 DEBUG --- status code --- 22:48:01 DEBUG --- status code --- 22:48:01 DEBUG --- status code --- 22:48:01 DEBUG --- stdout --- 22:48:01 DEBUG --- status code --- 22:48:01 DEBUG --- status code --- 22:48:01 DEBUG --- status code --- 22:48:01 DEBUG --- status code --- 22:48:01 DEBUG --- status code --- 22:48:01 DEBUG --- status code --- 22:48:01 DEBUG --- status code --- 22:48:01 DEBUG --- status code --- 22:48:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:48:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:48:01 DEBUG --- status code --- 22:48:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:48:01 DEBUG --- status code --- 22:48:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:48:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:48:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:48:01 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 5084Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 18142m 60% 15343Mi 13% gke-xlou-cdm-default-pool-1b0769c8-nhlf 13158m 44% 10804Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 83m 0% 2644Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 5635m 18% 6546Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 20761m 69% 17095Mi 15% gke-xlou-cdm-default-pool-1b0769c8-xnmp 58m 0% 3140Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 13290m 44% 10461Mi 9% gke-xlou-cdm-ds-0f601751-hmm7 6161m 20% 5671Mi 5% gke-xlou-cdm-ds-0f601751-j488 18064m 60% 14676Mi 13% gke-xlou-cdm-ds-0f601751-mc5d 11749m 39% 3368Mi 3% gke-xlou-cdm-ds-0f601751-nj6q 13677m 45% 10537Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 6145m 20% 7350Mi 6% gke-xlou-cdm-frontend-a8771548-i82n 7225m 45% 6928Mi 11% 22:48:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:48:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:48:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:48:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:48:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:48:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:48:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:48:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:48:01 DEBUG --- http response --- 22:48:01 DEBUG --- http response --- 22:48:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:48:01 DEBUG --- http response --- 22:48:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:48:01 INFO [http_cmd]: http status code OK 22:48:01 DEBUG --- http response --- 22:48:01 DEBUG --- http response --- 22:48:01 DEBUG --- http response --- 22:48:01 DEBUG --- stderr --- 22:48:01 INFO [http_cmd]: http status code OK 22:48:01 DEBUG --- http response --- 22:48:01 DEBUG --- http response --- 22:48:01 DEBUG --- http response --- 22:48:01 INFO [http_cmd]: http status code OK 22:48:01 INFO [http_cmd]: http status code OK 22:48:01 DEBUG --- http response --- 22:48:01 INFO [http_cmd]: http status code OK 22:48:01 DEBUG --- http response --- 22:48:01 DEBUG --- http response --- 22:48:01 DEBUG --- http response --- 22:48:01 DEBUG --- http response --- 22:48:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:48:01 INFO [http_cmd]: http status code OK 22:48:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:48:01 DEBUG --- http response --- 22:48:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:48:01 DEBUG --- http response --- 22:48:01 DEBUG --- status code --- 22:48:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 22:48:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:48:01 INFO [http_cmd]: http status code OK 22:48:01 INFO [http_cmd]: http status code OK 22:48:01 INFO [http_cmd]: http status code OK 22:48:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:48:01 DEBUG 22:48:01 DEBUG --- status code --- 22:48:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:48:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:48:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:48:01 DEBUG --- status code --- 22:48:01 DEBUG --- status code --- 22:48:01 DEBUG --- status code --- 22:48:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:48:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:48:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:48:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:48:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:48:01 DEBUG ----- output ----- 22:48:01 DEBUG --- status code --- 22:48:01 DEBUG ----- output ----- 22:48:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:48:01 DEBUG ----- output ----- 22:48:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:48:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:48:01 DEBUG ----- output ----- 22:48:01 DEBUG ----- output ----- 22:48:01 DEBUG --- status code --- 22:48:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:48:01 DEBUG --- status code --- 22:48:01 DEBUG ----- output ----- 22:48:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:48:01 DEBUG ----- output ----- 22:48:01 DEBUG ----- output ----- 22:48:01 DEBUG ----- output ----- 22:48:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:48:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:48:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:48:01 DEBUG ----- output ----- 22:48:01 DEBUG ----- output ----- 22:48:01 DEBUG ----- output ----- 22:48:01 DEBUG ----- output ----- 22:48:01 DEBUG ----- output ----- 22:48:01 DEBUG { 22:48:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:48:01 DEBUG { 22:48:01 DEBUG ----- output ----- 22:48:01 DEBUG { 22:48:01 DEBUG ----- output ----- 22:48:01 DEBUG --- http response --- 22:48:01 DEBUG { 22:48:01 DEBUG { 22:48:01 DEBUG --- http response --- 22:48:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:48:01 DEBUG { 22:48:01 DEBUG --- http response --- 22:48:01 DEBUG { 22:48:01 DEBUG { 22:48:01 DEBUG { 22:48:01 DEBUG --- http response --- 22:48:01 DEBUG --- http response --- 22:48:01 DEBUG --- http response --- 22:48:01 DEBUG { 22:48:01 DEBUG { 22:48:01 DEBUG { 22:48:01 DEBUG { 22:48:01 DEBUG { 22:48:01 DEBUG "status": "success", 22:48:01 DEBUG --- http response --- 22:48:01 DEBUG "status": "success", 22:48:01 DEBUG { 22:48:01 DEBUG "status": "success", 22:48:01 DEBUG { 22:48:01 DEBUG "status": "success", 22:48:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:48:01 DEBUG "status": "success", 22:48:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 22:48:01 DEBUG --- http response --- 22:48:01 DEBUG "status": "success", 22:48:01 DEBUG "status": "success", 22:48:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:48:01 DEBUG --- status code --- 22:48:01 DEBUG "status": "success", 22:48:01 DEBUG "status": "success", 22:48:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:48:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:48:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:48:01 DEBUG "status": "success", 22:48:01 DEBUG "status": "success", 22:48:01 DEBUG "status": "success", 22:48:01 DEBUG "status": "success", 22:48:01 DEBUG "status": "success", 22:48:01 DEBUG "data": { 22:48:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:48:01 DEBUG "data": { 22:48:01 DEBUG "status": "success", 22:48:01 DEBUG "data": { 22:48:01 DEBUG "status": "success", 22:48:01 DEBUG "data": { 22:48:01 DEBUG ----- output ----- 22:48:01 DEBUG "data": { 22:48:01 DEBUG ----- output ----- 22:48:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:48:01 DEBUG "data": { 22:48:01 DEBUG "data": { 22:48:01 DEBUG ----- output ----- 22:48:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:48:01 DEBUG "data": { 22:48:01 DEBUG "data": { 22:48:01 DEBUG ----- output ----- 22:48:01 DEBUG ----- output ----- 22:48:01 DEBUG ----- output ----- 22:48:01 DEBUG "data": { 22:48:01 DEBUG "data": { 22:48:01 DEBUG "data": { 22:48:01 DEBUG "data": { 22:48:01 DEBUG "data": { 22:48:01 DEBUG "resultType": "vector", 22:48:01 DEBUG ----- output ----- 22:48:01 DEBUG "resultType": "vector", 22:48:01 DEBUG "data": { 22:48:01 DEBUG "resultType": "vector", 22:48:01 DEBUG "data": { 22:48:01 DEBUG "resultType": "vector", 22:48:01 DEBUG { 22:48:01 DEBUG "resultType": "vector", 22:48:01 DEBUG { 22:48:01 DEBUG ----- output ----- 22:48:01 DEBUG "resultType": "vector", 22:48:01 DEBUG "resultType": "vector", 22:48:01 DEBUG { 22:48:01 DEBUG --- http response --- 22:48:01 DEBUG "resultType": "vector", 22:48:01 DEBUG "resultType": "vector", 22:48:01 DEBUG { 22:48:01 DEBUG { 22:48:01 DEBUG { 22:48:01 DEBUG "resultType": "vector", 22:48:01 DEBUG "resultType": "vector", 22:48:01 DEBUG "resultType": "vector", 22:48:01 DEBUG "resultType": "vector", 22:48:01 DEBUG "resultType": "vector", 22:48:01 DEBUG "result": [ 22:48:01 DEBUG { 22:48:01 DEBUG "result": [ 22:48:01 DEBUG "resultType": "vector", 22:48:01 DEBUG "result": [ 22:48:01 DEBUG "resultType": "vector", 22:48:01 DEBUG "result": [ 22:48:01 DEBUG "status": "success", 22:48:01 DEBUG "result": [ 22:48:01 DEBUG "status": "success", 22:48:01 DEBUG { 22:48:01 DEBUG "result": [ 22:48:01 DEBUG "result": [ 22:48:01 DEBUG "status": "success", 22:48:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:48:01 DEBUG "result": [ 22:48:01 DEBUG "result": [ 22:48:01 DEBUG "status": "success", 22:48:01 DEBUG "status": "success", 22:48:01 DEBUG "status": "success", 22:48:01 DEBUG "result": [ 22:48:01 DEBUG "result": [ 22:48:01 DEBUG "result": [ 22:48:01 DEBUG "result": [ 22:48:01 DEBUG "result": [ 22:48:01 DEBUG { 22:48:01 DEBUG "status": "success", 22:48:01 DEBUG { 22:48:01 DEBUG "result": [ 22:48:01 DEBUG { 22:48:01 DEBUG "result": [ 22:48:01 DEBUG { 22:48:01 DEBUG "data": { 22:48:01 DEBUG { 22:48:01 DEBUG "metric": { 22:48:01 DEBUG "status": "success", 22:48:01 DEBUG "data": { 22:48:01 DEBUG { 22:48:01 DEBUG "data": { 22:48:01 DEBUG ----- output ----- 22:48:01 DEBUG { 22:48:01 DEBUG { 22:48:01 DEBUG "data": { 22:48:01 DEBUG "data": { 22:48:01 DEBUG "data": { 22:48:01 DEBUG { 22:48:01 DEBUG { 22:48:01 DEBUG { 22:48:01 DEBUG { 22:48:01 DEBUG { 22:48:01 DEBUG "metric": { 22:48:01 DEBUG "data": { 22:48:01 DEBUG "metric": { 22:48:01 DEBUG { 22:48:01 DEBUG "metric": { 22:48:01 DEBUG { 22:48:01 DEBUG "metric": { 22:48:01 DEBUG "resultType": "vector", 22:48:01 DEBUG "data": { 22:48:01 DEBUG "pod": "am-555f4db595-fz8sn" 22:48:01 DEBUG "resultType": "vector", 22:48:01 DEBUG { 22:48:01 DEBUG "metric": { 22:48:01 DEBUG "resultType": "vector", 22:48:01 DEBUG { 22:48:01 DEBUG "metric": { 22:48:01 DEBUG "metric": { 22:48:01 DEBUG "resultType": "vector", 22:48:01 DEBUG "resultType": "vector", 22:48:01 DEBUG "resultType": "vector", 22:48:01 DEBUG "metric": { 22:48:01 DEBUG "metric": { 22:48:01 DEBUG "metric": { 22:48:01 DEBUG "metric": { 22:48:01 DEBUG "metric": { 22:48:01 DEBUG "pod": "ds-idrepo-2" 22:48:01 DEBUG "resultType": "vector", 22:48:01 DEBUG "pod": "am-555f4db595-fz8sn" 22:48:01 DEBUG "metric": { 22:48:01 DEBUG "pod": "ds-idrepo-2" 22:48:01 DEBUG "metric": { 22:48:01 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:48:01 DEBUG "result": [ 22:48:01 DEBUG "resultType": "vector", 22:48:01 DEBUG }, 22:48:01 DEBUG "result": [ 22:48:01 DEBUG "metric": { 22:48:01 DEBUG "pod": "am-555f4db595-fz8sn" 22:48:01 DEBUG "result": [ 22:48:01 DEBUG "status": "success", 22:48:01 DEBUG "instance": "10.142.0.107:9100" 22:48:01 DEBUG "pod": "am-555f4db595-fz8sn" 22:48:01 DEBUG "result": [ 22:48:01 DEBUG "result": [ 22:48:01 DEBUG "result": [ 22:48:01 DEBUG "pod": "am-555f4db595-fz8sn" 22:48:01 DEBUG "pod": "ds-idrepo-2" 22:48:01 DEBUG "pod": "ds-cts-1" 22:48:01 DEBUG "pod": "ds-idrepo-2" 22:48:01 DEBUG "instance": "10.142.0.107:9100" 22:48:01 DEBUG }, 22:48:01 DEBUG "result": [ 22:48:01 DEBUG }, 22:48:01 DEBUG "pod": "ds-idrepo-2" 22:48:01 DEBUG }, 22:48:01 DEBUG }, 22:48:01 DEBUG "pod": "ds-cts-1" 22:48:01 DEBUG { 22:48:01 DEBUG "result": [ 22:48:01 DEBUG "value": [ 22:48:01 DEBUG { 22:48:01 DEBUG "pod": "am-555f4db595-fz8sn" 22:48:01 DEBUG }, 22:48:01 DEBUG { 22:48:01 DEBUG "data": { 22:48:01 DEBUG }, 22:48:01 DEBUG }, 22:48:01 DEBUG { 22:48:01 DEBUG { 22:48:01 DEBUG { 22:48:01 DEBUG }, 22:48:01 DEBUG }, 22:48:01 DEBUG }, 22:48:01 DEBUG }, 22:48:01 DEBUG }, 22:48:01 DEBUG "value": [ 22:48:01 DEBUG { 22:48:01 DEBUG "value": [ 22:48:01 DEBUG }, 22:48:01 DEBUG "value": [ 22:48:01 DEBUG "value": [ 22:48:01 DEBUG }, 22:48:01 DEBUG "metric": { 22:48:01 DEBUG { 22:48:01 DEBUG 22:48:01 DEBUG "metric": { 22:48:01 DEBUG }, 22:48:01 DEBUG "value": [ 22:48:01 DEBUG "metric": { 22:48:01 DEBUG "resultType": "vector", 22:48:01 DEBUG "value": [ 22:48:01 DEBUG "value": [ 22:48:01 DEBUG "metric": { 22:48:01 DEBUG "metric": { 22:48:01 DEBUG "metric": { 22:48:01 DEBUG "value": [ 22:48:01 DEBUG "value": [ 22:48:01 DEBUG "value": [ 22:48:01 DEBUG "value": [ 22:48:01 DEBUG "value": [ 22:48:01 DEBUG 22:48:01 DEBUG "metric": { 22:48:01 DEBUG 22:48:01 DEBUG "value": [ 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG "value": [ 22:48:01 DEBUG "instance": "10.142.0.107:9100" 22:48:01 DEBUG "metric": { 22:48:01 DEBUG 22:48:01 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:48:01 DEBUG "value": [ 22:48:01 DEBUG 22:48:01 DEBUG "pod": "ds-idrepo-2" 22:48:01 DEBUG }, 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG "pod": "ds-idrepo-2" 22:48:01 DEBUG "pod": "ds-cts-1" 22:48:01 DEBUG "instance": "10.142.0.107:9100" 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG "pod": "am-555f4db595-fz8sn" 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG }, 22:48:01 DEBUG "pod": "ds-cts-1" 22:48:01 DEBUG 22:48:01 DEBUG }, 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG "result": [ 22:48:01 DEBUG "value": [ 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG }, 22:48:01 DEBUG }, 22:48:01 DEBUG }, 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG }, 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG "value": [ 22:48:01 DEBUG }, 22:48:01 DEBUG [...] 22:48:01 DEBUG "value": [ 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG { 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG "value": [ 22:48:01 DEBUG "value": [ 22:48:01 DEBUG "value": [ 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG [...] 22:48:01 DEBUG "value": [ 22:48:01 DEBUG [...] 22:48:01 DEBUG 22:48:01 DEBUG [...] 22:48:01 DEBUG [...] 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG "value": [ 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG [...] 22:48:01 DEBUG "metric": { 22:48:01 DEBUG 22:48:01 DEBUG [...] 22:48:01 DEBUG [...] 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG [...] 22:48:01 DEBUG [...] 22:48:01 DEBUG [...] 22:48:01 DEBUG [...] 22:48:01 DEBUG [...] 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG [...] 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG [...] 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG [...] 22:48:01 DEBUG 22:48:01 DEBUG "instance": "10.142.0.107:9100" 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG "pod": "am-555f4db595-kzxsp" 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG }, 22:48:01 DEBUG [...] 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG [...] 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG }, 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG "value": [ 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG [...] 22:48:01 DEBUG [...] 22:48:01 DEBUG [...] 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG "pod": "ds-idrepo-0" 22:48:01 DEBUG [...] 22:48:01 DEBUG "pod": "am-555f4db595-kzxsp" 22:48:01 DEBUG 22:48:01 DEBUG "pod": "ds-idrepo-0" 22:48:01 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG [...] 22:48:01 DEBUG [...] 22:48:01 DEBUG "value": [ 22:48:01 DEBUG 22:48:01 DEBUG "pod": "am-555f4db595-kzxsp" 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG "instance": "10.142.0.43:9100" 22:48:01 DEBUG "pod": "am-555f4db595-kzxsp" 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG "pod": "am-555f4db595-kzxsp" 22:48:01 DEBUG "pod": "ds-idrepo-0" 22:48:01 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:48:01 DEBUG "pod": "ds-idrepo-0" 22:48:01 DEBUG "instance": "10.142.0.43:9100" 22:48:01 DEBUG }, 22:48:01 DEBUG 22:48:01 DEBUG }, 22:48:01 DEBUG "pod": "ds-idrepo-0" 22:48:01 DEBUG }, 22:48:01 DEBUG }, 22:48:01 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG 1713304081, 22:48:01 DEBUG "pod": "am-555f4db595-kzxsp" 22:48:01 DEBUG }, 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG }, 22:48:01 DEBUG }, 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG }, 22:48:01 DEBUG }, 22:48:01 DEBUG }, 22:48:01 DEBUG }, 22:48:01 DEBUG }, 22:48:01 DEBUG "value": [ 22:48:01 DEBUG 22:48:01 DEBUG "value": [ 22:48:01 DEBUG }, 22:48:01 DEBUG "value": [ 22:48:01 DEBUG "value": [ 22:48:01 DEBUG }, 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG "0.027" 22:48:01 DEBUG }, 22:48:01 DEBUG "value": [ 22:48:01 DEBUG 22:48:01 DEBUG "pod": "ds-idrepo-0" 22:48:01 DEBUG }, 22:48:01 DEBUG "value": [ 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG "value": [ 22:48:01 DEBUG 1713304081, 22:48:01 DEBUG "value": [ 22:48:01 DEBUG "value": [ 22:48:01 DEBUG "value": [ 22:48:01 DEBUG 1713304081, 22:48:01 DEBUG 22:48:01 DEBUG 1713304081, 22:48:01 DEBUG "value": [ 22:48:01 DEBUG 1713304081, 22:48:01 DEBUG 1713304081, 22:48:01 DEBUG "value": [ 22:48:01 DEBUG "instance": "10.142.0.43:9100" 22:48:01 DEBUG 22:48:01 DEBUG 22:48:01 DEBUG ] 22:48:01 DEBUG "value": [ 22:48:01 DEBUG 1713304081, 22:48:01 DEBUG [...] 22:48:01 DEBUG "value": [ 22:48:01 DEBUG "value": [ 22:48:01 DEBUG 1713304081, 22:48:01 DEBUG "pod": "ds-cts-2" 22:48:01 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:48:01 DEBUG "instance": "10.142.0.43:9100" 22:48:01 DEBUG "2164.366666666667" 22:48:01 DEBUG "value": [ 22:48:01 DEBUG 1713304081, 22:48:01 DEBUG 1713304081, 22:48:01 DEBUG 1713304081, 22:48:01 DEBUG "0" 22:48:01 DEBUG "pod": "am-555f4db595-kzxsp" 22:48:01 DEBUG "1989.1" 22:48:01 DEBUG 1713304081, 22:48:01 DEBUG "0" 22:48:01 DEBUG "6.726712329846738" 22:48:01 DEBUG 1713304081, 22:48:01 DEBUG }, 22:48:01 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:48:01 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:48:01 DEBUG } 22:48:01 DEBUG 1713304081, 22:48:01 DEBUG "2164.1666666666665" 22:48:01 DEBUG 22:48:01 DEBUG 1713304081, 22:48:01 DEBUG 1713304081, 22:48:01 DEBUG "0" 22:48:01 DEBUG }, 22:48:01 DEBUG }, 22:48:01 DEBUG }, 22:48:01 DEBUG ] 22:48:01 DEBUG 1713304081, 22:48:01 DEBUG "0" 22:48:01 DEBUG "0" 22:48:01 DEBUG "48983013.36666666" 22:48:01 DEBUG ] 22:48:01 DEBUG }, 22:48:01 DEBUG ] 22:48:01 DEBUG "0" 22:48:01 DEBUG ] 22:48:01 DEBUG ] 22:48:01 DEBUG "0" 22:48:01 DEBUG "value": [ 22:48:01 DEBUG }, 22:48:01 DEBUG }, 22:48:01 DEBUG ] 22:48:01 DEBUG "3533401" 22:48:01 DEBUG ] 22:48:01 DEBUG 22:48:01 DEBUG "19971484672" 22:48:01 DEBUG "0" 22:48:01 DEBUG ] 22:48:01 DEBUG "value": [ 22:48:01 DEBUG "value": [ 22:48:01 DEBUG "value": [ 22:48:01 DEBUG } 22:48:01 DEBUG "0.00002497341803209604" 22:48:01 DEBUG ] 22:48:01 DEBUG ] 22:48:01 DEBUG ] 22:48:01 DEBUG } 22:48:01 DEBUG } 22:48:01 DEBUG } 22:48:01 DEBUG ] 22:48:01 DEBUG } 22:48:01 DEBUG } 22:48:01 DEBUG ] 22:48:01 DEBUG 1713304081, 22:48:01 DEBUG "value": [ 22:48:01 DEBUG "value": [ 22:48:01 DEBUG } 22:48:01 DEBUG ] 22:48:01 DEBUG } 22:48:01 DEBUG 22:48:01 DEBUG ] 22:48:01 DEBUG ] 22:48:01 DEBUG } 22:48:01 DEBUG 1713304081, 22:48:01 DEBUG 1713304081, 22:48:01 DEBUG 1713304081, 22:48:01 DEBUG ] 22:48:01 DEBUG ] 22:48:01 DEBUG } 22:48:01 DEBUG } 22:48:01 DEBUG "value": [ 22:48:01 DEBUG ] 22:48:01 DEBUG ] 22:48:01 DEBUG ] 22:48:01 DEBUG } 22:48:01 DEBUG ] 22:48:01 DEBUG ] 22:48:01 DEBUG } 22:48:01 DEBUG "0.0005333333333330377" 22:48:01 DEBUG 1713304081, 22:48:01 DEBUG 1713304081, 22:48:01 DEBUG } 22:48:01 DEBUG } 22:48:01 DEBUG ] 22:48:01 DEBUG "instance": "10.142.0.43:9100" 22:48:01 DEBUG } 22:48:01 DEBUG } 22:48:01 DEBUG ] 22:48:01 DEBUG "0" 22:48:01 DEBUG "0" 22:48:01 DEBUG "48974742.4" 22:48:01 DEBUG } 22:48:01 DEBUG } 22:48:01 DEBUG ] 22:48:01 DEBUG ] 22:48:01 DEBUG 1713304081, 22:48:01 DEBUG "2164.8333333333335" 22:48:01 DEBUG } 22:48:01 DEBUG } 22:48:01 DEBUG ] 22:48:01 DEBUG } 22:48:01 DEBUG } 22:48:01 DEBUG ] 22:48:01 DEBUG } 22:48:01 DEBUG "0" 22:48:01 DEBUG "1829429248" 22:48:01 DEBUG ] 22:48:01 DEBUG } 22:48:01 DEBUG }, 22:48:01 DEBUG ] 22:48:01 DEBUG ] 22:48:01 DEBUG } 22:48:01 DEBUG ] 22:48:01 DEBUG ] 22:48:01 DEBUG ] 22:48:01 DEBUG } 22:48:01 DEBUG ] 22:48:01 DEBUG } 22:48:01 DEBUG } 22:48:01 DEBUG ] 22:48:01 DEBUG } 22:48:01 DEBUG } 22:48:01 DEBUG } 22:48:01 DEBUG } 22:48:01 DEBUG } 22:48:01 DEBUG } 22:48:01 DEBUG ] 22:48:01 DEBUG ] 22:48:01 DEBUG ] 22:48:01 DEBUG } 22:48:01 DEBUG } 22:48:01 DEBUG "value": [ 22:48:01 DEBUG } 22:48:01 DEBUG } 22:48:01 DEBUG } 22:48:01 DEBUG } 22:48:01 DEBUG } 22:48:01 DEBUG } 22:48:01 DEBUG ] 22:48:01 DEBUG } 22:48:01 DEBUG } 22:48:01 DEBUG } 22:48:01 DEBUG } 22:48:01 DEBUG } 22:48:01 DEBUG } 22:48:01 DEBUG ] 22:48:01 DEBUG } 22:48:01 DEBUG } 22:48:01 DEBUG } 22:48:01 DEBUG } 22:48:01 DEBUG 1713304081, 22:48:01 DEBUG } 22:48:01 DEBUG } 22:48:01 DEBUG ] 22:48:01 DEBUG ] 22:48:01 DEBUG } 22:48:01 DEBUG } 22:48:01 DEBUG ] 22:48:01 DEBUG } 22:48:01 DEBUG } 22:48:01 DEBUG ] 22:48:01 DEBUG ] 22:48:01 DEBUG "0" 22:48:01 DEBUG } 22:48:01 DEBUG } 22:48:01 DEBUG } 22:48:01 DEBUG } 22:48:01 DEBUG } 22:48:01 DEBUG } 22:48:01 DEBUG } 22:48:01 DEBUG ] 22:48:01 DEBUG } 22:48:01 DEBUG } 22:48:01 DEBUG } 22:48:01 DEBUG } 22:48:01 DEBUG } 22:48:01 DEBUG } 22:48:01 DEBUG ] 22:48:01 DEBUG } 22:48:01 DEBUG } 22:48:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713304137" 22:48:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304137" 22:48:57 ERROR [http_cmd]: ERROR 22:48:57 INFO [http_cmd]: http status code OK 22:48:57 ERROR --- status code --- 22:48:57 DEBUG --- status code --- 22:48:57 ERROR http status code is 400 (expected [200, 401, 403, 404]) 22:48:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:48:57 ERROR --- http response --- 22:48:57 DEBUG --- http response --- 22:48:57 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 22:48:57 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 22:48:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 22:48:59 INFO 22:48:59 INFO [loop_until]: kubectl --namespace=xlou top pods 22:48:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:48:59 INFO [loop_until]: OK (rc = 0) 22:48:59 DEBUG --- stdout --- 22:48:59 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 9Mi am-555f4db595-4lbss 12414m 8687Mi am-555f4db595-fz8sn 13378m 8559Mi am-555f4db595-kzxsp 12912m 8682Mi ds-cts-0 19472m 12781Mi ds-cts-1 17854m 13033Mi ds-cts-2 17833m 12965Mi ds-idrepo-0 5615m 5028Mi ds-idrepo-1 5607m 2908Mi ds-idrepo-2 5532m 3220Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 4m 1207Mi idm-b85f9d978-rl76f 4m 1226Mi lodemon-bc756c647-cr7x6 2m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-7c9b667974-9mr2s 6726m 1529Mi 22:48:59 DEBUG --- stderr --- 22:48:59 DEBUG 22:49:01 INFO 22:49:01 INFO [loop_until]: kubectl --namespace=xlou top node 22:49:01 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:49:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713304141" 22:49:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713304141" 22:49:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304141" 22:49:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304141" 22:49:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304141" 22:49:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304141" 22:49:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304141" 22:49:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304141" 22:49:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304141" 22:49:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304141" 22:49:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304141" 22:49:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304141" 22:49:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304141" 22:49:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304141" 22:49:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304141" 22:49:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304141" 22:49:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304141" 22:49:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304141" 22:49:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304141" 22:49:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304141" 22:49:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713304141" 22:49:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304141" 22:49:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713304141" 22:49:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304141" 22:49:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304141" 22:49:01 INFO [loop_until]: OK (rc = 0) 22:49:01 INFO [http_cmd]: http status code OK 22:49:01 INFO [http_cmd]: http status code OK 22:49:01 INFO [http_cmd]: http status code OK 22:49:01 INFO [http_cmd]: http status code OK 22:49:01 INFO [http_cmd]: http status code OK 22:49:01 INFO [http_cmd]: http status code OK 22:49:01 INFO [http_cmd]: http status code OK 22:49:01 INFO [http_cmd]: http status code OK 22:49:01 INFO [http_cmd]: http status code OK 22:49:01 INFO [http_cmd]: http status code OK 22:49:01 INFO [http_cmd]: http status code OK 22:49:01 INFO [http_cmd]: http status code OK 22:49:01 INFO [http_cmd]: http status code OK 22:49:01 INFO [http_cmd]: http status code OK 22:49:01 INFO [http_cmd]: http status code OK 22:49:01 INFO [http_cmd]: http status code OK 22:49:01 INFO [http_cmd]: http status code OK 22:49:01 INFO [http_cmd]: http status code OK 22:49:01 DEBUG --- stdout --- 22:49:01 DEBUG --- status code --- 22:49:01 INFO [http_cmd]: http status code OK 22:49:01 DEBUG --- status code --- 22:49:01 INFO [http_cmd]: http status code OK 22:49:01 DEBUG --- status code --- 22:49:01 DEBUG --- status code --- 22:49:01 DEBUG --- status code --- 22:49:01 DEBUG --- status code --- 22:49:01 DEBUG --- status code --- 22:49:01 DEBUG --- status code --- 22:49:01 DEBUG --- status code --- 22:49:01 DEBUG --- status code --- 22:49:01 DEBUG --- status code --- 22:49:01 DEBUG --- status code --- 22:49:01 DEBUG --- status code --- 22:49:01 INFO [http_cmd]: http status code OK 22:49:01 INFO [http_cmd]: http status code OK 22:49:01 DEBUG --- status code --- 22:49:01 DEBUG --- status code --- 22:49:01 DEBUG --- status code --- 22:49:01 DEBUG --- status code --- 22:49:01 DEBUG --- status code --- 22:49:01 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 5077Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 18470m 61% 15601Mi 13% gke-xlou-cdm-default-pool-1b0769c8-nhlf 13199m 44% 10792Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 79m 0% 2652Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 5452m 18% 6572Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 21053m 70% 17344Mi 15% gke-xlou-cdm-default-pool-1b0769c8-xnmp 58m 0% 3148Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 12982m 43% 10503Mi 9% gke-xlou-cdm-ds-0f601751-hmm7 6009m 20% 5657Mi 5% gke-xlou-cdm-ds-0f601751-j488 18312m 61% 14926Mi 13% gke-xlou-cdm-ds-0f601751-mc5d 11644m 39% 3420Mi 3% gke-xlou-cdm-ds-0f601751-nj6q 13905m 46% 10830Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 5965m 19% 7346Mi 6% gke-xlou-cdm-frontend-a8771548-i82n 6913m 43% 6941Mi 11% 22:49:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:49:01 DEBUG --- status code --- 22:49:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:49:01 DEBUG --- status code --- 22:49:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:49:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:49:01 DEBUG --- http response --- 22:49:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:49:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:49:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:49:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:49:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:49:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:49:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:49:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:49:01 DEBUG --- status code --- 22:49:01 DEBUG --- status code --- 22:49:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:49:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:49:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:49:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:49:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:49:01 DEBUG --- stderr --- 22:49:01 DEBUG --- http response --- 22:49:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:49:01 DEBUG --- http response --- 22:49:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:49:01 DEBUG --- http response --- 22:49:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:49:01 DEBUG --- http response --- 22:49:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:49:01 DEBUG --- http response --- 22:49:01 DEBUG --- http response --- 22:49:01 DEBUG --- http response --- 22:49:01 DEBUG --- http response --- 22:49:01 DEBUG --- http response --- 22:49:01 DEBUG --- http response --- 22:49:01 DEBUG --- http response --- 22:49:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:49:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:49:01 DEBUG --- http response --- 22:49:01 DEBUG --- http response --- 22:49:01 DEBUG --- http response --- 22:49:01 DEBUG --- http response --- 22:49:01 DEBUG --- http response --- 22:49:01 DEBUG 22:49:01 DEBUG --- http response --- 22:49:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:49:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:49:01 INFO [http_cmd]: http status code OK 22:49:01 DEBUG --- http response --- 22:49:01 DEBUG --- http response --- 22:49:01 INFO [http_cmd]: http status code OK 22:49:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:49:01 DEBUG ----- output ----- 22:49:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:49:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:49:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:49:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:49:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:49:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:49:01 INFO [http_cmd]: http status code OK 22:49:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:49:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:49:01 DEBUG ----- output ----- 22:49:01 DEBUG --- http response --- 22:49:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:49:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:49:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:49:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:49:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:49:01 DEBUG ----- output ----- 22:49:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:49:01 DEBUG ----- output ----- 22:49:01 DEBUG --- status code --- 22:49:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:49:01 DEBUG --- status code --- 22:49:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:49:01 DEBUG ----- output ----- 22:49:01 DEBUG { 22:49:01 DEBUG ----- output ----- 22:49:01 DEBUG ----- output ----- 22:49:01 DEBUG ----- output ----- 22:49:01 DEBUG ----- output ----- 22:49:01 DEBUG ----- output ----- 22:49:01 DEBUG ----- output ----- 22:49:01 DEBUG --- status code --- 22:49:01 DEBUG ----- output ----- 22:49:01 DEBUG --- http response --- 22:49:01 DEBUG { 22:49:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:49:01 DEBUG ----- output ----- 22:49:01 DEBUG ----- output ----- 22:49:01 DEBUG ----- output ----- 22:49:01 DEBUG ----- output ----- 22:49:01 DEBUG ----- output ----- 22:49:01 DEBUG { 22:49:01 DEBUG ----- output ----- 22:49:01 DEBUG { 22:49:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:49:01 DEBUG ----- output ----- 22:49:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:49:01 DEBUG ----- output ----- 22:49:01 DEBUG { 22:49:01 DEBUG "status": "success", 22:49:01 DEBUG { 22:49:01 DEBUG { 22:49:01 DEBUG { 22:49:01 DEBUG { 22:49:01 DEBUG { 22:49:01 DEBUG { 22:49:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:49:01 DEBUG { 22:49:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:49:01 DEBUG "status": "success", 22:49:01 DEBUG ----- output ----- 22:49:01 DEBUG { 22:49:01 DEBUG { 22:49:01 DEBUG { 22:49:01 DEBUG { 22:49:01 DEBUG { 22:49:01 DEBUG "status": "success", 22:49:01 DEBUG { 22:49:01 DEBUG "status": "success", 22:49:01 DEBUG --- http response --- 22:49:01 DEBUG { 22:49:01 DEBUG --- http response --- 22:49:01 DEBUG { 22:49:01 DEBUG "status": "success", 22:49:01 DEBUG "data": { 22:49:01 DEBUG "status": "success", 22:49:01 DEBUG "status": "success", 22:49:01 DEBUG "status": "success", 22:49:01 DEBUG "status": "success", 22:49:01 DEBUG "status": "success", 22:49:01 DEBUG "status": "success", 22:49:01 DEBUG --- http response --- 22:49:01 DEBUG "status": "success", 22:49:01 DEBUG ----- output ----- 22:49:01 DEBUG "data": { 22:49:01 DEBUG { 22:49:01 DEBUG "status": "success", 22:49:01 DEBUG "status": "success", 22:49:01 DEBUG "status": "success", 22:49:01 DEBUG "status": "success", 22:49:01 DEBUG "status": "success", 22:49:01 DEBUG "data": { 22:49:01 DEBUG "status": "success", 22:49:01 DEBUG "data": { 22:49:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:49:01 DEBUG "status": "success", 22:49:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:49:01 DEBUG "status": "success", 22:49:01 DEBUG "data": { 22:49:01 DEBUG "resultType": "vector", 22:49:01 DEBUG "data": { 22:49:01 DEBUG "data": { 22:49:01 DEBUG "data": { 22:49:01 DEBUG "data": { 22:49:01 DEBUG "data": { 22:49:01 DEBUG "data": { 22:49:01 DEBUG "data": { 22:49:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:49:01 DEBUG { 22:49:01 DEBUG "resultType": "vector", 22:49:01 DEBUG "status": "success", 22:49:01 DEBUG "data": { 22:49:01 DEBUG "data": { 22:49:01 DEBUG "data": { 22:49:01 DEBUG "data": { 22:49:01 DEBUG "data": { 22:49:01 DEBUG "resultType": "vector", 22:49:01 DEBUG "data": { 22:49:01 DEBUG "resultType": "vector", 22:49:01 DEBUG ----- output ----- 22:49:01 DEBUG "data": { 22:49:01 DEBUG ----- output ----- 22:49:01 DEBUG "data": { 22:49:01 DEBUG "resultType": "vector", 22:49:01 DEBUG "result": [ 22:49:01 DEBUG "resultType": "vector", 22:49:01 DEBUG "resultType": "vector", 22:49:01 DEBUG "resultType": "vector", 22:49:01 DEBUG "resultType": "vector", 22:49:01 DEBUG "resultType": "vector", 22:49:01 DEBUG "resultType": "vector", 22:49:01 DEBUG "resultType": "vector", 22:49:01 DEBUG ----- output ----- 22:49:01 DEBUG "status": "success", 22:49:01 DEBUG "result": [ 22:49:01 DEBUG "data": { 22:49:01 DEBUG "resultType": "vector", 22:49:01 DEBUG "resultType": "vector", 22:49:01 DEBUG "resultType": "vector", 22:49:01 DEBUG "resultType": "vector", 22:49:01 DEBUG "resultType": "vector", 22:49:01 DEBUG "result": [ 22:49:01 DEBUG "resultType": "vector", 22:49:01 DEBUG "result": [ 22:49:01 DEBUG { 22:49:01 DEBUG "resultType": "vector", 22:49:01 DEBUG { 22:49:01 DEBUG "resultType": "vector", 22:49:01 DEBUG "result": [ 22:49:01 DEBUG { 22:49:01 DEBUG "result": [ 22:49:01 DEBUG "result": [ 22:49:01 DEBUG "result": [ 22:49:01 DEBUG "result": [ 22:49:01 DEBUG "result": [ 22:49:01 DEBUG "result": [ 22:49:01 DEBUG "result": [ 22:49:01 DEBUG { 22:49:01 DEBUG "data": { 22:49:01 DEBUG { 22:49:01 DEBUG "resultType": "vector", 22:49:01 DEBUG "result": [ 22:49:01 DEBUG "result": [ 22:49:01 DEBUG "result": [ 22:49:01 DEBUG "result": [ 22:49:01 DEBUG "result": [ 22:49:01 DEBUG { 22:49:01 DEBUG "result": [ 22:49:01 DEBUG { 22:49:01 DEBUG "status": "success", 22:49:01 DEBUG "result": [ 22:49:01 DEBUG "status": "success", 22:49:01 DEBUG "result": [ 22:49:01 DEBUG { 22:49:01 DEBUG "metric": { 22:49:01 DEBUG { 22:49:01 DEBUG { 22:49:01 DEBUG { 22:49:01 DEBUG { 22:49:01 DEBUG { 22:49:01 DEBUG { 22:49:01 DEBUG { 22:49:01 DEBUG "status": "success", 22:49:01 DEBUG "resultType": "vector", 22:49:01 DEBUG "metric": { 22:49:01 DEBUG "result": [ 22:49:01 DEBUG { 22:49:01 DEBUG { 22:49:01 DEBUG { 22:49:01 DEBUG { 22:49:01 DEBUG { 22:49:01 DEBUG "metric": { 22:49:01 DEBUG { 22:49:01 DEBUG "metric": { 22:49:01 DEBUG "data": { 22:49:01 DEBUG { 22:49:01 DEBUG "data": { 22:49:01 DEBUG { 22:49:01 DEBUG "metric": { 22:49:01 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:49:01 DEBUG "metric": { 22:49:01 DEBUG "metric": { 22:49:01 DEBUG "metric": { 22:49:01 DEBUG "metric": { 22:49:01 DEBUG "metric": { 22:49:01 DEBUG "metric": { 22:49:01 DEBUG "metric": { 22:49:01 DEBUG "data": { 22:49:01 DEBUG "result": [ 22:49:01 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:49:01 DEBUG { 22:49:01 DEBUG "metric": { 22:49:01 DEBUG "metric": { 22:49:01 DEBUG "metric": { 22:49:01 DEBUG "metric": { 22:49:01 DEBUG "metric": { 22:49:01 DEBUG "pod": "am-555f4db595-fz8sn" 22:49:01 DEBUG "metric": { 22:49:01 DEBUG "pod": "ds-idrepo-2" 22:49:01 DEBUG "resultType": "vector", 22:49:01 DEBUG "metric": { 22:49:01 DEBUG "resultType": "vector", 22:49:01 DEBUG "metric": { 22:49:01 DEBUG "pod": "am-555f4db595-fz8sn" 22:49:01 DEBUG }, 22:49:01 DEBUG "pod": "am-555f4db595-fz8sn" 22:49:01 DEBUG "pod": "am-555f4db595-fz8sn" 22:49:01 DEBUG "pod": "am-555f4db595-fz8sn" 22:49:01 DEBUG "pod": "ds-idrepo-2" 22:49:01 DEBUG "instance": "10.142.0.107:9100" 22:49:01 DEBUG "pod": "ds-idrepo-2" 22:49:01 DEBUG "pod": "ds-idrepo-2" 22:49:01 DEBUG "resultType": "vector", 22:49:01 DEBUG { 22:49:01 DEBUG }, 22:49:01 DEBUG "metric": { 22:49:01 DEBUG "instance": "10.142.0.107:9100" 22:49:01 DEBUG "pod": "am-555f4db595-fz8sn" 22:49:01 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:49:01 DEBUG "instance": "10.142.0.107:9100" 22:49:01 DEBUG "instance": "10.142.0.107:9100" 22:49:01 DEBUG }, 22:49:01 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:49:01 DEBUG }, 22:49:01 DEBUG "result": [ 22:49:01 DEBUG "pod": "ds-idrepo-2" 22:49:01 DEBUG "result": [ 22:49:01 DEBUG "pod": "ds-idrepo-2" 22:49:01 DEBUG }, 22:49:01 DEBUG "value": [ 22:49:01 DEBUG }, 22:49:01 DEBUG }, 22:49:01 DEBUG }, 22:49:01 DEBUG }, 22:49:01 DEBUG }, 22:49:01 DEBUG }, 22:49:01 DEBUG }, 22:49:01 DEBUG "result": [ 22:49:01 DEBUG "metric": { 22:49:01 DEBUG "value": [ 22:49:01 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:49:01 DEBUG }, 22:49:01 DEBUG }, 22:49:01 DEBUG }, 22:49:01 DEBUG }, 22:49:01 DEBUG }, 22:49:01 DEBUG "value": [ 22:49:01 DEBUG }, 22:49:01 DEBUG "value": [ 22:49:01 DEBUG { 22:49:01 DEBUG }, 22:49:01 DEBUG { 22:49:01 DEBUG }, 22:49:01 DEBUG "value": [ 22:49:01 DEBUG 22:49:01 DEBUG "value": [ 22:49:01 DEBUG "value": [ 22:49:01 DEBUG "value": [ 22:49:01 DEBUG "value": [ 22:49:01 DEBUG "value": [ 22:49:01 DEBUG "value": [ 22:49:01 DEBUG "value": [ 22:49:01 DEBUG { 22:49:01 DEBUG "pod": "ds-idrepo-2" 22:49:01 DEBUG 22:49:01 DEBUG }, 22:49:01 DEBUG "value": [ 22:49:01 DEBUG "value": [ 22:49:01 DEBUG "value": [ 22:49:01 DEBUG "value": [ 22:49:01 DEBUG "value": [ 22:49:01 DEBUG 22:49:01 DEBUG "value": [ 22:49:01 DEBUG 22:49:01 DEBUG "metric": { 22:49:01 DEBUG "value": [ 22:49:01 DEBUG "metric": { 22:49:01 DEBUG "value": [ 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG "metric": { 22:49:01 DEBUG }, 22:49:01 DEBUG 22:49:01 DEBUG "value": [ 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG "pod": "am-555f4db595-fz8sn" 22:49:01 DEBUG 22:49:01 DEBUG "instance": "10.142.0.107:9100" 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG [...] 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:49:01 DEBUG "value": [ 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG }, 22:49:01 DEBUG 22:49:01 DEBUG }, 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG }, 22:49:01 DEBUG 22:49:01 DEBUG [...] 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG [...] 22:49:01 DEBUG 22:49:01 DEBUG [...] 22:49:01 DEBUG "value": [ 22:49:01 DEBUG 22:49:01 DEBUG "value": [ 22:49:01 DEBUG 22:49:01 DEBUG [...] 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG [...] 22:49:01 DEBUG [...] 22:49:01 DEBUG [...] 22:49:01 DEBUG [...] 22:49:01 DEBUG [...] 22:49:01 DEBUG [...] 22:49:01 DEBUG "value": [ 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG [...] 22:49:01 DEBUG [...] 22:49:01 DEBUG [...] 22:49:01 DEBUG [...] 22:49:01 DEBUG [...] 22:49:01 DEBUG 22:49:01 DEBUG [...] 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG [...] 22:49:01 DEBUG 22:49:01 DEBUG [...] 22:49:01 DEBUG 22:49:01 DEBUG [...] 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG [...] 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG [...] 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG }, 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG "pod": "login-ui-5fdf49fc66-2wnx7" 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG "instance": "10.142.0.43:9100" 22:49:01 DEBUG 22:49:01 DEBUG "pod": "am-555f4db595-kzxsp" 22:49:01 DEBUG 22:49:01 DEBUG "pod": "ds-idrepo-0" 22:49:01 DEBUG [...] 22:49:01 DEBUG 22:49:01 DEBUG [...] 22:49:01 DEBUG 22:49:01 DEBUG "pod": "am-555f4db595-kzxsp" 22:49:01 DEBUG 22:49:01 DEBUG "value": [ 22:49:01 DEBUG "pod": "am-555f4db595-kzxsp" 22:49:01 DEBUG "pod": "am-555f4db595-kzxsp" 22:49:01 DEBUG "pod": "ds-idrepo-0" 22:49:01 DEBUG "instance": "10.142.0.43:9100" 22:49:01 DEBUG "pod": "ds-idrepo-0" 22:49:01 DEBUG "pod": "ds-idrepo-0" 22:49:01 DEBUG [...] 22:49:01 DEBUG 22:49:01 DEBUG }, 22:49:01 DEBUG 22:49:01 DEBUG "instance": "10.142.0.43:9100" 22:49:01 DEBUG "pod": "am-555f4db595-kzxsp" 22:49:01 DEBUG "pod": "login-ui-5fdf49fc66-2wnx7" 22:49:01 DEBUG }, 22:49:01 DEBUG 22:49:01 DEBUG }, 22:49:01 DEBUG "pod": "login-ui-5fdf49fc66-2wnx7" 22:49:01 DEBUG }, 22:49:01 DEBUG 22:49:01 DEBUG "pod": "ds-idrepo-0" 22:49:01 DEBUG 22:49:01 DEBUG "pod": "ds-idrepo-0" 22:49:01 DEBUG }, 22:49:01 DEBUG "pod": "am-555f4db595-kzxsp" 22:49:01 DEBUG 1713304141, 22:49:01 DEBUG }, 22:49:01 DEBUG }, 22:49:01 DEBUG }, 22:49:01 DEBUG }, 22:49:01 DEBUG }, 22:49:01 DEBUG }, 22:49:01 DEBUG 22:49:01 DEBUG 22:49:01 DEBUG "value": [ 22:49:01 DEBUG "pod": "login-ui-5fdf49fc66-2wnx7" 22:49:01 DEBUG }, 22:49:01 DEBUG }, 22:49:01 DEBUG }, 22:49:01 DEBUG "value": [ 22:49:01 DEBUG "instance": "10.142.0.43:9100" 22:49:01 DEBUG "value": [ 22:49:01 DEBUG }, 22:49:01 DEBUG "value": [ 22:49:01 DEBUG 22:49:01 DEBUG }, 22:49:01 DEBUG 22:49:01 DEBUG }, 22:49:01 DEBUG "value": [ 22:49:01 DEBUG }, 22:49:01 DEBUG "6.743039091628545" 22:49:01 DEBUG "value": [ 22:49:01 DEBUG "value": [ 22:49:01 DEBUG "value": [ 22:49:01 DEBUG "value": [ 22:49:01 DEBUG "value": [ 22:49:01 DEBUG "value": [ 22:49:01 DEBUG 22:49:01 DEBUG "pod": "ds-cts-2" 22:49:01 DEBUG 1713304141, 22:49:01 DEBUG }, 22:49:01 DEBUG "value": [ 22:49:01 DEBUG "value": [ 22:49:01 DEBUG "value": [ 22:49:01 DEBUG 1713304141, 22:49:01 DEBUG }, 22:49:01 DEBUG 1713304141, 22:49:01 DEBUG "value": [ 22:49:01 DEBUG 1713304141, 22:49:01 DEBUG 22:49:01 DEBUG "value": [ 22:49:01 DEBUG 22:49:01 DEBUG "value": [ 22:49:01 DEBUG 1713304141, 22:49:01 DEBUG "value": [ 22:49:01 DEBUG ] 22:49:01 DEBUG 1713304141, 22:49:01 DEBUG 1713304141, 22:49:01 DEBUG 1713304141, 22:49:01 DEBUG 1713304141, 22:49:01 DEBUG 1713304141, 22:49:01 DEBUG 1713304141, 22:49:01 DEBUG 22:49:01 DEBUG }, 22:49:01 DEBUG "0" 22:49:01 DEBUG "value": [ 22:49:01 DEBUG 1713304141, 22:49:01 DEBUG 1713304141, 22:49:01 DEBUG 1713304141, 22:49:01 DEBUG "19972402176" 22:49:01 DEBUG "value": [ 22:49:01 DEBUG "0.027" 22:49:01 DEBUG 1713304141, 22:49:01 DEBUG "0" 22:49:01 DEBUG "pod": "am-555f4db595-kzxsp" 22:49:01 DEBUG 1713304141, 22:49:01 DEBUG "instance": "10.142.0.43:9100" 22:49:01 DEBUG 1713304141, 22:49:01 DEBUG "1999.5333333333333" 22:49:01 DEBUG 1713304141, 22:49:01 DEBUG } 22:49:01 DEBUG "2169.0666666666666" 22:49:01 DEBUG "2163.4666666666667" 22:49:01 DEBUG "0" 22:49:01 DEBUG "49202920.133333325" 22:49:01 DEBUG "0" 22:49:01 DEBUG "0" 22:49:01 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:49:01 DEBUG "value": [ 22:49:01 DEBUG ] 22:49:01 DEBUG 1713304141, 22:49:01 DEBUG "0.00046666666666700016" 22:49:01 DEBUG "3543115.5" 22:49:01 DEBUG "0.000027408758067784756" 22:49:01 DEBUG ] 22:49:01 DEBUG 1713304141, 22:49:01 DEBUG ] 22:49:01 DEBUG "0" 22:49:01 DEBUG ] 22:49:01 DEBUG }, 22:49:01 DEBUG "0" 22:49:01 DEBUG }, 22:49:01 DEBUG "0" 22:49:01 DEBUG ] 22:49:01 DEBUG "0" 22:49:01 DEBUG ] 22:49:01 DEBUG ] 22:49:01 DEBUG ] 22:49:01 DEBUG ] 22:49:01 DEBUG ] 22:49:01 DEBUG ] 22:49:01 DEBUG ] 22:49:01 DEBUG }, 22:49:01 DEBUG 1713304141, 22:49:01 DEBUG } 22:49:01 DEBUG "0" 22:49:01 DEBUG ] 22:49:01 DEBUG ] 22:49:01 DEBUG ] 22:49:01 DEBUG } 22:49:01 DEBUG "49193734.03333333" 22:49:01 DEBUG } 22:49:01 DEBUG ] 22:49:01 DEBUG } 22:49:01 DEBUG "value": [ 22:49:01 DEBUG ] 22:49:01 DEBUG "value": [ 22:49:01 DEBUG ] 22:49:01 DEBUG } 22:49:01 DEBUG ] 22:49:01 DEBUG } 22:49:01 DEBUG } 22:49:01 DEBUG } 22:49:01 DEBUG ] 22:49:01 DEBUG } 22:49:01 DEBUG } 22:49:01 DEBUG } 22:49:01 DEBUG "value": [ 22:49:01 DEBUG "0" 22:49:01 DEBUG ] 22:49:01 DEBUG ] 22:49:01 DEBUG } 22:49:01 DEBUG } 22:49:01 DEBUG } 22:49:01 DEBUG ] 22:49:01 DEBUG ] 22:49:01 DEBUG ] 22:49:01 DEBUG } 22:49:01 DEBUG ] 22:49:01 DEBUG 1713304141, 22:49:01 DEBUG } 22:49:01 DEBUG 1713304141, 22:49:01 DEBUG } 22:49:01 DEBUG ] 22:49:01 DEBUG } 22:49:01 DEBUG } 22:49:01 DEBUG ] 22:49:01 DEBUG } 22:49:01 DEBUG } 22:49:01 DEBUG ] 22:49:01 DEBUG ] 22:49:01 DEBUG ] 22:49:01 DEBUG 1713304141, 22:49:01 DEBUG ] 22:49:01 DEBUG } 22:49:01 DEBUG } 22:49:01 DEBUG ] 22:49:01 DEBUG ] 22:49:01 DEBUG ] 22:49:01 DEBUG } 22:49:01 DEBUG } 22:49:01 DEBUG } 22:49:01 DEBUG ] 22:49:01 DEBUG } 22:49:01 DEBUG "2164.1666666666665" 22:49:01 DEBUG ] 22:49:01 DEBUG "0.00033333333333333365" 22:49:01 DEBUG ] 22:49:01 DEBUG } 22:49:01 DEBUG ] 22:49:01 DEBUG } 22:49:01 DEBUG ] 22:49:01 DEBUG } 22:49:01 DEBUG } 22:49:01 DEBUG } 22:49:01 DEBUG } 22:49:01 DEBUG "1669627904" 22:49:01 DEBUG } 22:49:01 DEBUG } 22:49:01 DEBUG ] 22:49:01 DEBUG } 22:49:01 DEBUG } 22:49:01 DEBUG } 22:49:01 DEBUG } 22:49:01 DEBUG ] 22:49:01 DEBUG } 22:49:01 DEBUG } 22:49:01 DEBUG } 22:49:01 DEBUG ] 22:49:01 DEBUG } 22:49:01 DEBUG ] 22:49:01 DEBUG } 22:49:01 DEBUG } 22:49:01 DEBUG } 22:49:01 DEBUG } 22:49:01 DEBUG } 22:49:01 DEBUG } 22:49:01 DEBUG } 22:49:01 DEBUG } 22:49:01 DEBUG ] 22:49:01 DEBUG ] 22:49:01 DEBUG } 22:49:01 DEBUG } 22:49:01 DEBUG } 22:49:01 DEBUG } 22:49:01 DEBUG } 22:49:01 DEBUG } 22:49:01 DEBUG } 22:49:01 DEBUG } 22:49:01 DEBUG } 22:49:01 DEBUG } 22:49:01 DEBUG } 22:49:01 DEBUG } 22:49:01 DEBUG } 22:49:01 DEBUG } 22:49:01 DEBUG } 22:49:01 DEBUG } 22:49:01 DEBUG ] 22:49:01 DEBUG ] 22:49:01 DEBUG ] 22:49:01 DEBUG } 22:49:01 DEBUG } 22:49:01 DEBUG } 22:49:01 DEBUG } 22:49:01 DEBUG } 22:49:01 DEBUG } 22:49:01 DEBUG } 22:49:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713304197" 22:49:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304197" 22:49:57 ERROR [http_cmd]: ERROR 22:49:57 INFO [http_cmd]: http status code OK 22:49:57 ERROR --- status code --- 22:49:57 DEBUG --- status code --- 22:49:57 ERROR http status code is 400 (expected [200, 401, 403, 404]) 22:49:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:49:57 ERROR --- http response --- 22:49:57 DEBUG --- http response --- 22:49:57 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 22:49:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 22:49:57 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 22:49:59 INFO 22:49:59 INFO [loop_until]: kubectl --namespace=xlou top pods 22:49:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:49:59 INFO [loop_until]: OK (rc = 0) 22:49:59 DEBUG --- stdout --- 22:49:59 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 9Mi am-555f4db595-4lbss 13008m 8674Mi am-555f4db595-fz8sn 13425m 8555Mi am-555f4db595-kzxsp 13331m 8945Mi ds-cts-0 20292m 13098Mi ds-cts-1 18342m 13089Mi ds-cts-2 18128m 12617Mi ds-idrepo-0 5877m 5028Mi ds-idrepo-1 5524m 2907Mi ds-idrepo-2 5506m 3220Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 3m 1208Mi idm-b85f9d978-rl76f 4m 1225Mi lodemon-bc756c647-cr7x6 2m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-7c9b667974-9mr2s 6604m 1508Mi 22:49:59 DEBUG --- stderr --- 22:49:59 DEBUG 22:50:01 INFO 22:50:01 INFO [loop_until]: kubectl --namespace=xlou top node 22:50:01 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:50:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713304201" 22:50:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304201" 22:50:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304201" 22:50:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304201" 22:50:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713304201" 22:50:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304201" 22:50:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304201" 22:50:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304201" 22:50:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304201" 22:50:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304201" 22:50:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304201" 22:50:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304201" 22:50:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304201" 22:50:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304201" 22:50:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304201" 22:50:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304201" 22:50:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304201" 22:50:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304201" 22:50:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304201" 22:50:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304201" 22:50:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304201" 22:50:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304201" 22:50:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713304201" 22:50:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713304201" 22:50:01 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304201" 22:50:01 INFO [http_cmd]: http status code OK 22:50:01 INFO [http_cmd]: http status code OK 22:50:01 INFO [loop_until]: OK (rc = 0) 22:50:01 INFO [http_cmd]: http status code OK 22:50:01 INFO [http_cmd]: http status code OK 22:50:01 INFO [http_cmd]: http status code OK 22:50:01 INFO [http_cmd]: http status code OK 22:50:01 INFO [http_cmd]: http status code OK 22:50:01 INFO [http_cmd]: http status code OK 22:50:01 INFO [http_cmd]: http status code OK 22:50:01 INFO [http_cmd]: http status code OK 22:50:01 INFO [http_cmd]: http status code OK 22:50:01 INFO [http_cmd]: http status code OK 22:50:01 INFO [http_cmd]: http status code OK 22:50:01 INFO [http_cmd]: http status code OK 22:50:01 INFO [http_cmd]: http status code OK 22:50:01 INFO [http_cmd]: http status code OK 22:50:01 INFO [http_cmd]: http status code OK 22:50:01 INFO [http_cmd]: http status code OK 22:50:01 INFO [http_cmd]: http status code OK 22:50:01 DEBUG --- status code --- 22:50:01 DEBUG --- status code --- 22:50:01 INFO [http_cmd]: http status code OK 22:50:01 DEBUG --- stdout --- 22:50:01 DEBUG --- status code --- 22:50:01 DEBUG --- status code --- 22:50:01 DEBUG --- status code --- 22:50:01 DEBUG --- status code --- 22:50:01 DEBUG --- status code --- 22:50:01 DEBUG --- status code --- 22:50:01 DEBUG --- status code --- 22:50:01 DEBUG --- status code --- 22:50:01 DEBUG --- status code --- 22:50:01 DEBUG --- status code --- 22:50:01 DEBUG --- status code --- 22:50:01 DEBUG --- status code --- 22:50:01 INFO [http_cmd]: http status code OK 22:50:01 DEBUG --- status code --- 22:50:01 DEBUG --- status code --- 22:50:01 DEBUG --- status code --- 22:50:01 DEBUG --- status code --- 22:50:01 DEBUG --- status code --- 22:50:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:50:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:50:01 DEBUG --- status code --- 22:50:01 INFO [http_cmd]: http status code OK 22:50:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:50:01 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 62m 0% 5080Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 18405m 61% 15133Mi 13% gke-xlou-cdm-default-pool-1b0769c8-nhlf 13245m 44% 10797Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 81m 0% 2642Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 5577m 18% 6577Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 21144m 70% 17580Mi 15% gke-xlou-cdm-default-pool-1b0769c8-xnmp 60m 0% 3148Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 13211m 44% 10454Mi 9% gke-xlou-cdm-ds-0f601751-hmm7 6010m 20% 5658Mi 5% gke-xlou-cdm-ds-0f601751-j488 18315m 61% 14480Mi 12% gke-xlou-cdm-ds-0f601751-mc5d 11546m 38% 3351Mi 3% gke-xlou-cdm-ds-0f601751-nj6q 13786m 46% 10819Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 6037m 20% 7357Mi 6% gke-xlou-cdm-frontend-a8771548-i82n 6905m 43% 6941Mi 11% 22:50:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:50:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:50:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:50:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:50:01 INFO [http_cmd]: http status code OK 22:50:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:50:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:50:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:50:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:50:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:50:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:50:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:50:01 DEBUG --- status code --- 22:50:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:50:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:50:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:50:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:50:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:50:01 DEBUG --- http response --- 22:50:01 INFO [http_cmd]: http status code OK 22:50:01 DEBUG --- http response --- 22:50:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:50:01 INFO [http_cmd]: http status code OK 22:50:01 DEBUG --- status code --- 22:50:01 DEBUG --- http response --- 22:50:01 DEBUG --- stderr --- 22:50:01 DEBUG --- http response --- 22:50:01 DEBUG --- http response --- 22:50:01 DEBUG --- http response --- 22:50:01 DEBUG --- http response --- 22:50:01 DEBUG --- status code --- 22:50:01 DEBUG --- http response --- 22:50:01 DEBUG --- http response --- 22:50:01 DEBUG --- http response --- 22:50:01 DEBUG --- http response --- 22:50:01 DEBUG --- http response --- 22:50:01 DEBUG --- http response --- 22:50:01 DEBUG --- http response --- 22:50:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:50:01 DEBUG --- http response --- 22:50:01 DEBUG --- http response --- 22:50:01 DEBUG --- http response --- 22:50:01 DEBUG --- http response --- 22:50:01 DEBUG --- http response --- 22:50:01 DEBUG --- status code --- 22:50:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:50:01 DEBUG --- http response --- 22:50:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:50:01 DEBUG --- status code --- 22:50:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:50:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:50:01 DEBUG 22:50:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:50:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:50:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:50:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:50:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:50:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:50:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:50:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:50:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:50:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:50:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:50:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:50:01 DEBUG --- http response --- 22:50:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:50:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:50:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:50:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:50:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:50:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:50:01 DEBUG ----- output ----- 22:50:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:50:01 DEBUG ----- output ----- 22:50:01 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:50:01 DEBUG --- http response --- 22:50:01 DEBUG ----- output ----- 22:50:01 DEBUG ----- output ----- 22:50:01 DEBUG ----- output ----- 22:50:01 DEBUG ----- output ----- 22:50:01 DEBUG --- http response --- 22:50:01 DEBUG ----- output ----- 22:50:01 DEBUG ----- output ----- 22:50:01 DEBUG ----- output ----- 22:50:01 DEBUG ----- output ----- 22:50:01 DEBUG ----- output ----- 22:50:01 DEBUG ----- output ----- 22:50:01 DEBUG { 22:50:01 DEBUG ----- output ----- 22:50:01 DEBUG ----- output ----- 22:50:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:50:01 DEBUG ----- output ----- 22:50:01 DEBUG ----- output ----- 22:50:01 DEBUG ----- output ----- 22:50:01 DEBUG --- http response --- 22:50:01 DEBUG ----- output ----- 22:50:01 DEBUG { 22:50:01 DEBUG ----- output ----- 22:50:01 DEBUG { 22:50:01 DEBUG --- http response --- 22:50:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:50:01 DEBUG { 22:50:01 DEBUG { 22:50:01 DEBUG { 22:50:01 DEBUG { 22:50:01 DEBUG { 22:50:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:50:01 DEBUG { 22:50:01 DEBUG { 22:50:01 DEBUG { 22:50:01 DEBUG { 22:50:01 DEBUG ----- output ----- 22:50:01 DEBUG "status": "success", 22:50:01 DEBUG { 22:50:01 DEBUG { 22:50:01 DEBUG ----- output ----- 22:50:01 DEBUG { 22:50:01 DEBUG { 22:50:01 DEBUG { 22:50:01 DEBUG { 22:50:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:50:01 DEBUG "status": "success", 22:50:01 DEBUG { 22:50:01 DEBUG "status": "success", 22:50:01 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:50:01 DEBUG ----- output ----- 22:50:01 DEBUG "status": "success", 22:50:01 DEBUG "status": "success", 22:50:01 DEBUG "status": "success", 22:50:01 DEBUG "status": "success", 22:50:01 DEBUG "status": "success", 22:50:01 DEBUG ----- output ----- 22:50:01 DEBUG "status": "success", 22:50:01 DEBUG "status": "success", 22:50:01 DEBUG "status": "success", 22:50:01 DEBUG "status": "success", 22:50:01 DEBUG { 22:50:01 DEBUG "data": { 22:50:01 DEBUG "status": "success", 22:50:01 DEBUG "status": "success", 22:50:01 DEBUG { 22:50:01 DEBUG "status": "success", 22:50:01 DEBUG "status": "success", 22:50:01 DEBUG "status": "success", 22:50:01 DEBUG "status": "success", 22:50:01 DEBUG ----- output ----- 22:50:01 DEBUG "data": { 22:50:01 DEBUG "status": "success", 22:50:01 DEBUG "data": { 22:50:01 DEBUG ----- output ----- 22:50:01 DEBUG { 22:50:01 DEBUG "data": { 22:50:01 DEBUG "data": { 22:50:01 DEBUG "data": { 22:50:01 DEBUG "data": { 22:50:01 DEBUG "data": { 22:50:01 DEBUG { 22:50:01 DEBUG "data": { 22:50:01 DEBUG "data": { 22:50:01 DEBUG "data": { 22:50:01 DEBUG "data": { 22:50:01 DEBUG "status": "success", 22:50:01 DEBUG "resultType": "vector", 22:50:01 DEBUG "data": { 22:50:01 DEBUG "data": { 22:50:01 DEBUG "status": "success", 22:50:01 DEBUG "data": { 22:50:01 DEBUG "data": { 22:50:01 DEBUG "data": { 22:50:01 DEBUG "data": { 22:50:01 DEBUG { 22:50:01 DEBUG "resultType": "vector", 22:50:01 DEBUG "data": { 22:50:01 DEBUG "resultType": "vector", 22:50:01 DEBUG { 22:50:01 DEBUG "status": "success", 22:50:01 DEBUG "resultType": "vector", 22:50:01 DEBUG "resultType": "vector", 22:50:01 DEBUG "resultType": "vector", 22:50:01 DEBUG "resultType": "vector", 22:50:01 DEBUG "resultType": "vector", 22:50:01 DEBUG "status": "success", 22:50:01 DEBUG "resultType": "vector", 22:50:01 DEBUG "resultType": "vector", 22:50:01 DEBUG "resultType": "vector", 22:50:01 DEBUG "resultType": "vector", 22:50:01 DEBUG "data": { 22:50:01 DEBUG "result": [ 22:50:01 DEBUG "resultType": "vector", 22:50:01 DEBUG "resultType": "vector", 22:50:01 DEBUG "data": { 22:50:01 DEBUG "resultType": "vector", 22:50:01 DEBUG "result": [ 22:50:01 DEBUG "resultType": "vector", 22:50:01 DEBUG "resultType": "vector", 22:50:01 DEBUG "status": "success", 22:50:01 DEBUG "result": [ 22:50:01 DEBUG "resultType": "vector", 22:50:01 DEBUG "result": [ 22:50:01 DEBUG "status": "success", 22:50:01 DEBUG "data": { 22:50:01 DEBUG "result": [ 22:50:01 DEBUG "result": [ 22:50:01 DEBUG "result": [ 22:50:01 DEBUG "result": [ 22:50:01 DEBUG "result": [ 22:50:01 DEBUG "data": { 22:50:01 DEBUG "result": [ 22:50:01 DEBUG "result": [ 22:50:01 DEBUG "result": [ 22:50:01 DEBUG "result": [ 22:50:01 DEBUG "resultType": "vector", 22:50:01 DEBUG { 22:50:01 DEBUG "result": [ 22:50:01 DEBUG "result": [ 22:50:01 DEBUG "resultType": "vector", 22:50:01 DEBUG { 22:50:01 DEBUG "resultType": "vector", 22:50:01 DEBUG "result": [ 22:50:01 DEBUG "result": [ 22:50:01 DEBUG "data": { 22:50:01 DEBUG { 22:50:01 DEBUG "result": [ 22:50:01 DEBUG { 22:50:01 DEBUG "data": { 22:50:01 DEBUG "resultType": "vector", 22:50:01 DEBUG { 22:50:01 DEBUG { 22:50:01 DEBUG { 22:50:01 DEBUG { 22:50:01 DEBUG { 22:50:01 DEBUG "resultType": "vector", 22:50:01 DEBUG { 22:50:01 DEBUG { 22:50:01 DEBUG { 22:50:01 DEBUG { 22:50:01 DEBUG "result": [ 22:50:01 DEBUG "metric": { 22:50:01 DEBUG { 22:50:01 DEBUG { 22:50:01 DEBUG "result": [ 22:50:01 DEBUG "metric": { 22:50:01 DEBUG "result": [ 22:50:01 DEBUG { 22:50:01 DEBUG { 22:50:01 DEBUG "resultType": "vector", 22:50:01 DEBUG "metric": { 22:50:01 DEBUG { 22:50:01 DEBUG "metric": { 22:50:01 DEBUG "resultType": "vector", 22:50:01 DEBUG "result": [ 22:50:01 DEBUG "metric": { 22:50:01 DEBUG "metric": { 22:50:01 DEBUG "metric": { 22:50:01 DEBUG "metric": { 22:50:01 DEBUG "metric": { 22:50:01 DEBUG "result": [ 22:50:01 DEBUG "metric": { 22:50:01 DEBUG "metric": { 22:50:01 DEBUG "pod": "ds-idrepo-2" 22:50:01 DEBUG "metric": { 22:50:01 DEBUG { 22:50:01 DEBUG "pod": "ds-idrepo-2" 22:50:01 DEBUG "metric": { 22:50:01 DEBUG "metric": { 22:50:01 DEBUG { 22:50:01 DEBUG "pod": "ds-idrepo-2" 22:50:01 DEBUG { 22:50:01 DEBUG "metric": { 22:50:01 DEBUG "metric": { 22:50:01 DEBUG "result": [ 22:50:01 DEBUG "pod": "am-555f4db595-fz8sn" 22:50:01 DEBUG "metric": { 22:50:01 DEBUG "instance": "10.142.0.107:9100" 22:50:01 DEBUG "result": [ 22:50:01 DEBUG { 22:50:01 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:50:01 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:50:01 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:50:01 DEBUG "pod": "ds-idrepo-2" 22:50:01 DEBUG "pod": "am-555f4db595-fz8sn" 22:50:01 DEBUG { 22:50:01 DEBUG "pod": "ds-idrepo-2" 22:50:01 DEBUG }, 22:50:01 DEBUG "pod": "am-555f4db595-fz8sn" 22:50:01 DEBUG "metric": { 22:50:01 DEBUG }, 22:50:01 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:50:01 DEBUG "pod": "ds-idrepo-2" 22:50:01 DEBUG "metric": { 22:50:01 DEBUG }, 22:50:01 DEBUG "metric": { 22:50:01 DEBUG "instance": "10.142.0.107:9100" 22:50:01 DEBUG "instance": "10.142.0.107:9100" 22:50:01 DEBUG { 22:50:01 DEBUG }, 22:50:01 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:50:01 DEBUG }, 22:50:01 DEBUG { 22:50:01 DEBUG "metric": { 22:50:01 DEBUG }, 22:50:01 DEBUG }, 22:50:01 DEBUG }, 22:50:01 DEBUG }, 22:50:01 DEBUG }, 22:50:01 DEBUG "metric": { 22:50:01 DEBUG }, 22:50:01 DEBUG "value": [ 22:50:01 DEBUG }, 22:50:01 DEBUG "pod": "am-555f4db595-fz8sn" 22:50:01 DEBUG "value": [ 22:50:01 DEBUG }, 22:50:01 DEBUG }, 22:50:01 DEBUG "metric": { 22:50:01 DEBUG "pod": "am-555f4db595-fz8sn" 22:50:01 DEBUG "value": [ 22:50:01 DEBUG "pod": "am-555f4db595-fz8sn" 22:50:01 DEBUG }, 22:50:01 DEBUG }, 22:50:01 DEBUG "metric": { 22:50:01 DEBUG "value": [ 22:50:01 DEBUG }, 22:50:01 DEBUG "value": [ 22:50:01 DEBUG "metric": { 22:50:01 DEBUG "instance": "10.142.0.107:9100" 22:50:01 DEBUG "value": [ 22:50:01 DEBUG "value": [ 22:50:01 DEBUG "value": [ 22:50:01 DEBUG "value": [ 22:50:01 DEBUG "value": [ 22:50:01 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:50:01 DEBUG "value": [ 22:50:01 DEBUG 22:50:01 DEBUG "value": [ 22:50:01 DEBUG }, 22:50:01 DEBUG 22:50:01 DEBUG "value": [ 22:50:01 DEBUG "value": [ 22:50:01 DEBUG "pod": "am-555f4db595-fz8sn" 22:50:01 DEBUG }, 22:50:01 DEBUG 22:50:01 DEBUG }, 22:50:01 DEBUG "value": [ 22:50:01 DEBUG "value": [ 22:50:01 DEBUG "pod": "ds-idrepo-2" 22:50:01 DEBUG 22:50:01 DEBUG "value": [ 22:50:01 DEBUG 22:50:01 DEBUG "instance": "10.142.0.107:9100" 22:50:01 DEBUG }, 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG }, 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG "value": [ 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG }, 22:50:01 DEBUG "value": [ 22:50:01 DEBUG 22:50:01 DEBUG "value": [ 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG }, 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG }, 22:50:01 DEBUG "value": [ 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG "value": [ 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG [...] 22:50:01 DEBUG 22:50:01 DEBUG "value": [ 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG "value": [ 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG "value": [ 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG [...] 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG [...] 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG [...] 22:50:01 DEBUG 22:50:01 DEBUG [...] 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG [...] 22:50:01 DEBUG [...] 22:50:01 DEBUG [...] 22:50:01 DEBUG [...] 22:50:01 DEBUG [...] 22:50:01 DEBUG 22:50:01 DEBUG [...] 22:50:01 DEBUG 22:50:01 DEBUG [...] 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG [...] 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG [...] 22:50:01 DEBUG [...] 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG [...] 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG [...] 22:50:01 DEBUG [...] 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG [...] 22:50:01 DEBUG 22:50:01 DEBUG [...] 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG [...] 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG [...] 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG "pod": "ds-idrepo-0" 22:50:01 DEBUG 22:50:01 DEBUG [...] 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG [...] 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG [...] 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG }, 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG "pod": "ds-idrepo-0" 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG "pod": "am-555f4db595-kzxsp" 22:50:01 DEBUG 22:50:01 DEBUG "instance": "10.142.0.43:9100" 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:50:01 DEBUG "pod": "end-user-ui-58878d7668-zg4h2" 22:50:01 DEBUG "pod": "end-user-ui-58878d7668-zg4h2" 22:50:01 DEBUG "pod": "ds-idrepo-0" 22:50:01 DEBUG "pod": "am-555f4db595-kzxsp" 22:50:01 DEBUG "pod": "ds-idrepo-0" 22:50:01 DEBUG 22:50:01 DEBUG "pod": "ds-idrepo-0" 22:50:01 DEBUG "pod": "am-555f4db595-kzxsp" 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG "value": [ 22:50:01 DEBUG "pod": "ds-idrepo-0" 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG }, 22:50:01 DEBUG 22:50:01 DEBUG "instance": "10.142.0.43:9100" 22:50:01 DEBUG "instance": "10.142.0.43:9100" 22:50:01 DEBUG 22:50:01 DEBUG }, 22:50:01 DEBUG "pod": "end-user-ui-58878d7668-zg4h2" 22:50:01 DEBUG }, 22:50:01 DEBUG 22:50:01 DEBUG 22:50:01 DEBUG }, 22:50:01 DEBUG }, 22:50:01 DEBUG }, 22:50:01 DEBUG }, 22:50:01 DEBUG }, 22:50:01 DEBUG }, 22:50:01 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:50:01 DEBUG }, 22:50:01 DEBUG }, 22:50:01 DEBUG "pod": "am-555f4db595-kzxsp" 22:50:01 DEBUG "pod": "end-user-ui-58878d7668-zg4h2" 22:50:01 DEBUG 1713304201, 22:50:01 DEBUG }, 22:50:01 DEBUG 22:50:01 DEBUG "pod": "am-555f4db595-kzxsp" 22:50:01 DEBUG "value": [ 22:50:01 DEBUG "pod": "am-555f4db595-kzxsp" 22:50:01 DEBUG }, 22:50:01 DEBUG }, 22:50:01 DEBUG 22:50:01 DEBUG "value": [ 22:50:01 DEBUG }, 22:50:01 DEBUG "value": [ 22:50:01 DEBUG 22:50:01 DEBUG "instance": "10.142.0.43:9100" 22:50:01 DEBUG "value": [ 22:50:01 DEBUG "value": [ 22:50:01 DEBUG "value": [ 22:50:01 DEBUG "value": [ 22:50:01 DEBUG "value": [ 22:50:01 DEBUG "value": [ 22:50:01 DEBUG }, 22:50:01 DEBUG "value": [ 22:50:01 DEBUG "value": [ 22:50:01 DEBUG }, 22:50:01 DEBUG "value": [ 22:50:01 DEBUG "0" 22:50:01 DEBUG "value": [ 22:50:01 DEBUG "pod": "am-555f4db595-kzxsp" 22:50:01 DEBUG }, 22:50:01 DEBUG 1713304201, 22:50:01 DEBUG }, 22:50:01 DEBUG "value": [ 22:50:01 DEBUG "value": [ 22:50:01 DEBUG "pod": "ds-cts-2" 22:50:01 DEBUG 1713304201, 22:50:01 DEBUG "value": [ 22:50:01 DEBUG 1713304201, 22:50:01 DEBUG "instance": "10.142.0.43:9100" 22:50:01 DEBUG }, 22:50:01 DEBUG 1713304201, 22:50:01 DEBUG 1713304201, 22:50:01 DEBUG 1713304201, 22:50:01 DEBUG 1713304201, 22:50:01 DEBUG 1713304201, 22:50:01 DEBUG 1713304201, 22:50:01 DEBUG "value": [ 22:50:01 DEBUG 1713304201, 22:50:01 DEBUG 1713304201, 22:50:01 DEBUG }, 22:50:01 DEBUG 1713304201, 22:50:01 DEBUG ] 22:50:01 DEBUG 1713304201, 22:50:01 DEBUG }, 22:50:01 DEBUG "value": [ 22:50:01 DEBUG "0" 22:50:01 DEBUG "value": [ 22:50:01 DEBUG 1713304201, 22:50:01 DEBUG 1713304201, 22:50:01 DEBUG }, 22:50:01 DEBUG "2164.7" 22:50:01 DEBUG 1713304201, 22:50:01 DEBUG "19973186560" 22:50:01 DEBUG }, 22:50:01 DEBUG "value": [ 22:50:01 DEBUG "6.768435464978173" 22:50:01 DEBUG "0" 22:50:01 DEBUG "0" 22:50:01 DEBUG "0" 22:50:01 DEBUG "0.027" 22:50:01 DEBUG "0" 22:50:01 DEBUG 1713304201, 22:50:01 DEBUG "0" 22:50:01 DEBUG "1995.6333333333332" 22:50:01 DEBUG "value": [ 22:50:01 DEBUG "2164.4666666666667" 22:50:01 DEBUG } 22:50:01 DEBUG ] 22:50:01 DEBUG "value": [ 22:50:01 DEBUG 1713304201, 22:50:01 DEBUG ] 22:50:01 DEBUG 1713304201, 22:50:01 DEBUG "0.00036666666666652263" 22:50:01 DEBUG "49299039.56666666" 22:50:01 DEBUG "value": [ 22:50:01 DEBUG ] 22:50:01 DEBUG "0" 22:50:01 DEBUG ] 22:50:01 DEBUG "value": [ 22:50:01 DEBUG 1713304201, 22:50:01 DEBUG ] 22:50:01 DEBUG ] 22:50:01 DEBUG ] 22:50:01 DEBUG ] 22:50:01 DEBUG ] 22:50:01 DEBUG ] 22:50:01 DEBUG "1682612224" 22:50:01 DEBUG ] 22:50:01 DEBUG ] 22:50:01 DEBUG 1713304201, 22:50:01 DEBUG ] 22:50:01 DEBUG "0" 22:50:01 DEBUG } 22:50:01 DEBUG 1713304201, 22:50:01 DEBUG "2164.5666666666666" 22:50:01 DEBUG } 22:50:01 DEBUG "0" 22:50:01 DEBUG ] 22:50:01 DEBUG ] 22:50:01 DEBUG 1713304201, 22:50:01 DEBUG } 22:50:01 DEBUG ] 22:50:01 DEBUG } 22:50:01 DEBUG 1713304201, 22:50:01 DEBUG "0" 22:50:01 DEBUG } 22:50:01 DEBUG } 22:50:01 DEBUG } 22:50:01 DEBUG } 22:50:01 DEBUG } 22:50:01 DEBUG } 22:50:01 DEBUG ] 22:50:01 DEBUG } 22:50:01 DEBUG } 22:50:01 DEBUG "0.000032117224880383355" 22:50:01 DEBUG } 22:50:01 DEBUG ] 22:50:01 DEBUG } 22:50:01 DEBUG "3552854" 22:50:01 DEBUG ] 22:50:01 DEBUG ] 22:50:01 DEBUG ] 22:50:01 DEBUG } 22:50:01 DEBUG } 22:50:01 DEBUG "0" 22:50:01 DEBUG ] 22:50:01 DEBUG } 22:50:01 DEBUG ] 22:50:01 DEBUG "49289596.36666666" 22:50:01 DEBUG ] 22:50:01 DEBUG ] 22:50:01 DEBUG ] 22:50:01 DEBUG ] 22:50:01 DEBUG ] 22:50:01 DEBUG ] 22:50:01 DEBUG ] 22:50:01 DEBUG } 22:50:01 DEBUG ] 22:50:01 DEBUG ] 22:50:01 DEBUG ] 22:50:01 DEBUG ] 22:50:01 DEBUG } 22:50:01 DEBUG ] 22:50:01 DEBUG } 22:50:01 DEBUG } 22:50:01 DEBUG } 22:50:01 DEBUG ] 22:50:01 DEBUG ] 22:50:01 DEBUG ] 22:50:01 DEBUG } 22:50:01 DEBUG ] 22:50:01 DEBUG } 22:50:01 DEBUG ] 22:50:01 DEBUG } 22:50:01 DEBUG } 22:50:01 DEBUG } 22:50:01 DEBUG } 22:50:01 DEBUG } 22:50:01 DEBUG } 22:50:01 DEBUG } 22:50:01 DEBUG } 22:50:01 DEBUG } 22:50:01 DEBUG } 22:50:01 DEBUG } 22:50:01 DEBUG } 22:50:01 DEBUG ] 22:50:01 DEBUG } 22:50:01 DEBUG ] 22:50:01 DEBUG } 22:50:01 DEBUG ] 22:50:01 DEBUG } 22:50:01 DEBUG } 22:50:01 DEBUG } 22:50:01 DEBUG } 22:50:01 DEBUG } 22:50:01 DEBUG } 22:50:01 DEBUG } 22:50:01 DEBUG ] 22:50:01 DEBUG } 22:50:01 DEBUG } 22:50:01 DEBUG } 22:50:01 DEBUG } 22:50:01 DEBUG } 22:50:01 DEBUG ] 22:50:01 DEBUG } 22:50:01 DEBUG } 22:50:01 DEBUG ] 22:50:01 DEBUG } 22:50:01 DEBUG } 22:50:01 DEBUG ] 22:50:01 DEBUG } 22:50:01 DEBUG } 22:50:01 DEBUG } 22:50:01 DEBUG } 22:50:01 DEBUG ] 22:50:01 DEBUG } 22:50:01 DEBUG ] 22:50:01 DEBUG } 22:50:01 DEBUG } 22:50:01 DEBUG } 22:50:01 DEBUG } 22:50:01 DEBUG } 22:50:01 DEBUG } 22:50:01 DEBUG } 22:50:01 DEBUG } 22:50:01 DEBUG } 22:50:01 DEBUG } 22:50:01 DEBUG } 22:50:01 DEBUG } 22:50:01 DEBUG } 22:50:01 DEBUG } 22:50:01 DEBUG } 22:50:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304257" 22:50:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713304257" 22:50:57 ERROR [http_cmd]: ERROR 22:50:57 INFO [http_cmd]: http status code OK 22:50:57 ERROR --- status code --- 22:50:57 DEBUG --- status code --- 22:50:57 ERROR http status code is 400 (expected [200, 401, 403, 404]) 22:50:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:50:57 ERROR --- http response --- 22:50:57 DEBUG --- http response --- 22:50:57 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 22:50:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 22:50:57 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 22:50:59 INFO 22:50:59 INFO [loop_until]: kubectl --namespace=xlou top pods 22:50:59 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:51:00 INFO [loop_until]: OK (rc = 0) 22:51:00 DEBUG --- stdout --- 22:51:00 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 9Mi am-555f4db595-4lbss 13182m 8681Mi am-555f4db595-fz8sn 12626m 8557Mi am-555f4db595-kzxsp 13424m 8946Mi ds-cts-0 20090m 13213Mi ds-cts-1 18283m 12823Mi ds-cts-2 17809m 12686Mi ds-idrepo-0 5747m 5029Mi ds-idrepo-1 5529m 2908Mi ds-idrepo-2 5531m 3220Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 3m 1208Mi idm-b85f9d978-rl76f 3m 1226Mi lodemon-bc756c647-cr7x6 2m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-7c9b667974-9mr2s 6657m 1510Mi 22:51:00 DEBUG --- stderr --- 22:51:00 DEBUG 22:51:01 INFO 22:51:01 INFO [loop_until]: kubectl --namespace=xlou top node 22:51:01 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:51:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304262" 22:51:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304262" 22:51:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304262" 22:51:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304262" 22:51:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304262" 22:51:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304262" 22:51:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304262" 22:51:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304262" 22:51:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713304262" 22:51:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713304262" 22:51:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304262" 22:51:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304262" 22:51:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304262" 22:51:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304262" 22:51:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304262" 22:51:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304262" 22:51:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304262" 22:51:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304262" 22:51:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304262" 22:51:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713304262" 22:51:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304262" 22:51:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304262" 22:51:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304262" 22:51:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713304262" 22:51:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304262" 22:51:02 INFO [loop_until]: OK (rc = 0) 22:51:02 INFO [http_cmd]: http status code OK 22:51:02 INFO [http_cmd]: http status code OK 22:51:02 INFO [http_cmd]: http status code OK 22:51:02 INFO [http_cmd]: http status code OK 22:51:02 INFO [http_cmd]: http status code OK 22:51:02 INFO [http_cmd]: http status code OK 22:51:02 INFO [http_cmd]: http status code OK 22:51:02 INFO [http_cmd]: http status code OK 22:51:02 INFO [http_cmd]: http status code OK 22:51:02 INFO [http_cmd]: http status code OK 22:51:02 INFO [http_cmd]: http status code OK 22:51:02 INFO [http_cmd]: http status code OK 22:51:02 INFO [http_cmd]: http status code OK 22:51:02 INFO [http_cmd]: http status code OK 22:51:02 INFO [http_cmd]: http status code OK 22:51:02 DEBUG --- stdout --- 22:51:02 INFO [http_cmd]: http status code OK 22:51:02 DEBUG --- status code --- 22:51:02 INFO [http_cmd]: http status code OK 22:51:02 INFO [http_cmd]: http status code OK 22:51:02 DEBUG --- status code --- 22:51:02 DEBUG --- status code --- 22:51:02 DEBUG --- status code --- 22:51:02 DEBUG --- status code --- 22:51:02 DEBUG --- status code --- 22:51:02 DEBUG --- status code --- 22:51:02 DEBUG --- status code --- 22:51:02 DEBUG --- status code --- 22:51:02 DEBUG --- status code --- 22:51:02 DEBUG --- status code --- 22:51:02 INFO [http_cmd]: http status code OK 22:51:02 INFO [http_cmd]: http status code OK 22:51:02 DEBUG --- status code --- 22:51:02 INFO [http_cmd]: http status code OK 22:51:02 DEBUG --- status code --- 22:51:02 DEBUG --- status code --- 22:51:02 DEBUG --- status code --- 22:51:02 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 65m 0% 5083Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 18060m 60% 15346Mi 13% gke-xlou-cdm-default-pool-1b0769c8-nhlf 13146m 44% 10795Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 76m 0% 2645Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 5488m 18% 6569Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 21042m 70% 17090Mi 15% gke-xlou-cdm-default-pool-1b0769c8-xnmp 57m 0% 3153Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 13245m 44% 10459Mi 9% gke-xlou-cdm-ds-0f601751-hmm7 6128m 20% 5662Mi 5% gke-xlou-cdm-ds-0f601751-j488 17342m 58% 14693Mi 13% gke-xlou-cdm-ds-0f601751-mc5d 11595m 38% 3387Mi 3% gke-xlou-cdm-ds-0f601751-nj6q 13486m 45% 10845Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 6062m 20% 7349Mi 6% gke-xlou-cdm-frontend-a8771548-i82n 6912m 43% 6942Mi 11% 22:51:02 DEBUG --- status code --- 22:51:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:51:02 DEBUG --- status code --- 22:51:02 DEBUG --- status code --- 22:51:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:51:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:51:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:51:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:51:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:51:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:51:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:51:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:51:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:51:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:51:02 DEBUG --- status code --- 22:51:02 DEBUG --- status code --- 22:51:02 INFO [http_cmd]: http status code OK 22:51:02 INFO [http_cmd]: http status code OK 22:51:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:51:02 DEBUG --- status code --- 22:51:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:51:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:51:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:51:02 DEBUG --- stderr --- 22:51:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:51:02 DEBUG --- http response --- 22:51:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:51:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:51:02 DEBUG --- http response --- 22:51:02 DEBUG --- http response --- 22:51:02 DEBUG --- http response --- 22:51:02 DEBUG --- http response --- 22:51:02 DEBUG --- http response --- 22:51:02 DEBUG --- http response --- 22:51:02 DEBUG --- http response --- 22:51:02 DEBUG --- http response --- 22:51:02 DEBUG --- http response --- 22:51:02 DEBUG --- http response --- 22:51:02 INFO [http_cmd]: http status code OK 22:51:02 DEBUG --- status code --- 22:51:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:51:02 DEBUG --- status code --- 22:51:02 INFO [http_cmd]: http status code OK 22:51:02 DEBUG --- status code --- 22:51:02 DEBUG --- http response --- 22:51:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:51:02 DEBUG --- http response --- 22:51:02 DEBUG --- http response --- 22:51:02 DEBUG --- http response --- 22:51:02 DEBUG 22:51:02 DEBUG --- http response --- 22:51:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:51:02 DEBUG --- http response --- 22:51:02 DEBUG --- http response --- 22:51:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:51:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:51:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:51:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:51:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 22:51:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:51:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:51:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:51:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:51:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:51:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:51:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:51:02 DEBUG --- http response --- 22:51:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:51:02 DEBUG --- status code --- 22:51:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:51:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:51:02 DEBUG --- http response --- 22:51:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:51:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:51:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:51:02 DEBUG ----- output ----- 22:51:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 22:51:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:51:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:51:02 DEBUG ----- output ----- 22:51:02 DEBUG ----- output ----- 22:51:02 DEBUG ----- output ----- 22:51:02 DEBUG ----- output ----- 22:51:02 DEBUG ----- output ----- 22:51:02 DEBUG ----- output ----- 22:51:02 DEBUG ----- output ----- 22:51:02 DEBUG ----- output ----- 22:51:02 DEBUG ----- output ----- 22:51:02 DEBUG --- http response --- 22:51:02 DEBUG --- http response --- 22:51:02 DEBUG ----- output ----- 22:51:02 DEBUG --- http response --- 22:51:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:51:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:51:02 DEBUG --- http response --- 22:51:02 DEBUG ----- output ----- 22:51:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:51:02 DEBUG ----- output ----- 22:51:02 DEBUG ----- output ----- 22:51:02 DEBUG ----- output ----- 22:51:02 DEBUG { 22:51:02 DEBUG ----- output ----- 22:51:02 DEBUG ----- output ----- 22:51:02 DEBUG ----- output ----- 22:51:02 DEBUG { 22:51:02 DEBUG { 22:51:02 DEBUG { 22:51:02 DEBUG { 22:51:02 DEBUG { 22:51:02 DEBUG { 22:51:02 DEBUG { 22:51:02 DEBUG { 22:51:02 DEBUG { 22:51:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:51:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:51:02 DEBUG { 22:51:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:51:02 DEBUG ----- output ----- 22:51:02 DEBUG --- http response --- 22:51:02 DEBUG { 22:51:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:51:02 DEBUG ----- output ----- 22:51:02 DEBUG { 22:51:02 DEBUG { 22:51:02 DEBUG { 22:51:02 DEBUG "status": "success", 22:51:02 DEBUG { 22:51:02 DEBUG { 22:51:02 DEBUG { 22:51:02 DEBUG "status": "success", 22:51:02 DEBUG "status": "success", 22:51:02 DEBUG "status": "success", 22:51:02 DEBUG "status": "success", 22:51:02 DEBUG "status": "success", 22:51:02 DEBUG "status": "success", 22:51:02 DEBUG "status": "success", 22:51:02 DEBUG "status": "success", 22:51:02 DEBUG "status": "success", 22:51:02 DEBUG ----- output ----- 22:51:02 DEBUG ----- output ----- 22:51:02 DEBUG "status": "success", 22:51:02 DEBUG ----- output ----- 22:51:02 DEBUG { 22:51:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:51:02 DEBUG "status": "success", 22:51:02 DEBUG ----- output ----- 22:51:02 DEBUG { 22:51:02 DEBUG "status": "success", 22:51:02 DEBUG "status": "success", 22:51:02 DEBUG "status": "success", 22:51:02 DEBUG "data": { 22:51:02 DEBUG "status": "success", 22:51:02 DEBUG "status": "success", 22:51:02 DEBUG "status": "success", 22:51:02 DEBUG "data": { 22:51:02 DEBUG "data": { 22:51:02 DEBUG "data": { 22:51:02 DEBUG "data": { 22:51:02 DEBUG "data": { 22:51:02 DEBUG "data": { 22:51:02 DEBUG "data": { 22:51:02 DEBUG "data": { 22:51:02 DEBUG "data": { 22:51:02 DEBUG { 22:51:02 DEBUG { 22:51:02 DEBUG "data": { 22:51:02 DEBUG "resultType": "vector", 22:51:02 DEBUG "status": "success", 22:51:02 DEBUG ----- output ----- 22:51:02 DEBUG "data": { 22:51:02 DEBUG { 22:51:02 DEBUG "status": "success", 22:51:02 DEBUG "data": { 22:51:02 DEBUG "data": { 22:51:02 DEBUG "data": { 22:51:02 DEBUG "resultType": "vector", 22:51:02 DEBUG "data": { 22:51:02 DEBUG "data": { 22:51:02 DEBUG "data": { 22:51:02 DEBUG "resultType": "vector", 22:51:02 DEBUG "resultType": "vector", 22:51:02 DEBUG "resultType": "vector", 22:51:02 DEBUG "resultType": "vector", 22:51:02 DEBUG "resultType": "vector", 22:51:02 DEBUG "resultType": "vector", 22:51:02 DEBUG "resultType": "vector", 22:51:02 DEBUG "resultType": "vector", 22:51:02 DEBUG "resultType": "vector", 22:51:02 DEBUG "status": "success", 22:51:02 DEBUG "status": "success", 22:51:02 DEBUG { 22:51:02 DEBUG "result": [ 22:51:02 DEBUG "data": { 22:51:02 DEBUG { 22:51:02 DEBUG "resultType": "vector", 22:51:02 DEBUG "status": "success", 22:51:02 DEBUG "data": { 22:51:02 DEBUG "resultType": "vector", 22:51:02 DEBUG "resultType": "vector", 22:51:02 DEBUG "resultType": "vector", 22:51:02 DEBUG "result": [ 22:51:02 DEBUG "resultType": "vector", 22:51:02 DEBUG "resultType": "vector", 22:51:02 DEBUG "resultType": "vector", 22:51:02 DEBUG "result": [ 22:51:02 DEBUG "result": [ 22:51:02 DEBUG "result": [ 22:51:02 DEBUG "result": [ 22:51:02 DEBUG "result": [ 22:51:02 DEBUG "result": [ 22:51:02 DEBUG "result": [ 22:51:02 DEBUG "result": [ 22:51:02 DEBUG "result": [ 22:51:02 DEBUG "data": { 22:51:02 DEBUG "data": { 22:51:02 DEBUG "status": "success", 22:51:02 DEBUG { 22:51:02 DEBUG "resultType": "vector", 22:51:02 DEBUG "status": "success", 22:51:02 DEBUG "result": [ 22:51:02 DEBUG "data": { 22:51:02 DEBUG "resultType": "vector", 22:51:02 DEBUG "result": [ 22:51:02 DEBUG "result": [ 22:51:02 DEBUG "result": [ 22:51:02 DEBUG { 22:51:02 DEBUG "result": [ 22:51:02 DEBUG "result": [ 22:51:02 DEBUG "result": [ 22:51:02 DEBUG { 22:51:02 DEBUG { 22:51:02 DEBUG { 22:51:02 DEBUG { 22:51:02 DEBUG { 22:51:02 DEBUG { 22:51:02 DEBUG { 22:51:02 DEBUG { 22:51:02 DEBUG { 22:51:02 DEBUG "resultType": "vector", 22:51:02 DEBUG "resultType": "vector", 22:51:02 DEBUG "data": { 22:51:02 DEBUG "resultType": "vector", 22:51:02 DEBUG "result": [ 22:51:02 DEBUG "data": { 22:51:02 DEBUG { 22:51:02 DEBUG "resultType": "vector", 22:51:02 DEBUG "result": [ 22:51:02 DEBUG { 22:51:02 DEBUG { 22:51:02 DEBUG { 22:51:02 DEBUG "metric": { 22:51:02 DEBUG { 22:51:02 DEBUG { 22:51:02 DEBUG { 22:51:02 DEBUG "metric": { 22:51:02 DEBUG "metric": { 22:51:02 DEBUG "metric": { 22:51:02 DEBUG "metric": { 22:51:02 DEBUG "metric": { 22:51:02 DEBUG "metric": { 22:51:02 DEBUG "metric": { 22:51:02 DEBUG "metric": { 22:51:02 DEBUG "metric": { 22:51:02 DEBUG "result": [ 22:51:02 DEBUG "result": [ 22:51:02 DEBUG "result": [ 22:51:02 DEBUG "metric": { 22:51:02 DEBUG { 22:51:02 DEBUG "resultType": "vector", 22:51:02 DEBUG "metric": { 22:51:02 DEBUG "pod": "ds-idrepo-2" 22:51:02 DEBUG "result": [ 22:51:02 DEBUG "metric": { 22:51:02 DEBUG "metric": { 22:51:02 DEBUG "metric": { 22:51:02 DEBUG "pod": "am-555f4db595-fz8sn" 22:51:02 DEBUG "metric": { 22:51:02 DEBUG "metric": { 22:51:02 DEBUG "metric": { 22:51:02 DEBUG "pod": "ds-idrepo-2" 22:51:02 DEBUG "pod": "ds-idrepo-2" 22:51:02 DEBUG "pod": "ds-idrepo-2" 22:51:02 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:51:02 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:51:02 DEBUG "pod": "ds-idrepo-2" 22:51:02 DEBUG "instance": "10.142.0.107:9100" 22:51:02 DEBUG "pod": "am-555f4db595-fz8sn" 22:51:02 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:51:02 DEBUG { 22:51:02 DEBUG { 22:51:02 DEBUG { 22:51:02 DEBUG "pod": "am-555f4db595-fz8sn" 22:51:02 DEBUG "metric": { 22:51:02 DEBUG "result": [ 22:51:02 DEBUG { 22:51:02 DEBUG }, 22:51:02 DEBUG { 22:51:02 DEBUG "pod": "am-555f4db595-fz8sn" 22:51:02 DEBUG "pod": "am-555f4db595-fz8sn" 22:51:02 DEBUG "pod": "am-555f4db595-fz8sn" 22:51:02 DEBUG }, 22:51:02 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:51:02 DEBUG "instance": "10.142.0.107:9100" 22:51:02 DEBUG "instance": "10.142.0.107:9100" 22:51:02 DEBUG }, 22:51:02 DEBUG "value": [ 22:51:02 DEBUG }, 22:51:02 DEBUG }, 22:51:02 DEBUG }, 22:51:02 DEBUG }, 22:51:02 DEBUG }, 22:51:02 DEBUG }, 22:51:02 DEBUG }, 22:51:02 DEBUG "metric": { 22:51:02 DEBUG "metric": { 22:51:02 DEBUG "metric": { 22:51:02 DEBUG }, 22:51:02 DEBUG "pod": "ds-idrepo-2" 22:51:02 DEBUG { 22:51:02 DEBUG "metric": { 22:51:02 DEBUG "value": [ 22:51:02 DEBUG "metric": { 22:51:02 DEBUG }, 22:51:02 DEBUG }, 22:51:02 DEBUG }, 22:51:02 DEBUG "value": [ 22:51:02 DEBUG }, 22:51:02 DEBUG }, 22:51:02 DEBUG }, 22:51:02 DEBUG }, 22:51:02 DEBUG 22:51:02 DEBUG "value": [ 22:51:02 DEBUG "value": [ 22:51:02 DEBUG "value": [ 22:51:02 DEBUG "value": [ 22:51:02 DEBUG "value": [ 22:51:02 DEBUG "value": [ 22:51:02 DEBUG "value": [ 22:51:02 DEBUG "pod": "am-555f4db595-fz8sn" 22:51:02 DEBUG "pod": "ds-idrepo-2" 22:51:02 DEBUG "instance": "10.142.0.107:9100" 22:51:02 DEBUG "value": [ 22:51:02 DEBUG }, 22:51:02 DEBUG "metric": { 22:51:02 DEBUG "instance": "10.142.0.107:9100" 22:51:02 DEBUG }, 22:51:02 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:51:02 DEBUG "value": [ 22:51:02 DEBUG "value": [ 22:51:02 DEBUG "value": [ 22:51:02 DEBUG 22:51:02 DEBUG "value": [ 22:51:02 DEBUG "value": [ 22:51:02 DEBUG "value": [ 22:51:02 DEBUG "value": [ 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG }, 22:51:02 DEBUG }, 22:51:02 DEBUG }, 22:51:02 DEBUG 22:51:02 DEBUG "value": [ 22:51:02 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:51:02 DEBUG 22:51:02 DEBUG "value": [ 22:51:02 DEBUG }, 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG "value": [ 22:51:02 DEBUG "value": [ 22:51:02 DEBUG "value": [ 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG }, 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG "value": [ 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG [...] 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG "value": [ 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG [...] 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG [...] 22:51:02 DEBUG [...] 22:51:02 DEBUG [...] 22:51:02 DEBUG [...] 22:51:02 DEBUG [...] 22:51:02 DEBUG [...] 22:51:02 DEBUG [...] 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG [...] 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG [...] 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG [...] 22:51:02 DEBUG [...] 22:51:02 DEBUG 22:51:02 DEBUG [...] 22:51:02 DEBUG [...] 22:51:02 DEBUG [...] 22:51:02 DEBUG [...] 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG [...] 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG [...] 22:51:02 DEBUG [...] 22:51:02 DEBUG [...] 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG [...] 22:51:02 DEBUG [...] 22:51:02 DEBUG [...] 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG "pod": "ds-idrepo-0" 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG [...] 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG "pod": "am-555f4db595-kzxsp" 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG }, 22:51:02 DEBUG "pod": "ds-idrepo-0" 22:51:02 DEBUG "pod": "ds-cts-2" 22:51:02 DEBUG "pod": "am-555f4db595-kzxsp" 22:51:02 DEBUG "pod": "ds-idrepo-0" 22:51:02 DEBUG "instance": "10.142.0.43:9100" 22:51:02 DEBUG "pod": "am-555f4db595-kzxsp" 22:51:02 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG "pod": "am-555f4db595-kzxsp" 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG "pod": "ds-idrepo-0" 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG "pod": "am-555f4db595-kzxsp" 22:51:02 DEBUG "pod": "am-555f4db595-kzxsp" 22:51:02 DEBUG }, 22:51:02 DEBUG "pod": "am-555f4db595-kzxsp" 22:51:02 DEBUG "instance": "10.142.0.43:9100" 22:51:02 DEBUG "instance": "10.142.0.43:9100" 22:51:02 DEBUG "pod": "ds-idrepo-0" 22:51:02 DEBUG "value": [ 22:51:02 DEBUG }, 22:51:02 DEBUG }, 22:51:02 DEBUG }, 22:51:02 DEBUG }, 22:51:02 DEBUG }, 22:51:02 DEBUG }, 22:51:02 DEBUG }, 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG 22:51:02 DEBUG }, 22:51:02 DEBUG "pod": "ds-cts-2" 22:51:02 DEBUG 22:51:02 DEBUG }, 22:51:02 DEBUG "instance": "10.142.0.43:9100" 22:51:02 DEBUG "pod": "am-555f4db595-kzxsp" 22:51:02 DEBUG "pod": "ds-cts-2" 22:51:02 DEBUG }, 22:51:02 DEBUG }, 22:51:02 DEBUG "value": [ 22:51:02 DEBUG }, 22:51:02 DEBUG }, 22:51:02 DEBUG }, 22:51:02 DEBUG }, 22:51:02 DEBUG 1713304262, 22:51:02 DEBUG "value": [ 22:51:02 DEBUG "value": [ 22:51:02 DEBUG 1713304262, 22:51:02 DEBUG "value": [ 22:51:02 DEBUG "value": [ 22:51:02 DEBUG "value": [ 22:51:02 DEBUG "value": [ 22:51:02 DEBUG "pod": "am-555f4db595-kzxsp" 22:51:02 DEBUG "pod": "ds-idrepo-0" 22:51:02 DEBUG "instance": "10.142.0.43:9100" 22:51:02 DEBUG }, 22:51:02 DEBUG }, 22:51:02 DEBUG 22:51:02 DEBUG "value": [ 22:51:02 DEBUG }, 22:51:02 DEBUG }, 22:51:02 DEBUG }, 22:51:02 DEBUG "value": [ 22:51:02 DEBUG "value": [ 22:51:02 DEBUG 1713304262, 22:51:02 DEBUG "value": [ 22:51:02 DEBUG "value": [ 22:51:02 DEBUG "value": [ 22:51:02 DEBUG "value": [ 22:51:02 DEBUG "0" 22:51:02 DEBUG 1713304262, 22:51:02 DEBUG "value": [ 22:51:02 DEBUG "0" 22:51:02 DEBUG 1713304262, 22:51:02 DEBUG 1713304262, 22:51:02 DEBUG 1713304262, 22:51:02 DEBUG 1713304262, 22:51:02 DEBUG }, 22:51:02 DEBUG }, 22:51:02 DEBUG "value": [ 22:51:02 DEBUG "value": [ 22:51:02 DEBUG "value": [ 22:51:02 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:51:02 DEBUG 1713304262, 22:51:02 DEBUG "value": [ 22:51:02 DEBUG "value": [ 22:51:02 DEBUG "value": [ 22:51:02 DEBUG 1713304262, 22:51:02 DEBUG 1713304262, 22:51:02 DEBUG "2169.7666666666664" 22:51:02 DEBUG 1713304262, 22:51:02 DEBUG 1713304262, 22:51:02 DEBUG 1713304262, 22:51:02 DEBUG 1713304262, 22:51:02 DEBUG ] 22:51:02 DEBUG "0" 22:51:02 DEBUG 1713304262, 22:51:02 DEBUG ] 22:51:02 DEBUG "0" 22:51:02 DEBUG "19973667840" 22:51:02 DEBUG "0.027" 22:51:02 DEBUG "6.745759687533091" 22:51:02 DEBUG "value": [ 22:51:02 DEBUG "value": [ 22:51:02 DEBUG 1713304262, 22:51:02 DEBUG 1713304262, 22:51:02 DEBUG 1713304262, 22:51:02 DEBUG }, 22:51:02 DEBUG "0" 22:51:02 DEBUG 1713304262, 22:51:02 DEBUG 1713304262, 22:51:02 DEBUG 1713304262, 22:51:02 DEBUG "0" 22:51:02 DEBUG "2168.866666666667" 22:51:02 DEBUG ] 22:51:02 DEBUG "0" 22:51:02 DEBUG "0.00043333333333350765" 22:51:02 DEBUG "49072785.46666666" 22:51:02 DEBUG "0" 22:51:02 DEBUG } 22:51:02 DEBUG ] 22:51:02 DEBUG "559.8305200574046" 22:51:02 DEBUG } 22:51:02 DEBUG ] 22:51:02 DEBUG ] 22:51:02 DEBUG ] 22:51:02 DEBUG ] 22:51:02 DEBUG 1713304262, 22:51:02 DEBUG 1713304262, 22:51:02 DEBUG "1999.6" 22:51:02 DEBUG "49082402.699999996" 22:51:02 DEBUG "0" 22:51:02 DEBUG "value": [ 22:51:02 DEBUG ] 22:51:02 DEBUG "0" 22:51:02 DEBUG "2168.266666666667" 22:51:02 DEBUG "0" 22:51:02 DEBUG ] 22:51:02 DEBUG ] 22:51:02 DEBUG } 22:51:02 DEBUG ] 22:51:02 DEBUG ] 22:51:02 DEBUG ] 22:51:02 DEBUG ] 22:51:02 DEBUG ] 22:51:02 DEBUG } 22:51:02 DEBUG ] 22:51:02 DEBUG ] 22:51:02 DEBUG } 22:51:02 DEBUG } 22:51:02 DEBUG } 22:51:02 DEBUG } 22:51:02 DEBUG "3563231.5" 22:51:02 DEBUG "0" 22:51:02 DEBUG ] 22:51:02 DEBUG ] 22:51:02 DEBUG ] 22:51:02 DEBUG 1713304262, 22:51:02 DEBUG } 22:51:02 DEBUG ] 22:51:02 DEBUG ] 22:51:02 DEBUG ] 22:51:02 DEBUG } 22:51:02 DEBUG } 22:51:02 DEBUG ] 22:51:02 DEBUG } 22:51:02 DEBUG } 22:51:02 DEBUG } 22:51:02 DEBUG } 22:51:02 DEBUG } 22:51:02 DEBUG ] 22:51:02 DEBUG } 22:51:02 DEBUG } 22:51:02 DEBUG ] 22:51:02 DEBUG ] 22:51:02 DEBUG ] 22:51:02 DEBUG ] 22:51:02 DEBUG ] 22:51:02 DEBUG ] 22:51:02 DEBUG } 22:51:02 DEBUG } 22:51:02 DEBUG } 22:51:02 DEBUG "1682550784" 22:51:02 DEBUG ] 22:51:02 DEBUG } 22:51:02 DEBUG } 22:51:02 DEBUG } 22:51:02 DEBUG ] 22:51:02 DEBUG ] 22:51:02 DEBUG } 22:51:02 DEBUG ] 22:51:02 DEBUG ] 22:51:02 DEBUG ] 22:51:02 DEBUG ] 22:51:02 DEBUG } 22:51:02 DEBUG } 22:51:02 DEBUG ] 22:51:02 DEBUG } 22:51:02 DEBUG } 22:51:02 DEBUG } 22:51:02 DEBUG } 22:51:02 DEBUG } 22:51:02 DEBUG } 22:51:02 DEBUG } 22:51:02 DEBUG ] 22:51:02 DEBUG ] 22:51:02 DEBUG ] 22:51:02 DEBUG ] 22:51:02 DEBUG } 22:51:02 DEBUG ] 22:51:02 DEBUG ] 22:51:02 DEBUG ] 22:51:02 DEBUG } 22:51:02 DEBUG } 22:51:02 DEBUG } 22:51:02 DEBUG } 22:51:02 DEBUG } 22:51:02 DEBUG } 22:51:02 DEBUG } 22:51:02 DEBUG } 22:51:02 DEBUG } 22:51:02 DEBUG } 22:51:02 DEBUG } 22:51:02 DEBUG } 22:51:02 DEBUG ] 22:51:02 DEBUG ] 22:51:02 DEBUG } 22:51:02 DEBUG } 22:51:02 DEBUG } 22:51:02 DEBUG } 22:51:02 DEBUG } 22:51:02 DEBUG } 22:51:02 DEBUG } 22:51:02 DEBUG } 22:51:02 DEBUG } 22:51:02 DEBUG } 22:51:02 DEBUG } 22:51:02 DEBUG } 22:51:02 DEBUG } 22:51:02 DEBUG } 22:51:02 DEBUG } 22:51:02 DEBUG } 22:51:02 DEBUG } 22:51:02 DEBUG } 22:51:02 DEBUG } 22:51:02 DEBUG } 22:51:02 DEBUG ] 22:51:02 DEBUG } 22:51:02 DEBUG } 22:51:02 DEBUG } 22:51:02 DEBUG } 22:51:02 DEBUG } 22:51:02 DEBUG } 22:51:02 DEBUG } 22:51:02 DEBUG } 22:51:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713304317" 22:51:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304317" 22:51:57 ERROR [http_cmd]: ERROR 22:51:57 INFO [http_cmd]: http status code OK 22:51:57 ERROR --- status code --- 22:51:57 DEBUG --- status code --- 22:51:57 ERROR http status code is 400 (expected [200, 401, 403, 404]) 22:51:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:51:57 ERROR --- http response --- 22:51:57 DEBUG --- http response --- 22:51:57 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 22:51:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 22:51:57 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 22:52:00 INFO 22:52:00 INFO [loop_until]: kubectl --namespace=xlou top pods 22:52:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:52:00 INFO [loop_until]: OK (rc = 0) 22:52:00 DEBUG --- stdout --- 22:52:00 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 9Mi am-555f4db595-4lbss 13371m 8675Mi am-555f4db595-fz8sn 13220m 8555Mi am-555f4db595-kzxsp 13726m 8946Mi ds-cts-0 20046m 12785Mi ds-cts-1 18933m 12922Mi ds-cts-2 18081m 12953Mi ds-idrepo-0 5854m 5028Mi ds-idrepo-1 5656m 2908Mi ds-idrepo-2 5497m 3220Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 3m 1207Mi idm-b85f9d978-rl76f 4m 1226Mi lodemon-bc756c647-cr7x6 2m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-7c9b667974-9mr2s 6751m 1512Mi 22:52:00 DEBUG --- stderr --- 22:52:00 DEBUG 22:52:02 INFO 22:52:02 INFO [loop_until]: kubectl --namespace=xlou top node 22:52:02 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:52:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304322" 22:52:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304322" 22:52:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713304322" 22:52:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304322" 22:52:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304322" 22:52:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304322" 22:52:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304322" 22:52:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304322" 22:52:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304322" 22:52:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304322" 22:52:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304322" 22:52:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304322" 22:52:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304322" 22:52:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304322" 22:52:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304322" 22:52:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713304322" 22:52:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713304322" 22:52:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304322" 22:52:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713304322" 22:52:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304322" 22:52:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304322" 22:52:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304322" 22:52:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304322" 22:52:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304322" 22:52:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304322" 22:52:02 INFO [loop_until]: OK (rc = 0) 22:52:02 INFO [http_cmd]: http status code OK 22:52:02 INFO [http_cmd]: http status code OK 22:52:02 INFO [http_cmd]: http status code OK 22:52:02 INFO [http_cmd]: http status code OK 22:52:02 INFO [http_cmd]: http status code OK 22:52:02 INFO [http_cmd]: http status code OK 22:52:02 INFO [http_cmd]: http status code OK 22:52:02 INFO [http_cmd]: http status code OK 22:52:02 INFO [http_cmd]: http status code OK 22:52:02 INFO [http_cmd]: http status code OK 22:52:02 INFO [http_cmd]: http status code OK 22:52:02 INFO [http_cmd]: http status code OK 22:52:02 INFO [http_cmd]: http status code OK 22:52:02 INFO [http_cmd]: http status code OK 22:52:02 DEBUG --- stdout --- 22:52:02 INFO [http_cmd]: http status code OK 22:52:02 DEBUG --- status code --- 22:52:02 DEBUG --- status code --- 22:52:02 INFO [http_cmd]: http status code OK 22:52:02 DEBUG --- status code --- 22:52:02 DEBUG --- status code --- 22:52:02 DEBUG --- status code --- 22:52:02 DEBUG --- status code --- 22:52:02 DEBUG --- status code --- 22:52:02 DEBUG --- status code --- 22:52:02 DEBUG --- status code --- 22:52:02 DEBUG --- status code --- 22:52:02 DEBUG --- status code --- 22:52:02 DEBUG --- status code --- 22:52:02 DEBUG --- status code --- 22:52:02 DEBUG --- status code --- 22:52:02 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 63m 0% 5078Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 18553m 62% 15589Mi 13% gke-xlou-cdm-default-pool-1b0769c8-nhlf 13149m 44% 10781Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 81m 0% 2652Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 5562m 18% 6563Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 20903m 70% 17253Mi 15% gke-xlou-cdm-default-pool-1b0769c8-xnmp 58m 0% 3151Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 13188m 44% 10461Mi 9% gke-xlou-cdm-ds-0f601751-hmm7 6036m 20% 5653Mi 5% gke-xlou-cdm-ds-0f601751-j488 18455m 61% 14334Mi 12% gke-xlou-cdm-ds-0f601751-mc5d 11606m 38% 3342Mi 2% gke-xlou-cdm-ds-0f601751-nj6q 14004m 46% 10831Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 6013m 20% 7361Mi 6% gke-xlou-cdm-frontend-a8771548-i82n 6899m 43% 6944Mi 11% 22:52:02 DEBUG --- status code --- 22:52:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:52:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:52:02 DEBUG --- status code --- 22:52:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:52:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:52:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:52:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:52:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:52:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:52:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:52:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:52:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:52:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:52:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:52:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:52:02 DEBUG --- stderr --- 22:52:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:52:02 DEBUG --- http response --- 22:52:02 DEBUG --- http response --- 22:52:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:52:02 DEBUG --- http response --- 22:52:02 DEBUG --- http response --- 22:52:02 DEBUG --- http response --- 22:52:02 DEBUG --- http response --- 22:52:02 DEBUG --- http response --- 22:52:02 DEBUG --- http response --- 22:52:02 DEBUG --- http response --- 22:52:02 DEBUG --- http response --- 22:52:02 INFO [http_cmd]: http status code OK 22:52:02 INFO [http_cmd]: http status code OK 22:52:02 DEBUG --- http response --- 22:52:02 DEBUG --- http response --- 22:52:02 INFO [http_cmd]: http status code OK 22:52:02 INFO [http_cmd]: http status code OK 22:52:02 DEBUG --- http response --- 22:52:02 INFO [http_cmd]: http status code OK 22:52:02 INFO [http_cmd]: http status code OK 22:52:02 INFO [http_cmd]: http status code OK 22:52:02 DEBUG --- http response --- 22:52:02 INFO [http_cmd]: http status code OK 22:52:02 DEBUG 22:52:02 DEBUG --- http response --- 22:52:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:52:02 INFO [http_cmd]: http status code OK 22:52:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:52:02 DEBUG --- http response --- 22:52:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:52:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 22:52:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:52:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:52:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:52:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:52:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:52:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:52:02 DEBUG --- status code --- 22:52:02 DEBUG --- status code --- 22:52:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:52:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 22:52:02 DEBUG --- status code --- 22:52:02 DEBUG --- status code --- 22:52:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:52:02 DEBUG --- status code --- 22:52:02 DEBUG --- status code --- 22:52:02 DEBUG --- status code --- 22:52:02 DEBUG --- status code --- 22:52:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 22:52:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:52:02 DEBUG ----- output ----- 22:52:02 DEBUG --- status code --- 22:52:02 DEBUG ----- output ----- 22:52:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:52:02 DEBUG ----- output ----- 22:52:02 DEBUG ----- output ----- 22:52:02 DEBUG ----- output ----- 22:52:02 DEBUG ----- output ----- 22:52:02 DEBUG ----- output ----- 22:52:02 DEBUG ----- output ----- 22:52:02 DEBUG ----- output ----- 22:52:02 DEBUG ----- output ----- 22:52:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:52:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:52:02 DEBUG ----- output ----- 22:52:02 DEBUG ----- output ----- 22:52:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:52:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:52:02 DEBUG ----- output ----- 22:52:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:52:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:52:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:52:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:52:02 DEBUG ----- output ----- 22:52:02 DEBUG ----- output ----- 22:52:02 DEBUG { 22:52:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:52:02 DEBUG { 22:52:02 DEBUG ----- output ----- 22:52:02 DEBUG { 22:52:02 DEBUG { 22:52:02 DEBUG { 22:52:02 DEBUG { 22:52:02 DEBUG { 22:52:02 DEBUG { 22:52:02 DEBUG { 22:52:02 DEBUG { 22:52:02 DEBUG --- http response --- 22:52:02 DEBUG --- http response --- 22:52:02 DEBUG { 22:52:02 DEBUG { 22:52:02 DEBUG --- http response --- 22:52:02 DEBUG --- http response --- 22:52:02 DEBUG { 22:52:02 DEBUG --- http response --- 22:52:02 DEBUG --- http response --- 22:52:02 DEBUG --- http response --- 22:52:02 DEBUG --- http response --- 22:52:02 DEBUG { 22:52:02 DEBUG { 22:52:02 DEBUG "status": "success", 22:52:02 DEBUG --- http response --- 22:52:02 DEBUG "status": "success", 22:52:02 DEBUG { 22:52:02 DEBUG "status": "success", 22:52:02 DEBUG "status": "success", 22:52:02 DEBUG "status": "success", 22:52:02 DEBUG "status": "success", 22:52:02 DEBUG "status": "success", 22:52:02 DEBUG "status": "success", 22:52:02 DEBUG "status": "success", 22:52:02 DEBUG "status": "success", 22:52:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:52:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:52:02 DEBUG "status": "success", 22:52:02 DEBUG "status": "success", 22:52:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:52:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:52:02 DEBUG "status": "success", 22:52:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:52:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:52:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:52:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 22:52:02 DEBUG "status": "success", 22:52:02 DEBUG "status": "success", 22:52:02 DEBUG "data": { 22:52:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:52:02 DEBUG "data": { 22:52:02 DEBUG "status": "success", 22:52:02 DEBUG "data": { 22:52:02 DEBUG "data": { 22:52:02 DEBUG "data": { 22:52:02 DEBUG "data": { 22:52:02 DEBUG "data": { 22:52:02 DEBUG "data": { 22:52:02 DEBUG "data": { 22:52:02 DEBUG "data": { 22:52:02 DEBUG ----- output ----- 22:52:02 DEBUG ----- output ----- 22:52:02 DEBUG "data": { 22:52:02 DEBUG "data": { 22:52:02 DEBUG ----- output ----- 22:52:02 DEBUG ----- output ----- 22:52:02 DEBUG "data": { 22:52:02 DEBUG ----- output ----- 22:52:02 DEBUG ----- output ----- 22:52:02 DEBUG ----- output ----- 22:52:02 DEBUG ----- output ----- 22:52:02 DEBUG "data": { 22:52:02 DEBUG "data": { 22:52:02 DEBUG "resultType": "vector", 22:52:02 DEBUG ----- output ----- 22:52:02 DEBUG "resultType": "vector", 22:52:02 DEBUG "data": { 22:52:02 DEBUG "resultType": "vector", 22:52:02 DEBUG "resultType": "vector", 22:52:02 DEBUG "resultType": "vector", 22:52:02 DEBUG "resultType": "vector", 22:52:02 DEBUG "resultType": "vector", 22:52:02 DEBUG "resultType": "vector", 22:52:02 DEBUG "resultType": "vector", 22:52:02 DEBUG "resultType": "vector", 22:52:02 DEBUG { 22:52:02 DEBUG { 22:52:02 DEBUG "resultType": "vector", 22:52:02 DEBUG "resultType": "vector", 22:52:02 DEBUG { 22:52:02 DEBUG { 22:52:02 DEBUG "resultType": "vector", 22:52:02 DEBUG { 22:52:02 DEBUG { 22:52:02 DEBUG { 22:52:02 DEBUG { 22:52:02 DEBUG "resultType": "vector", 22:52:02 DEBUG "resultType": "vector", 22:52:02 DEBUG "result": [ 22:52:02 DEBUG { 22:52:02 DEBUG "result": [ 22:52:02 DEBUG "resultType": "vector", 22:52:02 DEBUG "result": [ 22:52:02 DEBUG "result": [ 22:52:02 DEBUG "result": [ 22:52:02 DEBUG "result": [ 22:52:02 DEBUG "result": [ 22:52:02 DEBUG "result": [ 22:52:02 DEBUG "result": [ 22:52:02 DEBUG "result": [ 22:52:02 DEBUG "status": "success", 22:52:02 DEBUG "status": "success", 22:52:02 DEBUG "result": [ 22:52:02 DEBUG "result": [ 22:52:02 DEBUG "status": "success", 22:52:02 DEBUG "status": "success", 22:52:02 DEBUG "result": [ 22:52:02 DEBUG "status": "success", 22:52:02 DEBUG "status": "success", 22:52:02 DEBUG "status": "success", 22:52:02 DEBUG "status": "success", 22:52:02 DEBUG "result": [ 22:52:02 DEBUG "result": [ 22:52:02 DEBUG { 22:52:02 DEBUG "status": "success", 22:52:02 DEBUG { 22:52:02 DEBUG "result": [ 22:52:02 DEBUG { 22:52:02 DEBUG { 22:52:02 DEBUG { 22:52:02 DEBUG { 22:52:02 DEBUG { 22:52:02 DEBUG { 22:52:02 DEBUG { 22:52:02 DEBUG { 22:52:02 DEBUG "data": { 22:52:02 DEBUG "data": { 22:52:02 DEBUG { 22:52:02 DEBUG { 22:52:02 DEBUG "data": { 22:52:02 DEBUG "data": { 22:52:02 DEBUG { 22:52:02 DEBUG "data": { 22:52:02 DEBUG "data": { 22:52:02 DEBUG "data": { 22:52:02 DEBUG "data": { 22:52:02 DEBUG { 22:52:02 DEBUG { 22:52:02 DEBUG "metric": { 22:52:02 DEBUG "data": { 22:52:02 DEBUG "metric": { 22:52:02 DEBUG { 22:52:02 DEBUG "metric": { 22:52:02 DEBUG "metric": { 22:52:02 DEBUG "metric": { 22:52:02 DEBUG "metric": { 22:52:02 DEBUG "metric": { 22:52:02 DEBUG "metric": { 22:52:02 DEBUG "metric": { 22:52:02 DEBUG "metric": { 22:52:02 DEBUG "resultType": "vector", 22:52:02 DEBUG "resultType": "vector", 22:52:02 DEBUG "metric": { 22:52:02 DEBUG "metric": { 22:52:02 DEBUG "resultType": "vector", 22:52:02 DEBUG "resultType": "vector", 22:52:02 DEBUG "metric": { 22:52:02 DEBUG "resultType": "vector", 22:52:02 DEBUG "resultType": "vector", 22:52:02 DEBUG "resultType": "vector", 22:52:02 DEBUG "resultType": "vector", 22:52:02 DEBUG "metric": { 22:52:02 DEBUG "metric": { 22:52:02 DEBUG "pod": "ds-idrepo-2" 22:52:02 DEBUG "resultType": "vector", 22:52:02 DEBUG "pod": "am-555f4db595-fz8sn" 22:52:02 DEBUG "metric": { 22:52:02 DEBUG "instance": "10.142.0.107:9100" 22:52:02 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:52:02 DEBUG "pod": "ds-idrepo-2" 22:52:02 DEBUG "pod": "ds-idrepo-2" 22:52:02 DEBUG "pod": "am-555f4db595-fz8sn" 22:52:02 DEBUG "pod": "ds-idrepo-2" 22:52:02 DEBUG "instance": "10.142.0.107:9100" 22:52:02 DEBUG "pod": "ds-idrepo-2" 22:52:02 DEBUG "result": [ 22:52:02 DEBUG "result": [ 22:52:02 DEBUG "pod": "am-555f4db595-fz8sn" 22:52:02 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:52:02 DEBUG "result": [ 22:52:02 DEBUG "result": [ 22:52:02 DEBUG "pod": "am-555f4db595-fz8sn" 22:52:02 DEBUG }, 22:52:02 DEBUG "result": [ 22:52:02 DEBUG "result": [ 22:52:02 DEBUG "result": [ 22:52:02 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:52:02 DEBUG "pod": "ds-idrepo-2" 22:52:02 DEBUG }, 22:52:02 DEBUG "result": [ 22:52:02 DEBUG }, 22:52:02 DEBUG "pod": "am-555f4db595-fz8sn" 22:52:02 DEBUG }, 22:52:02 DEBUG }, 22:52:02 DEBUG }, 22:52:02 DEBUG }, 22:52:02 DEBUG }, 22:52:02 DEBUG }, 22:52:02 DEBUG }, 22:52:02 DEBUG }, 22:52:02 DEBUG { 22:52:02 DEBUG { 22:52:02 DEBUG }, 22:52:02 DEBUG }, 22:52:02 DEBUG { 22:52:02 DEBUG { 22:52:02 DEBUG "result": [ 22:52:02 DEBUG "value": [ 22:52:02 DEBUG { 22:52:02 DEBUG { 22:52:02 DEBUG { 22:52:02 DEBUG }, 22:52:02 DEBUG }, 22:52:02 DEBUG "value": [ 22:52:02 DEBUG { 22:52:02 DEBUG "value": [ 22:52:02 DEBUG }, 22:52:02 DEBUG "value": [ 22:52:02 DEBUG "value": [ 22:52:02 DEBUG "value": [ 22:52:02 DEBUG "value": [ 22:52:02 DEBUG "value": [ 22:52:02 DEBUG "value": [ 22:52:02 DEBUG "value": [ 22:52:02 DEBUG "value": [ 22:52:02 DEBUG "metric": { 22:52:02 DEBUG "metric": { 22:52:02 DEBUG "value": [ 22:52:02 DEBUG "value": [ 22:52:02 DEBUG "metric": { 22:52:02 DEBUG "metric": { 22:52:02 DEBUG { 22:52:02 DEBUG 22:52:02 DEBUG "metric": { 22:52:02 DEBUG "metric": { 22:52:02 DEBUG "metric": { 22:52:02 DEBUG "value": [ 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG "metric": { 22:52:02 DEBUG 22:52:02 DEBUG "value": [ 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:52:02 DEBUG "pod": "am-555f4db595-fz8sn" 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG "instance": "10.142.0.107:9100" 22:52:02 DEBUG "pod": "ds-idrepo-2" 22:52:02 DEBUG "metric": { 22:52:02 DEBUG 22:52:02 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:52:02 DEBUG "pod": "am-555f4db595-fz8sn" 22:52:02 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG "instance": "10.142.0.107:9100" 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG }, 22:52:02 DEBUG }, 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG }, 22:52:02 DEBUG "value": [ 22:52:02 DEBUG }, 22:52:02 DEBUG "value": [ 22:52:02 DEBUG 22:52:02 DEBUG }, 22:52:02 DEBUG }, 22:52:02 DEBUG }, 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG }, 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG "value": [ 22:52:02 DEBUG "value": [ 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG "value": [ 22:52:02 DEBUG 22:52:02 DEBUG "instance": "10.142.0.107:9100" 22:52:02 DEBUG 22:52:02 DEBUG [...] 22:52:02 DEBUG "value": [ 22:52:02 DEBUG "value": [ 22:52:02 DEBUG "value": [ 22:52:02 DEBUG [...] 22:52:02 DEBUG [...] 22:52:02 DEBUG "value": [ 22:52:02 DEBUG [...] 22:52:02 DEBUG 22:52:02 DEBUG [...] 22:52:02 DEBUG [...] 22:52:02 DEBUG [...] 22:52:02 DEBUG [...] 22:52:02 DEBUG [...] 22:52:02 DEBUG [...] 22:52:02 DEBUG [...] 22:52:02 DEBUG [...] 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG [...] 22:52:02 DEBUG [...] 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG }, 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG [...] 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG "value": [ 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG [...] 22:52:02 DEBUG 22:52:02 DEBUG [...] 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG [...] 22:52:02 DEBUG [...] 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG [...] 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG "pod": "am-555f4db595-kzxsp" 22:52:02 DEBUG [...] 22:52:02 DEBUG [...] 22:52:02 DEBUG [...] 22:52:02 DEBUG "pod": "am-555f4db595-kzxsp" 22:52:02 DEBUG "pod": "ds-cts-2" 22:52:02 DEBUG [...] 22:52:02 DEBUG "pod": "am-555f4db595-kzxsp" 22:52:02 DEBUG 22:52:02 DEBUG "instance": "10.142.0.43:9100" 22:52:02 DEBUG "pod": "am-555f4db595-kzxsp" 22:52:02 DEBUG "pod": "ds-idrepo-0" 22:52:02 DEBUG "pod": "ds-idrepo-0" 22:52:02 DEBUG "pod": "am-555f4db595-kzxsp" 22:52:02 DEBUG "pod": "ds-idrepo-0" 22:52:02 DEBUG "instance": "10.142.0.43:9100" 22:52:02 DEBUG "pod": "ds-idrepo-0" 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG "pod": "am-555f4db595-kzxsp" 22:52:02 DEBUG "pod": "am-555f4db595-kzxsp" 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG }, 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG }, 22:52:02 DEBUG }, 22:52:02 DEBUG 22:52:02 DEBUG }, 22:52:02 DEBUG "pod": "am-555f4db595-kzxsp" 22:52:02 DEBUG }, 22:52:02 DEBUG }, 22:52:02 DEBUG }, 22:52:02 DEBUG }, 22:52:02 DEBUG }, 22:52:02 DEBUG }, 22:52:02 DEBUG }, 22:52:02 DEBUG }, 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG }, 22:52:02 DEBUG }, 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG [...] 22:52:02 DEBUG 22:52:02 DEBUG "value": [ 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG "value": [ 22:52:02 DEBUG "value": [ 22:52:02 DEBUG 22:52:02 DEBUG "value": [ 22:52:02 DEBUG }, 22:52:02 DEBUG "value": [ 22:52:02 DEBUG "value": [ 22:52:02 DEBUG "value": [ 22:52:02 DEBUG "value": [ 22:52:02 DEBUG "value": [ 22:52:02 DEBUG "value": [ 22:52:02 DEBUG "value": [ 22:52:02 DEBUG "value": [ 22:52:02 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:52:02 DEBUG 22:52:02 DEBUG "value": [ 22:52:02 DEBUG "value": [ 22:52:02 DEBUG 22:52:02 DEBUG "pod": "ds-idrepo-0" 22:52:02 DEBUG 22:52:02 DEBUG "pod": "ds-idrepo-0" 22:52:02 DEBUG 1713304322, 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 22:52:02 DEBUG 1713304322, 22:52:02 DEBUG 1713304322, 22:52:02 DEBUG 22:52:02 DEBUG 1713304322, 22:52:02 DEBUG "value": [ 22:52:02 DEBUG 1713304322, 22:52:02 DEBUG 1713304322, 22:52:02 DEBUG 1713304322, 22:52:02 DEBUG 1713304322, 22:52:02 DEBUG 1713304322, 22:52:02 DEBUG 1713304322, 22:52:02 DEBUG "0" 22:52:02 DEBUG 1713304322, 22:52:02 DEBUG }, 22:52:02 DEBUG 22:52:02 DEBUG "pod": "am-555f4db595-kzxsp" 22:52:02 DEBUG 1713304322, 22:52:02 DEBUG "instance": "10.142.0.43:9100" 22:52:02 DEBUG }, 22:52:02 DEBUG 22:52:02 DEBUG }, 22:52:02 DEBUG "2164.0333333333333" 22:52:02 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:52:02 DEBUG "pod": "am-555f4db595-kzxsp" 22:52:02 DEBUG "pod": "am-555f4db595-kzxsp" 22:52:02 DEBUG }, 22:52:02 DEBUG "0" 22:52:02 DEBUG "instance": "10.142.0.43:9100" 22:52:02 DEBUG "0.027" 22:52:02 DEBUG 1713304322, 22:52:02 DEBUG "19974407168" 22:52:02 DEBUG "281027408.8252227" 22:52:02 DEBUG "0" 22:52:02 DEBUG "0" 22:52:02 DEBUG "0" 22:52:02 DEBUG 1713304322, 22:52:02 DEBUG ] 22:52:02 DEBUG "0.0006999999999995529" 22:52:02 DEBUG "value": [ 22:52:02 DEBUG 1713304322, 22:52:02 DEBUG }, 22:52:02 DEBUG "2164.4666666666667" 22:52:02 DEBUG }, 22:52:02 DEBUG "value": [ 22:52:02 DEBUG 22:52:02 DEBUG "value": [ 22:52:02 DEBUG ] 22:52:02 DEBUG }, 22:52:02 DEBUG }, 22:52:02 DEBUG "0" 22:52:02 DEBUG "value": [ 22:52:02 DEBUG ] 22:52:02 DEBUG }, 22:52:02 DEBUG ] 22:52:02 DEBUG "1996.3999999999999" 22:52:02 DEBUG ] 22:52:02 DEBUG ] 22:52:02 DEBUG ] 22:52:02 DEBUG ] 22:52:02 DEBUG ] 22:52:02 DEBUG "0" 22:52:02 DEBUG } 22:52:02 DEBUG ] 22:52:02 DEBUG 1713304322, 22:52:02 DEBUG "490.6905076365607" 22:52:02 DEBUG "value": [ 22:52:02 DEBUG ] 22:52:02 DEBUG "value": [ 22:52:02 DEBUG 1713304322, 22:52:02 DEBUG "instance": "10.142.0.43:9100" 22:52:02 DEBUG 1713304322, 22:52:02 DEBUG } 22:52:02 DEBUG "value": [ 22:52:02 DEBUG "value": [ 22:52:02 DEBUG ] 22:52:02 DEBUG 1713304322, 22:52:02 DEBUG } 22:52:02 DEBUG "value": [ 22:52:02 DEBUG } 22:52:02 DEBUG ] 22:52:02 DEBUG } 22:52:02 DEBUG } 22:52:02 DEBUG } 22:52:02 DEBUG } 22:52:02 DEBUG } 22:52:02 DEBUG ] 22:52:02 DEBUG ] 22:52:02 DEBUG } 22:52:02 DEBUG "1684725760" 22:52:02 DEBUG ] 22:52:02 DEBUG 1713304322, 22:52:02 DEBUG } 22:52:02 DEBUG 1713304322, 22:52:02 DEBUG "0" 22:52:02 DEBUG }, 22:52:02 DEBUG "0" 22:52:02 DEBUG ] 22:52:02 DEBUG 1713304322, 22:52:02 DEBUG 1713304322, 22:52:02 DEBUG } 22:52:02 DEBUG "7.920068575912578" 22:52:02 DEBUG ] 22:52:02 DEBUG 1713304322, 22:52:02 DEBUG ] 22:52:02 DEBUG } 22:52:02 DEBUG ] 22:52:02 DEBUG ] 22:52:02 DEBUG ] 22:52:02 DEBUG ] 22:52:02 DEBUG ] 22:52:02 DEBUG } 22:52:02 DEBUG } 22:52:02 DEBUG ] 22:52:02 DEBUG ] 22:52:02 DEBUG } 22:52:02 DEBUG "3573683" 22:52:02 DEBUG ] 22:52:02 DEBUG "49394586.46666666" 22:52:02 DEBUG ] 22:52:02 DEBUG "value": [ 22:52:02 DEBUG ] 22:52:02 DEBUG } 22:52:02 DEBUG "6.808220064739652" 22:52:02 DEBUG "2162.2" 22:52:02 DEBUG ] 22:52:02 DEBUG ] 22:52:02 DEBUG } 22:52:02 DEBUG "0" 22:52:02 DEBUG } 22:52:02 DEBUG } 22:52:02 DEBUG } 22:52:02 DEBUG } 22:52:02 DEBUG } 22:52:02 DEBUG } 22:52:02 DEBUG } 22:52:02 DEBUG ] 22:52:02 DEBUG } 22:52:02 DEBUG } 22:52:02 DEBUG } 22:52:02 DEBUG ] 22:52:02 DEBUG ] 22:52:02 DEBUG } 22:52:02 DEBUG ] 22:52:02 DEBUG } 22:52:02 DEBUG 1713304322, 22:52:02 DEBUG } 22:52:02 DEBUG } 22:52:02 DEBUG ] 22:52:02 DEBUG ] 22:52:02 DEBUG } 22:52:02 DEBUG } 22:52:02 DEBUG } 22:52:02 DEBUG ] 22:52:02 DEBUG ] 22:52:02 DEBUG } 22:52:02 DEBUG } 22:52:02 DEBUG } 22:52:02 DEBUG } 22:52:02 DEBUG } 22:52:02 DEBUG } 22:52:02 DEBUG } 22:52:02 DEBUG ] 22:52:02 DEBUG } 22:52:02 DEBUG } 22:52:02 DEBUG } 22:52:02 DEBUG } 22:52:02 DEBUG ] 22:52:02 DEBUG "49384731.3" 22:52:02 DEBUG ] 22:52:02 DEBUG } 22:52:02 DEBUG } 22:52:02 DEBUG } 22:52:02 DEBUG ] 22:52:02 DEBUG } 22:52:02 DEBUG } 22:52:02 DEBUG } 22:52:02 DEBUG } 22:52:02 DEBUG } 22:52:02 DEBUG ] 22:52:02 DEBUG ] 22:52:02 DEBUG } 22:52:02 DEBUG ] 22:52:02 DEBUG } 22:52:02 DEBUG ] 22:52:02 DEBUG ] 22:52:02 DEBUG } 22:52:02 DEBUG ] 22:52:02 DEBUG } 22:52:02 DEBUG } 22:52:02 DEBUG } 22:52:02 DEBUG } 22:52:02 DEBUG } 22:52:02 DEBUG } 22:52:02 DEBUG } 22:52:02 DEBUG } 22:52:02 DEBUG } 22:52:02 DEBUG } 22:52:02 DEBUG } 22:52:02 DEBUG } 22:52:02 DEBUG } 22:52:02 DEBUG ] 22:52:02 DEBUG } 22:52:02 DEBUG } 22:52:02 DEBUG } 22:52:02 DEBUG } 22:52:02 DEBUG } 22:52:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304377" 22:52:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713304377" 22:52:57 INFO [http_cmd]: http status code OK 22:52:57 ERROR [http_cmd]: ERROR 22:52:57 DEBUG --- status code --- 22:52:57 ERROR --- status code --- 22:52:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:52:57 ERROR http status code is 400 (expected [200, 401, 403, 404]) 22:52:57 DEBUG --- http response --- 22:52:57 ERROR --- http response --- 22:52:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 22:52:57 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 22:52:57 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 22:53:00 INFO 22:53:00 INFO [loop_until]: kubectl --namespace=xlou top pods 22:53:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:53:00 INFO [loop_until]: OK (rc = 0) 22:53:00 DEBUG --- stdout --- 22:53:00 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 9Mi am-555f4db595-4lbss 13356m 8677Mi am-555f4db595-fz8sn 14041m 8557Mi am-555f4db595-kzxsp 13434m 8971Mi ds-cts-0 19727m 13086Mi ds-cts-1 18151m 12752Mi ds-cts-2 17926m 12491Mi ds-idrepo-0 5821m 5029Mi ds-idrepo-1 5440m 2907Mi ds-idrepo-2 5336m 3220Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 4m 1207Mi idm-b85f9d978-rl76f 4m 1225Mi lodemon-bc756c647-cr7x6 2m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-7c9b667974-9mr2s 6845m 1538Mi 22:53:00 DEBUG --- stderr --- 22:53:00 DEBUG 22:53:02 INFO 22:53:02 INFO [loop_until]: kubectl --namespace=xlou top node 22:53:02 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:53:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713304382" 22:53:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304382" 22:53:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304382" 22:53:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713304382" 22:53:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304382" 22:53:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304382" 22:53:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304382" 22:53:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304382" 22:53:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304382" 22:53:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304382" 22:53:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304382" 22:53:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304382" 22:53:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304382" 22:53:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304382" 22:53:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304382" 22:53:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304382" 22:53:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713304382" 22:53:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304382" 22:53:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304382" 22:53:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304382" 22:53:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713304382" 22:53:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304382" 22:53:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304382" 22:53:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304382" 22:53:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304382" 22:53:02 INFO [loop_until]: OK (rc = 0) 22:53:02 INFO [http_cmd]: http status code OK 22:53:02 INFO [http_cmd]: http status code OK 22:53:02 INFO [http_cmd]: http status code OK 22:53:02 INFO [http_cmd]: http status code OK 22:53:02 INFO [http_cmd]: http status code OK 22:53:02 INFO [http_cmd]: http status code OK 22:53:02 INFO [http_cmd]: http status code OK 22:53:02 INFO [http_cmd]: http status code OK 22:53:02 INFO [http_cmd]: http status code OK 22:53:02 INFO [http_cmd]: http status code OK 22:53:02 INFO [http_cmd]: http status code OK 22:53:02 INFO [http_cmd]: http status code OK 22:53:02 INFO [http_cmd]: http status code OK 22:53:02 INFO [http_cmd]: http status code OK 22:53:02 INFO [http_cmd]: http status code OK 22:53:02 INFO [http_cmd]: http status code OK 22:53:02 INFO [http_cmd]: http status code OK 22:53:02 DEBUG --- stdout --- 22:53:02 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 5082Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 18145m 60% 15244Mi 13% gke-xlou-cdm-default-pool-1b0769c8-nhlf 12995m 43% 10833Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 90m 0% 2639Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 5589m 18% 6572Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 20918m 70% 17525Mi 15% gke-xlou-cdm-default-pool-1b0769c8-xnmp 55m 0% 3156Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 13600m 45% 10468Mi 9% gke-xlou-cdm-ds-0f601751-hmm7 5917m 19% 5661Mi 5% gke-xlou-cdm-ds-0f601751-j488 17892m 59% 14537Mi 13% gke-xlou-cdm-ds-0f601751-mc5d 11678m 39% 3349Mi 3% gke-xlou-cdm-ds-0f601751-nj6q 13757m 46% 10868Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 6019m 20% 7350Mi 6% gke-xlou-cdm-frontend-a8771548-i82n 6987m 43% 6941Mi 11% 22:53:02 DEBUG --- status code --- 22:53:02 DEBUG --- status code --- 22:53:02 DEBUG --- status code --- 22:53:02 DEBUG --- status code --- 22:53:02 DEBUG --- status code --- 22:53:02 DEBUG --- status code --- 22:53:02 DEBUG --- status code --- 22:53:02 DEBUG --- status code --- 22:53:02 DEBUG --- status code --- 22:53:02 DEBUG --- status code --- 22:53:02 DEBUG --- status code --- 22:53:02 INFO [http_cmd]: http status code OK 22:53:02 INFO [http_cmd]: http status code OK 22:53:02 INFO [http_cmd]: http status code OK 22:53:02 DEBUG --- status code --- 22:53:02 DEBUG --- status code --- 22:53:02 DEBUG --- status code --- 22:53:02 DEBUG --- status code --- 22:53:02 DEBUG --- status code --- 22:53:02 DEBUG --- status code --- 22:53:02 DEBUG --- stderr --- 22:53:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:53:02 INFO [http_cmd]: http status code OK 22:53:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:53:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:53:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:53:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:53:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:53:02 INFO [http_cmd]: http status code OK 22:53:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:53:02 INFO [http_cmd]: http status code OK 22:53:02 INFO [http_cmd]: http status code OK 22:53:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:53:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:53:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:53:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:53:02 DEBUG --- status code --- 22:53:02 DEBUG --- status code --- 22:53:02 DEBUG --- status code --- 22:53:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:53:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:53:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:53:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:53:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:53:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:53:02 DEBUG 22:53:02 DEBUG --- http response --- 22:53:02 DEBUG --- status code --- 22:53:02 DEBUG --- http response --- 22:53:02 DEBUG --- http response --- 22:53:02 DEBUG --- http response --- 22:53:02 DEBUG --- http response --- 22:53:02 DEBUG --- http response --- 22:53:02 DEBUG --- status code --- 22:53:02 DEBUG --- http response --- 22:53:02 DEBUG --- status code --- 22:53:02 DEBUG --- status code --- 22:53:02 DEBUG --- http response --- 22:53:02 DEBUG --- http response --- 22:53:02 DEBUG --- http response --- 22:53:02 DEBUG --- http response --- 22:53:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:53:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:53:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:53:02 DEBUG --- http response --- 22:53:02 DEBUG --- http response --- 22:53:02 DEBUG --- http response --- 22:53:02 DEBUG --- http response --- 22:53:02 DEBUG --- http response --- 22:53:02 DEBUG --- http response --- 22:53:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:53:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:53:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:53:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:53:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:53:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:53:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:53:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:53:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:53:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:53:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:53:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:53:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:53:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:53:02 DEBUG --- http response --- 22:53:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:53:02 INFO [http_cmd]: http status code OK 22:53:02 DEBUG --- http response --- 22:53:02 DEBUG --- http response --- 22:53:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:53:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:53:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:53:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:53:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:53:02 DEBUG --- http response --- 22:53:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:53:02 DEBUG ----- output ----- 22:53:02 DEBUG ----- output ----- 22:53:02 DEBUG ----- output ----- 22:53:02 DEBUG ----- output ----- 22:53:02 DEBUG ----- output ----- 22:53:02 DEBUG ----- output ----- 22:53:02 DEBUG --- http response --- 22:53:02 DEBUG ----- output ----- 22:53:02 DEBUG --- http response --- 22:53:02 DEBUG --- http response --- 22:53:02 DEBUG ----- output ----- 22:53:02 DEBUG ----- output ----- 22:53:02 DEBUG ----- output ----- 22:53:02 DEBUG ----- output ----- 22:53:02 DEBUG --- status code --- 22:53:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:53:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:53:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:53:02 DEBUG ----- output ----- 22:53:02 DEBUG ----- output ----- 22:53:02 DEBUG ----- output ----- 22:53:02 DEBUG ----- output ----- 22:53:02 DEBUG ----- output ----- 22:53:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:53:02 DEBUG ----- output ----- 22:53:02 DEBUG ----- output ----- 22:53:02 DEBUG { 22:53:02 DEBUG { 22:53:02 DEBUG { 22:53:02 DEBUG { 22:53:02 DEBUG { 22:53:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:53:02 DEBUG { 22:53:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:53:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:53:02 DEBUG { 22:53:02 DEBUG { 22:53:02 DEBUG { 22:53:02 DEBUG { 22:53:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:53:02 DEBUG ----- output ----- 22:53:02 DEBUG ----- output ----- 22:53:02 DEBUG ----- output ----- 22:53:02 DEBUG { 22:53:02 DEBUG { 22:53:02 DEBUG { 22:53:02 DEBUG { 22:53:02 DEBUG { 22:53:02 DEBUG { 22:53:02 DEBUG { 22:53:02 DEBUG { 22:53:02 DEBUG "status": "success", 22:53:02 DEBUG "status": "success", 22:53:02 DEBUG "status": "success", 22:53:02 DEBUG "status": "success", 22:53:02 DEBUG "status": "success", 22:53:02 DEBUG ----- output ----- 22:53:02 DEBUG "status": "success", 22:53:02 DEBUG ----- output ----- 22:53:02 DEBUG ----- output ----- 22:53:02 DEBUG "status": "success", 22:53:02 DEBUG "status": "success", 22:53:02 DEBUG "status": "success", 22:53:02 DEBUG "status": "success", 22:53:02 DEBUG --- http response --- 22:53:02 DEBUG { 22:53:02 DEBUG { 22:53:02 DEBUG { 22:53:02 DEBUG "status": "success", 22:53:02 DEBUG "status": "success", 22:53:02 DEBUG "status": "success", 22:53:02 DEBUG "status": "success", 22:53:02 DEBUG "status": "success", 22:53:02 DEBUG "status": "success", 22:53:02 DEBUG "status": "success", 22:53:02 DEBUG "status": "success", 22:53:02 DEBUG "data": { 22:53:02 DEBUG "data": { 22:53:02 DEBUG "data": { 22:53:02 DEBUG "data": { 22:53:02 DEBUG "data": { 22:53:02 DEBUG { 22:53:02 DEBUG "data": { 22:53:02 DEBUG { 22:53:02 DEBUG { 22:53:02 DEBUG "data": { 22:53:02 DEBUG "data": { 22:53:02 DEBUG "data": { 22:53:02 DEBUG "data": { 22:53:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:53:02 DEBUG "status": "success", 22:53:02 DEBUG "status": "success", 22:53:02 DEBUG "status": "success", 22:53:02 DEBUG "data": { 22:53:02 DEBUG "data": { 22:53:02 DEBUG "data": { 22:53:02 DEBUG "data": { 22:53:02 DEBUG "data": { 22:53:02 DEBUG "data": { 22:53:02 DEBUG "data": { 22:53:02 DEBUG "data": { 22:53:02 DEBUG "resultType": "vector", 22:53:02 DEBUG "resultType": "vector", 22:53:02 DEBUG "resultType": "vector", 22:53:02 DEBUG "resultType": "vector", 22:53:02 DEBUG "resultType": "vector", 22:53:02 DEBUG "status": "success", 22:53:02 DEBUG "resultType": "vector", 22:53:02 DEBUG "status": "success", 22:53:02 DEBUG "status": "success", 22:53:02 DEBUG "resultType": "vector", 22:53:02 DEBUG "resultType": "vector", 22:53:02 DEBUG "resultType": "vector", 22:53:02 DEBUG "resultType": "vector", 22:53:02 DEBUG ----- output ----- 22:53:02 DEBUG "data": { 22:53:02 DEBUG "data": { 22:53:02 DEBUG "data": { 22:53:02 DEBUG "resultType": "vector", 22:53:02 DEBUG "resultType": "vector", 22:53:02 DEBUG "resultType": "vector", 22:53:02 DEBUG "result": [ 22:53:02 DEBUG "resultType": "vector", 22:53:02 DEBUG "resultType": "vector", 22:53:02 DEBUG "resultType": "vector", 22:53:02 DEBUG "resultType": "vector", 22:53:02 DEBUG "result": [ 22:53:02 DEBUG "result": [ 22:53:02 DEBUG "result": [ 22:53:02 DEBUG "result": [ 22:53:02 DEBUG "result": [ 22:53:02 DEBUG "data": { 22:53:02 DEBUG "result": [ 22:53:02 DEBUG "data": { 22:53:02 DEBUG "resultType": "vector", 22:53:02 DEBUG "result": [ 22:53:02 DEBUG "result": [ 22:53:02 DEBUG "result": [ 22:53:02 DEBUG "result": [ 22:53:02 DEBUG { 22:53:02 DEBUG "resultType": "vector", 22:53:02 DEBUG "resultType": "vector", 22:53:02 DEBUG "resultType": "vector", 22:53:02 DEBUG "result": [ 22:53:02 DEBUG "result": [ 22:53:02 DEBUG "resultType": "vector", 22:53:02 DEBUG { 22:53:02 DEBUG "result": [ 22:53:02 DEBUG "result": [ 22:53:02 DEBUG "result": [ 22:53:02 DEBUG "result": [ 22:53:02 DEBUG { 22:53:02 DEBUG { 22:53:02 DEBUG { 22:53:02 DEBUG { 22:53:02 DEBUG { 22:53:02 DEBUG "resultType": "vector", 22:53:02 DEBUG { 22:53:02 DEBUG "data": { 22:53:02 DEBUG "result": [ 22:53:02 DEBUG { 22:53:02 DEBUG { 22:53:02 DEBUG { 22:53:02 DEBUG { 22:53:02 DEBUG "status": "success", 22:53:02 DEBUG "result": [ 22:53:02 DEBUG "result": [ 22:53:02 DEBUG "result": [ 22:53:02 DEBUG { 22:53:02 DEBUG { 22:53:02 DEBUG "result": [ 22:53:02 DEBUG "metric": { 22:53:02 DEBUG { 22:53:02 DEBUG { 22:53:02 DEBUG { 22:53:02 DEBUG { 22:53:02 DEBUG "metric": { 22:53:02 DEBUG "metric": { 22:53:02 DEBUG "metric": { 22:53:02 DEBUG "metric": { 22:53:02 DEBUG "metric": { 22:53:02 DEBUG "result": [ 22:53:02 DEBUG "metric": { 22:53:02 DEBUG "resultType": "vector", 22:53:02 DEBUG { 22:53:02 DEBUG "metric": { 22:53:02 DEBUG "metric": { 22:53:02 DEBUG "metric": { 22:53:02 DEBUG "metric": { 22:53:02 DEBUG "data": { 22:53:02 DEBUG { 22:53:02 DEBUG { 22:53:02 DEBUG { 22:53:02 DEBUG "metric": { 22:53:02 DEBUG "metric": { 22:53:02 DEBUG { 22:53:02 DEBUG "pod": "am-555f4db595-fz8sn" 22:53:02 DEBUG "metric": { 22:53:02 DEBUG "metric": { 22:53:02 DEBUG "metric": { 22:53:02 DEBUG "metric": { 22:53:02 DEBUG "pod": "am-555f4db595-fz8sn" 22:53:02 DEBUG "pod": "ds-idrepo-2" 22:53:02 DEBUG "pod": "ds-idrepo-2" 22:53:02 DEBUG "instance": "10.142.0.107:9100" 22:53:02 DEBUG "pod": "am-555f4db595-fz8sn" 22:53:02 DEBUG { 22:53:02 DEBUG "pod": "am-555f4db595-fz8sn" 22:53:02 DEBUG "result": [ 22:53:02 DEBUG "metric": { 22:53:02 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:53:02 DEBUG "instance": "10.142.0.107:9100" 22:53:02 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:53:02 DEBUG "pod": "ds-idrepo-2" 22:53:02 DEBUG "resultType": "vector", 22:53:02 DEBUG "metric": { 22:53:02 DEBUG "metric": { 22:53:02 DEBUG "metric": { 22:53:02 DEBUG "pod": "ds-idrepo-2" 22:53:02 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:53:02 DEBUG "metric": { 22:53:02 DEBUG }, 22:53:02 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:53:02 DEBUG "pod": "am-555f4db595-fz8sn" 22:53:02 DEBUG "pod": "am-555f4db595-fz8sn" 22:53:02 DEBUG "pod": "ds-idrepo-2" 22:53:02 DEBUG }, 22:53:02 DEBUG }, 22:53:02 DEBUG }, 22:53:02 DEBUG }, 22:53:02 DEBUG }, 22:53:02 DEBUG "metric": { 22:53:02 DEBUG }, 22:53:02 DEBUG { 22:53:02 DEBUG "instance": "10.142.0.107:9100" 22:53:02 DEBUG }, 22:53:02 DEBUG }, 22:53:02 DEBUG }, 22:53:02 DEBUG }, 22:53:02 DEBUG "result": [ 22:53:02 DEBUG "pod": "ds-idrepo-2" 22:53:02 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:53:02 DEBUG "pod": "am-555f4db595-fz8sn" 22:53:02 DEBUG }, 22:53:02 DEBUG }, 22:53:02 DEBUG "pod": "ds-idrepo-2" 22:53:02 DEBUG "value": [ 22:53:02 DEBUG }, 22:53:02 DEBUG }, 22:53:02 DEBUG }, 22:53:02 DEBUG }, 22:53:02 DEBUG "value": [ 22:53:02 DEBUG "value": [ 22:53:02 DEBUG "value": [ 22:53:02 DEBUG "value": [ 22:53:02 DEBUG "value": [ 22:53:02 DEBUG "instance": "10.142.0.107:9100" 22:53:02 DEBUG "value": [ 22:53:02 DEBUG "metric": { 22:53:02 DEBUG }, 22:53:02 DEBUG "value": [ 22:53:02 DEBUG "value": [ 22:53:02 DEBUG "value": [ 22:53:02 DEBUG "value": [ 22:53:02 DEBUG { 22:53:02 DEBUG }, 22:53:02 DEBUG }, 22:53:02 DEBUG }, 22:53:02 DEBUG "value": [ 22:53:02 DEBUG "value": [ 22:53:02 DEBUG }, 22:53:02 DEBUG 22:53:02 DEBUG "value": [ 22:53:02 DEBUG "value": [ 22:53:02 DEBUG "value": [ 22:53:02 DEBUG "value": [ 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG }, 22:53:02 DEBUG 22:53:02 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:53:02 DEBUG "value": [ 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG "metric": { 22:53:02 DEBUG "value": [ 22:53:02 DEBUG "value": [ 22:53:02 DEBUG "value": [ 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG "value": [ 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG "value": [ 22:53:02 DEBUG 22:53:02 DEBUG }, 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG "instance": "10.142.0.107:9100" 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG "value": [ 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG }, 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG [...] 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG [...] 22:53:02 DEBUG [...] 22:53:02 DEBUG [...] 22:53:02 DEBUG [...] 22:53:02 DEBUG [...] 22:53:02 DEBUG 22:53:02 DEBUG [...] 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG [...] 22:53:02 DEBUG [...] 22:53:02 DEBUG [...] 22:53:02 DEBUG [...] 22:53:02 DEBUG "value": [ 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG [...] 22:53:02 DEBUG 22:53:02 DEBUG [...] 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG [...] 22:53:02 DEBUG [...] 22:53:02 DEBUG [...] 22:53:02 DEBUG [...] 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG [...] 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG [...] 22:53:02 DEBUG [...] 22:53:02 DEBUG 22:53:02 DEBUG [...] 22:53:02 DEBUG 22:53:02 DEBUG [...] 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG [...] 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG [...] 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG [...] 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG "pod": "am-555f4db595-kzxsp" 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG "pod": "am-555f4db595-kzxsp" 22:53:02 DEBUG "pod": "ds-idrepo-0" 22:53:02 DEBUG "pod": "ds-idrepo-0" 22:53:02 DEBUG "instance": "10.142.0.43:9100" 22:53:02 DEBUG "pod": "am-555f4db595-kzxsp" 22:53:02 DEBUG 22:53:02 DEBUG "pod": "am-555f4db595-kzxsp" 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG "pod": "idm-b85f9d978-j5pbm" 22:53:02 DEBUG "instance": "10.142.0.43:9100" 22:53:02 DEBUG "pod": "idm-b85f9d978-j5pbm" 22:53:02 DEBUG "pod": "ds-idrepo-0" 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG "pod": "idm-b85f9d978-j5pbm" 22:53:02 DEBUG 22:53:02 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:53:02 DEBUG 22:53:02 DEBUG }, 22:53:02 DEBUG "pod": "idm-b85f9d978-j5pbm" 22:53:02 DEBUG "pod": "am-555f4db595-kzxsp" 22:53:02 DEBUG "pod": "am-555f4db595-kzxsp" 22:53:02 DEBUG "pod": "ds-cts-2" 22:53:02 DEBUG }, 22:53:02 DEBUG }, 22:53:02 DEBUG }, 22:53:02 DEBUG }, 22:53:02 DEBUG }, 22:53:02 DEBUG 22:53:02 DEBUG }, 22:53:02 DEBUG 22:53:02 DEBUG "instance": "10.142.0.43:9100" 22:53:02 DEBUG }, 22:53:02 DEBUG }, 22:53:02 DEBUG }, 22:53:02 DEBUG }, 22:53:02 DEBUG 22:53:02 DEBUG 22:53:02 DEBUG "pod": "ds-idrepo-0" 22:53:02 DEBUG }, 22:53:02 DEBUG "pod": "am-555f4db595-kzxsp" 22:53:02 DEBUG }, 22:53:02 DEBUG "pod": "ds-idrepo-0" 22:53:02 DEBUG "value": [ 22:53:02 DEBUG }, 22:53:02 DEBUG }, 22:53:02 DEBUG }, 22:53:02 DEBUG }, 22:53:02 DEBUG "value": [ 22:53:02 DEBUG "value": [ 22:53:02 DEBUG "value": [ 22:53:02 DEBUG "value": [ 22:53:02 DEBUG "value": [ 22:53:02 DEBUG "instance": "10.142.0.43:9100" 22:53:02 DEBUG "value": [ 22:53:02 DEBUG 22:53:02 DEBUG }, 22:53:02 DEBUG "value": [ 22:53:02 DEBUG "value": [ 22:53:02 DEBUG "value": [ 22:53:02 DEBUG "value": [ 22:53:02 DEBUG 22:53:02 DEBUG "pod": "ds-idrepo-0" 22:53:02 DEBUG }, 22:53:02 DEBUG "value": [ 22:53:02 DEBUG }, 22:53:02 DEBUG "value": [ 22:53:02 DEBUG }, 22:53:02 DEBUG 1713304382, 22:53:02 DEBUG "value": [ 22:53:02 DEBUG "value": [ 22:53:02 DEBUG "value": [ 22:53:02 DEBUG "value": [ 22:53:02 DEBUG 1713304382, 22:53:02 DEBUG 1713304382, 22:53:02 DEBUG 1713304382, 22:53:02 DEBUG 1713304382, 22:53:02 DEBUG 1713304382, 22:53:02 DEBUG }, 22:53:02 DEBUG 1713304382, 22:53:02 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:53:02 DEBUG "value": [ 22:53:02 DEBUG 1713304382, 22:53:02 DEBUG 1713304382, 22:53:02 DEBUG 1713304382, 22:53:02 DEBUG 1713304382, 22:53:02 DEBUG "instance": "10.142.0.43:9100" 22:53:02 DEBUG }, 22:53:02 DEBUG "value": [ 22:53:02 DEBUG 1713304382, 22:53:02 DEBUG "value": [ 22:53:02 DEBUG 1713304382, 22:53:02 DEBUG 1713304382, 22:53:02 DEBUG "2026.9333333333334" 22:53:02 DEBUG 1713304382, 22:53:02 DEBUG 1713304382, 22:53:02 DEBUG 1713304382, 22:53:02 DEBUG 1713304382, 22:53:02 DEBUG "2169.633333333333" 22:53:02 DEBUG "0" 22:53:02 DEBUG "0" 22:53:02 DEBUG "0.00046666666666700016" 22:53:02 DEBUG "2169.866666666667" 22:53:02 DEBUG ] 22:53:02 DEBUG } 22:53:02 DEBUG "0" 22:53:02 DEBUG 1713304382, 22:53:02 DEBUG "3740.1985330937587" 22:53:02 DEBUG "19975158784" 22:53:02 DEBUG "482.60626233467855" 22:53:02 DEBUG "0" 22:53:02 DEBUG }, 22:53:02 DEBUG "value": [ 22:53:02 DEBUG 1713304382, 22:53:02 DEBUG "0.003001178237946136" 22:53:02 DEBUG "value": [ 22:53:02 DEBUG "3583394.5" 22:53:02 DEBUG "1685835776" 22:53:02 DEBUG ] 22:53:02 DEBUG "0" 22:53:02 DEBUG "0.027" 22:53:02 DEBUG "2172.9666666666667" 22:53:02 DEBUG "0" 22:53:02 DEBUG ] 22:53:02 DEBUG ] 22:53:02 DEBUG ] 22:53:02 DEBUG ] 22:53:02 DEBUG "value": [ 22:53:02 DEBUG }, 22:53:02 DEBUG ] 22:53:02 DEBUG ] 22:53:02 DEBUG "49327211.266666666" 22:53:02 DEBUG ] 22:53:02 DEBUG ] 22:53:02 DEBUG ] 22:53:02 DEBUG ] 22:53:02 DEBUG "value": [ 22:53:02 DEBUG 1713304382, 22:53:02 DEBUG "0" 22:53:02 DEBUG ] 22:53:02 DEBUG 1713304382, 22:53:02 DEBUG ] 22:53:02 DEBUG ] 22:53:02 DEBUG } 22:53:02 DEBUG ] 22:53:02 DEBUG ] 22:53:02 DEBUG ] 22:53:02 DEBUG ] 22:53:02 DEBUG } 22:53:02 DEBUG } 22:53:02 DEBUG } 22:53:02 DEBUG } 22:53:02 DEBUG 1713304382, 22:53:02 DEBUG "value": [ 22:53:02 DEBUG } 22:53:02 DEBUG } 22:53:02 DEBUG ] 22:53:02 DEBUG } 22:53:02 DEBUG } 22:53:02 DEBUG } 22:53:02 DEBUG } 22:53:02 DEBUG 1713304382, 22:53:02 DEBUG "0" 22:53:02 DEBUG ] 22:53:02 DEBUG } 22:53:02 DEBUG "0" 22:53:02 DEBUG } 22:53:02 DEBUG } 22:53:02 DEBUG ] 22:53:02 DEBUG } 22:53:02 DEBUG } 22:53:02 DEBUG } 22:53:02 DEBUG } 22:53:02 DEBUG ] 22:53:02 DEBUG ] 22:53:02 DEBUG ] 22:53:02 DEBUG ] 22:53:02 DEBUG "49334887.966666654" 22:53:02 DEBUG 1713304382, 22:53:02 DEBUG } 22:53:02 DEBUG ] 22:53:02 DEBUG } 22:53:02 DEBUG ] 22:53:02 DEBUG ] 22:53:02 DEBUG ] 22:53:02 DEBUG ] 22:53:02 DEBUG "0" 22:53:02 DEBUG ] 22:53:02 DEBUG } 22:53:02 DEBUG ] 22:53:02 DEBUG ] 22:53:02 DEBUG ] 22:53:02 DEBUG ] 22:53:02 DEBUG } 22:53:02 DEBUG ] 22:53:02 DEBUG ] 22:53:02 DEBUG ] 22:53:02 DEBUG ] 22:53:02 DEBUG } 22:53:02 DEBUG } 22:53:02 DEBUG } 22:53:02 DEBUG } 22:53:02 DEBUG ] 22:53:02 DEBUG "6.743063861550088" 22:53:02 DEBUG } 22:53:02 DEBUG ] 22:53:02 DEBUG } 22:53:02 DEBUG } 22:53:02 DEBUG } 22:53:02 DEBUG } 22:53:02 DEBUG ] 22:53:02 DEBUG } 22:53:02 DEBUG ] 22:53:02 DEBUG } 22:53:02 DEBUG } 22:53:02 DEBUG } 22:53:02 DEBUG } 22:53:02 DEBUG } 22:53:02 DEBUG } 22:53:02 DEBUG } 22:53:02 DEBUG } 22:53:02 DEBUG } 22:53:02 DEBUG } 22:53:02 DEBUG } 22:53:02 DEBUG } 22:53:02 DEBUG } 22:53:02 DEBUG } 22:53:02 DEBUG ] 22:53:02 DEBUG } 22:53:02 DEBUG } 22:53:02 DEBUG } 22:53:02 DEBUG } 22:53:02 DEBUG } 22:53:02 DEBUG } 22:53:02 DEBUG } 22:53:02 DEBUG ] 22:53:02 DEBUG } 22:53:02 DEBUG } 22:53:02 DEBUG ] 22:53:02 DEBUG } 22:53:02 DEBUG } 22:53:02 DEBUG } 22:53:02 DEBUG } 22:53:02 DEBUG } 22:53:02 DEBUG } 22:53:02 DEBUG ] 22:53:02 DEBUG } 22:53:02 DEBUG } 22:53:02 DEBUG ] 22:53:02 DEBUG } 22:53:02 DEBUG } 22:53:02 DEBUG } 22:53:02 DEBUG } 22:53:02 DEBUG ] 22:53:02 DEBUG } 22:53:02 DEBUG } 22:53:02 DEBUG } 22:53:02 DEBUG } 22:53:02 DEBUG } 22:53:02 DEBUG } 22:53:02 DEBUG } 22:53:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713304437" 22:53:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304437" 22:53:57 ERROR [http_cmd]: ERROR 22:53:57 INFO [http_cmd]: http status code OK 22:53:57 ERROR --- status code --- 22:53:57 DEBUG --- status code --- 22:53:57 ERROR http status code is 400 (expected [200, 401, 403, 404]) 22:53:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:53:57 ERROR --- http response --- 22:53:57 DEBUG --- http response --- 22:53:57 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 22:53:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 22:53:57 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 22:54:00 INFO 22:54:00 INFO [loop_until]: kubectl --namespace=xlou top pods 22:54:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:54:00 INFO [loop_until]: OK (rc = 0) 22:54:00 DEBUG --- stdout --- 22:54:00 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 9Mi am-555f4db595-4lbss 13449m 8680Mi am-555f4db595-fz8sn 13511m 8561Mi am-555f4db595-kzxsp 13148m 8965Mi ds-cts-0 19956m 12695Mi ds-cts-1 19073m 13024Mi ds-cts-2 19245m 12791Mi ds-idrepo-0 5945m 5030Mi ds-idrepo-1 5613m 2908Mi ds-idrepo-2 5626m 3219Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 4m 1208Mi idm-b85f9d978-rl76f 3m 1226Mi lodemon-bc756c647-cr7x6 2m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-7c9b667974-9mr2s 6717m 1516Mi 22:54:00 DEBUG --- stderr --- 22:54:00 DEBUG 22:54:02 INFO 22:54:02 INFO [loop_until]: kubectl --namespace=xlou top node 22:54:02 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:54:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304442" 22:54:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304442" 22:54:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304442" 22:54:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304442" 22:54:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304442" 22:54:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304442" 22:54:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304442" 22:54:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304442" 22:54:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713304442" 22:54:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304442" 22:54:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304442" 22:54:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304442" 22:54:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713304442" 22:54:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713304442" 22:54:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304442" 22:54:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304442" 22:54:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304442" 22:54:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304442" 22:54:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713304442" 22:54:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304442" 22:54:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304442" 22:54:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304442" 22:54:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304442" 22:54:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304442" 22:54:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304442" 22:54:02 INFO [loop_until]: OK (rc = 0) 22:54:02 INFO [http_cmd]: http status code OK 22:54:02 INFO [http_cmd]: http status code OK 22:54:02 INFO [http_cmd]: http status code OK 22:54:02 INFO [http_cmd]: http status code OK 22:54:02 INFO [http_cmd]: http status code OK 22:54:02 INFO [http_cmd]: http status code OK 22:54:02 INFO [http_cmd]: http status code OK 22:54:02 INFO [http_cmd]: http status code OK 22:54:02 INFO [http_cmd]: http status code OK 22:54:02 INFO [http_cmd]: http status code OK 22:54:02 INFO [http_cmd]: http status code OK 22:54:02 INFO [http_cmd]: http status code OK 22:54:02 INFO [http_cmd]: http status code OK 22:54:02 INFO [http_cmd]: http status code OK 22:54:02 INFO [http_cmd]: http status code OK 22:54:02 INFO [http_cmd]: http status code OK 22:54:02 DEBUG --- stdout --- 22:54:02 INFO [http_cmd]: http status code OK 22:54:02 INFO [http_cmd]: http status code OK 22:54:02 DEBUG --- status code --- 22:54:02 DEBUG --- status code --- 22:54:02 DEBUG --- status code --- 22:54:02 DEBUG --- status code --- 22:54:02 DEBUG --- status code --- 22:54:02 DEBUG --- status code --- 22:54:02 DEBUG --- status code --- 22:54:02 DEBUG --- status code --- 22:54:02 DEBUG --- status code --- 22:54:02 DEBUG --- status code --- 22:54:02 DEBUG --- status code --- 22:54:02 DEBUG --- status code --- 22:54:02 DEBUG --- status code --- 22:54:02 DEBUG --- status code --- 22:54:02 DEBUG --- status code --- 22:54:02 DEBUG --- status code --- 22:54:02 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 62m 0% 5076Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 19045m 63% 15642Mi 14% gke-xlou-cdm-default-pool-1b0769c8-nhlf 13398m 44% 10793Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 80m 0% 2641Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 5673m 19% 6566Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 20769m 69% 17115Mi 15% gke-xlou-cdm-default-pool-1b0769c8-xnmp 61m 0% 3159Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 13414m 44% 10469Mi 9% gke-xlou-cdm-ds-0f601751-hmm7 6160m 20% 5656Mi 5% gke-xlou-cdm-ds-0f601751-j488 19199m 64% 14891Mi 13% gke-xlou-cdm-ds-0f601751-mc5d 11768m 39% 3351Mi 3% gke-xlou-cdm-ds-0f601751-nj6q 13960m 46% 10835Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 6190m 20% 7347Mi 6% gke-xlou-cdm-frontend-a8771548-i82n 6984m 43% 6941Mi 11% 22:54:02 INFO [http_cmd]: http status code OK 22:54:02 DEBUG --- status code --- 22:54:02 INFO [http_cmd]: http status code OK 22:54:02 DEBUG --- status code --- 22:54:02 INFO [http_cmd]: http status code OK 22:54:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:54:02 INFO [http_cmd]: http status code OK 22:54:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:54:02 INFO [http_cmd]: http status code OK 22:54:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:54:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:54:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:54:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:54:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:54:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:54:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:54:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:54:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:54:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:54:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:54:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:54:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:54:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:54:02 DEBUG --- stderr --- 22:54:02 DEBUG --- status code --- 22:54:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:54:02 DEBUG --- status code --- 22:54:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:54:02 DEBUG --- status code --- 22:54:02 DEBUG --- http response --- 22:54:02 DEBUG --- status code --- 22:54:02 INFO [http_cmd]: http status code OK 22:54:02 DEBUG --- http response --- 22:54:02 INFO [http_cmd]: http status code OK 22:54:02 DEBUG --- status code --- 22:54:02 DEBUG --- http response --- 22:54:02 DEBUG --- http response --- 22:54:02 DEBUG --- http response --- 22:54:02 DEBUG --- http response --- 22:54:02 DEBUG --- http response --- 22:54:02 DEBUG --- http response --- 22:54:02 DEBUG --- http response --- 22:54:02 DEBUG --- http response --- 22:54:02 DEBUG --- http response --- 22:54:02 DEBUG --- http response --- 22:54:02 DEBUG --- http response --- 22:54:02 DEBUG --- http response --- 22:54:02 DEBUG --- http response --- 22:54:02 DEBUG --- http response --- 22:54:02 DEBUG 22:54:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:54:02 DEBUG --- http response --- 22:54:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:54:02 DEBUG --- http response --- 22:54:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:54:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:54:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:54:02 DEBUG --- status code --- 22:54:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:54:02 DEBUG --- status code --- 22:54:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:54:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:54:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:54:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:54:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:54:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:54:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:54:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:54:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:54:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:54:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:54:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:54:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:54:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:54:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:54:02 DEBUG --- http response --- 22:54:02 DEBUG --- http response --- 22:54:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:54:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:54:02 DEBUG --- http response --- 22:54:02 DEBUG ----- output ----- 22:54:02 DEBUG --- http response --- 22:54:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:54:02 DEBUG ----- output ----- 22:54:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:54:02 DEBUG --- http response --- 22:54:02 DEBUG ----- output ----- 22:54:02 DEBUG ----- output ----- 22:54:02 DEBUG ----- output ----- 22:54:02 DEBUG ----- output ----- 22:54:02 DEBUG ----- output ----- 22:54:02 DEBUG ----- output ----- 22:54:02 DEBUG ----- output ----- 22:54:02 DEBUG ----- output ----- 22:54:02 DEBUG ----- output ----- 22:54:02 DEBUG ----- output ----- 22:54:02 DEBUG ----- output ----- 22:54:02 DEBUG ----- output ----- 22:54:02 DEBUG ----- output ----- 22:54:02 DEBUG ----- output ----- 22:54:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:54:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:54:02 DEBUG ----- output ----- 22:54:02 DEBUG ----- output ----- 22:54:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:54:02 DEBUG { 22:54:02 DEBUG --- http response --- 22:54:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:54:02 DEBUG { 22:54:02 DEBUG --- http response --- 22:54:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:54:02 DEBUG { 22:54:02 DEBUG { 22:54:02 DEBUG { 22:54:02 DEBUG { 22:54:02 DEBUG { 22:54:02 DEBUG { 22:54:02 DEBUG { 22:54:02 DEBUG { 22:54:02 DEBUG { 22:54:02 DEBUG { 22:54:02 DEBUG { 22:54:02 DEBUG { 22:54:02 DEBUG { 22:54:02 DEBUG { 22:54:02 DEBUG ----- output ----- 22:54:02 DEBUG ----- output ----- 22:54:02 DEBUG { 22:54:02 DEBUG { 22:54:02 DEBUG ----- output ----- 22:54:02 DEBUG "status": "success", 22:54:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:54:02 DEBUG ----- output ----- 22:54:02 DEBUG "status": "success", 22:54:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:54:02 DEBUG ----- output ----- 22:54:02 DEBUG "status": "success", 22:54:02 DEBUG "status": "success", 22:54:02 DEBUG "status": "success", 22:54:02 DEBUG "status": "success", 22:54:02 DEBUG "status": "success", 22:54:02 DEBUG "status": "success", 22:54:02 DEBUG "status": "success", 22:54:02 DEBUG "status": "success", 22:54:02 DEBUG "status": "success", 22:54:02 DEBUG "status": "success", 22:54:02 DEBUG "status": "success", 22:54:02 DEBUG "status": "success", 22:54:02 DEBUG "status": "success", 22:54:02 DEBUG "status": "success", 22:54:02 DEBUG { 22:54:02 DEBUG { 22:54:02 DEBUG "status": "success", 22:54:02 DEBUG "status": "success", 22:54:02 DEBUG { 22:54:02 DEBUG "data": { 22:54:02 DEBUG ----- output ----- 22:54:02 DEBUG { 22:54:02 DEBUG "data": { 22:54:02 DEBUG ----- output ----- 22:54:02 DEBUG { 22:54:02 DEBUG "data": { 22:54:02 DEBUG "data": { 22:54:02 DEBUG "data": { 22:54:02 DEBUG "data": { 22:54:02 DEBUG "data": { 22:54:02 DEBUG "data": { 22:54:02 DEBUG "data": { 22:54:02 DEBUG "data": { 22:54:02 DEBUG "data": { 22:54:02 DEBUG "data": { 22:54:02 DEBUG "data": { 22:54:02 DEBUG "data": { 22:54:02 DEBUG "data": { 22:54:02 DEBUG "data": { 22:54:02 DEBUG "status": "success", 22:54:02 DEBUG "status": "success", 22:54:02 DEBUG "data": { 22:54:02 DEBUG "data": { 22:54:02 DEBUG "status": "success", 22:54:02 DEBUG "resultType": "vector", 22:54:02 DEBUG { 22:54:02 DEBUG "status": "success", 22:54:02 DEBUG "resultType": "vector", 22:54:02 DEBUG { 22:54:02 DEBUG "status": "success", 22:54:02 DEBUG "resultType": "vector", 22:54:02 DEBUG "resultType": "vector", 22:54:02 DEBUG "resultType": "vector", 22:54:02 DEBUG "resultType": "vector", 22:54:02 DEBUG "resultType": "vector", 22:54:02 DEBUG "resultType": "vector", 22:54:02 DEBUG "resultType": "vector", 22:54:02 DEBUG "resultType": "vector", 22:54:02 DEBUG "resultType": "vector", 22:54:02 DEBUG "resultType": "vector", 22:54:02 DEBUG "resultType": "vector", 22:54:02 DEBUG "resultType": "vector", 22:54:02 DEBUG "resultType": "vector", 22:54:02 DEBUG "resultType": "vector", 22:54:02 DEBUG "data": { 22:54:02 DEBUG "data": { 22:54:02 DEBUG "resultType": "vector", 22:54:02 DEBUG "resultType": "vector", 22:54:02 DEBUG "data": { 22:54:02 DEBUG "result": [ 22:54:02 DEBUG "status": "success", 22:54:02 DEBUG "data": { 22:54:02 DEBUG "result": [ 22:54:02 DEBUG "status": "success", 22:54:02 DEBUG "data": { 22:54:02 DEBUG "result": [ 22:54:02 DEBUG "result": [ 22:54:02 DEBUG "result": [ 22:54:02 DEBUG "result": [ 22:54:02 DEBUG "result": [ 22:54:02 DEBUG "result": [ 22:54:02 DEBUG "result": [ 22:54:02 DEBUG "result": [ 22:54:02 DEBUG "result": [ 22:54:02 DEBUG "result": [ 22:54:02 DEBUG "result": [ 22:54:02 DEBUG "result": [ 22:54:02 DEBUG "result": [ 22:54:02 DEBUG "result": [ 22:54:02 DEBUG "resultType": "vector", 22:54:02 DEBUG "resultType": "vector", 22:54:02 DEBUG "result": [ 22:54:02 DEBUG "result": [ 22:54:02 DEBUG "resultType": "vector", 22:54:02 DEBUG { 22:54:02 DEBUG "data": { 22:54:02 DEBUG "resultType": "vector", 22:54:02 DEBUG { 22:54:02 DEBUG "data": { 22:54:02 DEBUG "resultType": "vector", 22:54:02 DEBUG { 22:54:02 DEBUG { 22:54:02 DEBUG { 22:54:02 DEBUG { 22:54:02 DEBUG { 22:54:02 DEBUG { 22:54:02 DEBUG { 22:54:02 DEBUG { 22:54:02 DEBUG { 22:54:02 DEBUG { 22:54:02 DEBUG { 22:54:02 DEBUG { 22:54:02 DEBUG { 22:54:02 DEBUG { 22:54:02 DEBUG "result": [ 22:54:02 DEBUG "result": [ 22:54:02 DEBUG { 22:54:02 DEBUG { 22:54:02 DEBUG "result": [ 22:54:02 DEBUG "metric": { 22:54:02 DEBUG "resultType": "vector", 22:54:02 DEBUG "result": [ 22:54:02 DEBUG "metric": { 22:54:02 DEBUG "resultType": "vector", 22:54:02 DEBUG "result": [ 22:54:02 DEBUG "metric": { 22:54:02 DEBUG "metric": { 22:54:02 DEBUG "metric": { 22:54:02 DEBUG "pod": "am-555f4db595-fz8sn" 22:54:02 DEBUG "metric": { 22:54:02 DEBUG "metric": { 22:54:02 DEBUG "metric": { 22:54:02 DEBUG "metric": { 22:54:02 DEBUG "metric": { 22:54:02 DEBUG "metric": { 22:54:02 DEBUG "metric": { 22:54:02 DEBUG "metric": { 22:54:02 DEBUG "metric": { 22:54:02 DEBUG "metric": { 22:54:02 DEBUG { 22:54:02 DEBUG { 22:54:02 DEBUG "metric": { 22:54:02 DEBUG "metric": { 22:54:02 DEBUG { 22:54:02 DEBUG "pod": "am-555f4db595-fz8sn" 22:54:02 DEBUG "result": [ 22:54:02 DEBUG { 22:54:02 DEBUG "pod": "ds-idrepo-2" 22:54:02 DEBUG "result": [ 22:54:02 DEBUG { 22:54:02 DEBUG "instance": "10.142.0.107:9100" 22:54:02 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:54:02 DEBUG }, 22:54:02 DEBUG "metric": { 22:54:02 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:54:02 DEBUG "pod": "ds-idrepo-2" 22:54:02 DEBUG "pod": "ds-idrepo-2" 22:54:02 DEBUG "pod": "am-555f4db595-fz8sn" 22:54:02 DEBUG "pod": "am-555f4db595-fz8sn" 22:54:02 DEBUG "pod": "am-555f4db595-fz8sn" 22:54:02 DEBUG "pod": "ds-idrepo-2" 22:54:02 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:54:02 DEBUG "pod": "am-555f4db595-fz8sn" 22:54:02 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:54:02 DEBUG "metric": { 22:54:02 DEBUG "metric": { 22:54:02 DEBUG "pod": "ds-idrepo-2" 22:54:02 DEBUG "instance": "10.142.0.107:9100" 22:54:02 DEBUG "metric": { 22:54:02 DEBUG }, 22:54:02 DEBUG { 22:54:02 DEBUG "metric": { 22:54:02 DEBUG }, 22:54:02 DEBUG { 22:54:02 DEBUG "metric": { 22:54:02 DEBUG }, 22:54:02 DEBUG }, 22:54:02 DEBUG "value": [ 22:54:02 DEBUG "instance": "10.142.0.107:9100" 22:54:02 DEBUG }, 22:54:02 DEBUG }, 22:54:02 DEBUG }, 22:54:02 DEBUG }, 22:54:02 DEBUG }, 22:54:02 DEBUG }, 22:54:02 DEBUG }, 22:54:02 DEBUG }, 22:54:02 DEBUG }, 22:54:02 DEBUG }, 22:54:02 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:54:02 DEBUG "instance": "10.142.0.107:9100" 22:54:02 DEBUG }, 22:54:02 DEBUG }, 22:54:02 DEBUG "value": [ 22:54:02 DEBUG "value": [ 22:54:02 DEBUG "metric": { 22:54:02 DEBUG "instance": "10.142.0.107:9100" 22:54:02 DEBUG "value": [ 22:54:02 DEBUG "metric": { 22:54:02 DEBUG "pod": "ds-idrepo-2" 22:54:02 DEBUG "value": [ 22:54:02 DEBUG "value": [ 22:54:02 DEBUG 22:54:02 DEBUG }, 22:54:02 DEBUG "value": [ 22:54:02 DEBUG "value": [ 22:54:02 DEBUG "value": [ 22:54:02 DEBUG "value": [ 22:54:02 DEBUG "value": [ 22:54:02 DEBUG "value": [ 22:54:02 DEBUG "value": [ 22:54:02 DEBUG "value": [ 22:54:02 DEBUG "value": [ 22:54:02 DEBUG "value": [ 22:54:02 DEBUG }, 22:54:02 DEBUG }, 22:54:02 DEBUG "value": [ 22:54:02 DEBUG "pod": "am-555f4db595-fz8sn" 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG }, 22:54:02 DEBUG 22:54:02 DEBUG "pod": "ds-idrepo-2" 22:54:02 DEBUG }, 22:54:02 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG "value": [ 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG "value": [ 22:54:02 DEBUG "value": [ 22:54:02 DEBUG 22:54:02 DEBUG }, 22:54:02 DEBUG "value": [ 22:54:02 DEBUG 22:54:02 DEBUG "value": [ 22:54:02 DEBUG 22:54:02 DEBUG }, 22:54:02 DEBUG "value": [ 22:54:02 DEBUG }, 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG "value": [ 22:54:02 DEBUG 22:54:02 DEBUG "value": [ 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG [...] 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG [...] 22:54:02 DEBUG 22:54:02 DEBUG [...] 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG [...] 22:54:02 DEBUG [...] 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG [...] 22:54:02 DEBUG [...] 22:54:02 DEBUG [...] 22:54:02 DEBUG [...] 22:54:02 DEBUG [...] 22:54:02 DEBUG [...] 22:54:02 DEBUG [...] 22:54:02 DEBUG [...] 22:54:02 DEBUG [...] 22:54:02 DEBUG [...] 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG [...] 22:54:02 DEBUG [...] 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG [...] 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG [...] 22:54:02 DEBUG [...] 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG [...] 22:54:02 DEBUG 22:54:02 DEBUG [...] 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG [...] 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG [...] 22:54:02 DEBUG 22:54:02 DEBUG [...] 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG "pod": "am-555f4db595-kzxsp" 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG "pod": "am-555f4db595-kzxsp" 22:54:02 DEBUG 22:54:02 DEBUG "pod": "ds-idrepo-0" 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG "instance": "10.142.0.43:9100" 22:54:02 DEBUG "pod": "ds-cts-2" 22:54:02 DEBUG }, 22:54:02 DEBUG 22:54:02 DEBUG "instance": "10.142.0.43:9100" 22:54:02 DEBUG "pod": "ds-idrepo-0" 22:54:02 DEBUG "pod": "ds-idrepo-0" 22:54:02 DEBUG "pod": "am-555f4db595-kzxsp" 22:54:02 DEBUG "pod": "am-555f4db595-kzxsp" 22:54:02 DEBUG "pod": "am-555f4db595-kzxsp" 22:54:02 DEBUG "pod": "ds-cts-2" 22:54:02 DEBUG "pod": "overseer-0-7c9b667974-9mr2s" 22:54:02 DEBUG "pod": "am-555f4db595-kzxsp" 22:54:02 DEBUG "pod": "ds-cts-2" 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG "pod": "ds-idrepo-0" 22:54:02 DEBUG "instance": "10.142.0.43:9100" 22:54:02 DEBUG 22:54:02 DEBUG }, 22:54:02 DEBUG 22:54:02 DEBUG }, 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG 22:54:02 DEBUG }, 22:54:02 DEBUG }, 22:54:02 DEBUG "value": [ 22:54:02 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:54:02 DEBUG }, 22:54:02 DEBUG }, 22:54:02 DEBUG }, 22:54:02 DEBUG }, 22:54:02 DEBUG }, 22:54:02 DEBUG }, 22:54:02 DEBUG }, 22:54:02 DEBUG }, 22:54:02 DEBUG }, 22:54:02 DEBUG }, 22:54:02 DEBUG "pod": "overseer-0-7c9b667974-9mr2s" 22:54:02 DEBUG "instance": "10.142.0.43:9100" 22:54:02 DEBUG }, 22:54:02 DEBUG }, 22:54:02 DEBUG "pod": "am-555f4db595-kzxsp" 22:54:02 DEBUG "value": [ 22:54:02 DEBUG "instance": "10.142.0.43:9100" 22:54:02 DEBUG "value": [ 22:54:02 DEBUG 22:54:02 DEBUG "pod": "ds-idrepo-0" 22:54:02 DEBUG 22:54:02 DEBUG "value": [ 22:54:02 DEBUG "value": [ 22:54:02 DEBUG 1713304442, 22:54:02 DEBUG }, 22:54:02 DEBUG "value": [ 22:54:02 DEBUG "value": [ 22:54:02 DEBUG "value": [ 22:54:02 DEBUG "value": [ 22:54:02 DEBUG "value": [ 22:54:02 DEBUG "value": [ 22:54:02 DEBUG "value": [ 22:54:02 DEBUG "value": [ 22:54:02 DEBUG "value": [ 22:54:02 DEBUG "value": [ 22:54:02 DEBUG }, 22:54:02 DEBUG }, 22:54:02 DEBUG "value": [ 22:54:02 DEBUG "value": [ 22:54:02 DEBUG }, 22:54:02 DEBUG 1713304442, 22:54:02 DEBUG }, 22:54:02 DEBUG 1713304442, 22:54:02 DEBUG "pod": "ds-idrepo-0" 22:54:02 DEBUG }, 22:54:02 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:54:02 DEBUG 1713304442, 22:54:02 DEBUG 1713304442, 22:54:02 DEBUG "2173" 22:54:02 DEBUG "value": [ 22:54:02 DEBUG 1713304442, 22:54:02 DEBUG 1713304442, 22:54:02 DEBUG 1713304442, 22:54:02 DEBUG 1713304442, 22:54:02 DEBUG 1713304442, 22:54:02 DEBUG 1713304442, 22:54:02 DEBUG 1713304442, 22:54:02 DEBUG 1713304442, 22:54:02 DEBUG 1713304442, 22:54:02 DEBUG 1713304442, 22:54:02 DEBUG "value": [ 22:54:02 DEBUG "value": [ 22:54:02 DEBUG 1713304442, 22:54:02 DEBUG 1713304442, 22:54:02 DEBUG "value": [ 22:54:02 DEBUG "2003.5" 22:54:02 DEBUG "value": [ 22:54:02 DEBUG "0" 22:54:02 DEBUG }, 22:54:02 DEBUG "value": [ 22:54:02 DEBUG }, 22:54:02 DEBUG "19976078336" 22:54:02 DEBUG "0" 22:54:02 DEBUG ] 22:54:02 DEBUG 1713304442, 22:54:02 DEBUG "0.0004666355576288775" 22:54:02 DEBUG "0" 22:54:02 DEBUG "0" 22:54:02 DEBUG "2173.3" 22:54:02 DEBUG "0" 22:54:02 DEBUG "2175.133333333333" 22:54:02 DEBUG "0" 22:54:02 DEBUG "0" 22:54:02 DEBUG "0.027" 22:54:02 DEBUG "0" 22:54:02 DEBUG 1713304442, 22:54:02 DEBUG 1713304442, 22:54:02 DEBUG "0" 22:54:02 DEBUG "49366575.52829812" 22:54:02 DEBUG 1713304442, 22:54:02 DEBUG ] 22:54:02 DEBUG 1713304442, 22:54:02 DEBUG ] 22:54:02 DEBUG "value": [ 22:54:02 DEBUG 1713304442, 22:54:02 DEBUG "value": [ 22:54:02 DEBUG ] 22:54:02 DEBUG ] 22:54:02 DEBUG } 22:54:02 DEBUG "1688088576" 22:54:02 DEBUG ] 22:54:02 DEBUG ] 22:54:02 DEBUG ] 22:54:02 DEBUG ] 22:54:02 DEBUG ] 22:54:02 DEBUG ] 22:54:02 DEBUG ] 22:54:02 DEBUG ] 22:54:02 DEBUG ] 22:54:02 DEBUG ] 22:54:02 DEBUG "0" 22:54:02 DEBUG "49376325.27831477" 22:54:02 DEBUG ] 22:54:02 DEBUG ] 22:54:02 DEBUG "3593803" 22:54:02 DEBUG } 22:54:02 DEBUG "0.0003333111125924941" 22:54:02 DEBUG } 22:54:02 DEBUG 1713304442, 22:54:02 DEBUG "0" 22:54:02 DEBUG 1713304442, 22:54:02 DEBUG } 22:54:02 DEBUG } 22:54:02 DEBUG ] 22:54:02 DEBUG ] 22:54:02 DEBUG } 22:54:02 DEBUG } 22:54:02 DEBUG } 22:54:02 DEBUG } 22:54:02 DEBUG } 22:54:02 DEBUG } 22:54:02 DEBUG } 22:54:02 DEBUG } 22:54:02 DEBUG } 22:54:02 DEBUG } 22:54:02 DEBUG ] 22:54:02 DEBUG ] 22:54:02 DEBUG } 22:54:02 DEBUG } 22:54:02 DEBUG ] 22:54:02 DEBUG ] 22:54:02 DEBUG ] 22:54:02 DEBUG ] 22:54:02 DEBUG "0" 22:54:02 DEBUG ] 22:54:02 DEBUG "6.8551172236364275" 22:54:02 DEBUG ] 22:54:02 DEBUG ] 22:54:02 DEBUG } 22:54:02 DEBUG } 22:54:02 DEBUG ] 22:54:02 DEBUG ] 22:54:02 DEBUG ] 22:54:02 DEBUG ] 22:54:02 DEBUG ] 22:54:02 DEBUG ] 22:54:02 DEBUG ] 22:54:02 DEBUG ] 22:54:02 DEBUG } 22:54:02 DEBUG ] 22:54:02 DEBUG } 22:54:02 DEBUG } 22:54:02 DEBUG ] 22:54:02 DEBUG ] 22:54:02 DEBUG } 22:54:02 DEBUG } 22:54:02 DEBUG } 22:54:02 DEBUG } 22:54:02 DEBUG ] 22:54:02 DEBUG } 22:54:02 DEBUG ] 22:54:02 DEBUG } 22:54:02 DEBUG } 22:54:02 DEBUG } 22:54:02 DEBUG ] 22:54:02 DEBUG } 22:54:02 DEBUG } 22:54:02 DEBUG } 22:54:02 DEBUG } 22:54:02 DEBUG } 22:54:02 DEBUG } 22:54:02 DEBUG } 22:54:02 DEBUG } 22:54:02 DEBUG ] 22:54:02 DEBUG } 22:54:02 DEBUG ] 22:54:02 DEBUG ] 22:54:02 DEBUG } 22:54:02 DEBUG } 22:54:02 DEBUG ] 22:54:02 DEBUG } 22:54:02 DEBUG ] 22:54:02 DEBUG } 22:54:02 DEBUG } 22:54:02 DEBUG ] 22:54:02 DEBUG } 22:54:02 DEBUG } 22:54:02 DEBUG } 22:54:02 DEBUG } 22:54:02 DEBUG } 22:54:02 DEBUG } 22:54:02 DEBUG } 22:54:02 DEBUG } 22:54:02 DEBUG } 22:54:02 DEBUG } 22:54:02 DEBUG } 22:54:02 DEBUG } 22:54:02 DEBUG } 22:54:02 DEBUG } 22:54:02 DEBUG } 22:54:02 DEBUG } 22:54:02 DEBUG } 22:54:02 DEBUG } 22:54:02 DEBUG } 22:54:02 DEBUG } 22:54:02 DEBUG ] 22:54:02 DEBUG } 22:54:02 DEBUG ] 22:54:02 DEBUG } 22:54:02 DEBUG } 22:54:02 DEBUG } 22:54:02 DEBUG } 22:54:02 DEBUG } 22:54:02 DEBUG } 22:54:02 DEBUG } 22:54:02 DEBUG } 22:54:02 DEBUG } 22:54:02 DEBUG } 22:54:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304497" 22:54:57 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713304497" 22:54:57 ERROR [http_cmd]: ERROR 22:54:57 INFO [http_cmd]: http status code OK 22:54:57 ERROR --- status code --- 22:54:57 DEBUG --- status code --- 22:54:57 ERROR http status code is 400 (expected [200, 401, 403, 404]) 22:54:57 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:54:57 ERROR --- http response --- 22:54:57 DEBUG --- http response --- 22:54:57 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 22:54:57 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 22:54:57 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 22:55:00 INFO 22:55:00 INFO [loop_until]: kubectl --namespace=xlou top pods 22:55:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:55:00 INFO [loop_until]: OK (rc = 0) 22:55:00 DEBUG --- stdout --- 22:55:00 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 9Mi am-555f4db595-4lbss 13462m 8677Mi am-555f4db595-fz8sn 13423m 8557Mi am-555f4db595-kzxsp 13268m 8966Mi ds-cts-0 19142m 12991Mi ds-cts-1 19239m 12715Mi ds-cts-2 18329m 12728Mi ds-idrepo-0 5915m 5030Mi ds-idrepo-1 5438m 2907Mi ds-idrepo-2 5498m 3220Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 4m 1207Mi idm-b85f9d978-rl76f 3m 1226Mi lodemon-bc756c647-cr7x6 2m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-7c9b667974-9mr2s 6868m 1518Mi 22:55:00 DEBUG --- stderr --- 22:55:00 DEBUG 22:55:02 INFO 22:55:02 INFO [loop_until]: kubectl --namespace=xlou top node 22:55:02 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:55:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304502" 22:55:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304502" 22:55:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304502" 22:55:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304502" 22:55:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304502" 22:55:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304502" 22:55:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304502" 22:55:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713304502" 22:55:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713304502" 22:55:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304502" 22:55:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304502" 22:55:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304502" 22:55:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304502" 22:55:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304502" 22:55:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304502" 22:55:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304502" 22:55:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304502" 22:55:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304502" 22:55:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304502" 22:55:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713304502" 22:55:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713304502" 22:55:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304502" 22:55:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304502" 22:55:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304502" 22:55:02 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304502" 22:55:02 INFO [loop_until]: OK (rc = 0) 22:55:02 INFO [http_cmd]: http status code OK 22:55:02 INFO [http_cmd]: http status code OK 22:55:02 INFO [http_cmd]: http status code OK 22:55:02 INFO [http_cmd]: http status code OK 22:55:02 INFO [http_cmd]: http status code OK 22:55:02 INFO [http_cmd]: http status code OK 22:55:02 INFO [http_cmd]: http status code OK 22:55:02 INFO [http_cmd]: http status code OK 22:55:02 INFO [http_cmd]: http status code OK 22:55:02 INFO [http_cmd]: http status code OK 22:55:02 INFO [http_cmd]: http status code OK 22:55:02 DEBUG --- status code --- 22:55:02 INFO [http_cmd]: http status code OK 22:55:02 INFO [http_cmd]: http status code OK 22:55:02 INFO [http_cmd]: http status code OK 22:55:02 INFO [http_cmd]: http status code OK 22:55:02 DEBUG --- stdout --- 22:55:02 INFO [http_cmd]: http status code OK 22:55:02 DEBUG --- status code --- 22:55:02 INFO [http_cmd]: http status code OK 22:55:02 INFO [http_cmd]: http status code OK 22:55:02 DEBUG --- status code --- 22:55:02 DEBUG --- status code --- 22:55:02 DEBUG --- status code --- 22:55:02 DEBUG --- status code --- 22:55:02 DEBUG --- status code --- 22:55:02 DEBUG --- status code --- 22:55:02 DEBUG --- status code --- 22:55:02 DEBUG --- status code --- 22:55:02 INFO [http_cmd]: http status code OK 22:55:02 DEBUG --- status code --- 22:55:02 INFO [http_cmd]: http status code OK 22:55:02 INFO [http_cmd]: http status code OK 22:55:02 INFO [http_cmd]: http status code OK 22:55:02 INFO [http_cmd]: http status code OK 22:55:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:55:02 DEBUG --- status code --- 22:55:02 INFO [http_cmd]: http status code OK 22:55:02 DEBUG --- status code --- 22:55:02 DEBUG --- status code --- 22:55:02 DEBUG --- status code --- 22:55:02 DEBUG --- status code --- 22:55:02 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 60m 0% 5085Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 18871m 63% 15279Mi 13% gke-xlou-cdm-default-pool-1b0769c8-nhlf 13234m 44% 10797Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 82m 0% 2649Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 5604m 18% 6561Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 20809m 69% 17481Mi 15% gke-xlou-cdm-default-pool-1b0769c8-xnmp 58m 0% 3156Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 13389m 44% 10464Mi 9% gke-xlou-cdm-ds-0f601751-hmm7 6110m 20% 5656Mi 5% gke-xlou-cdm-ds-0f601751-j488 18355m 61% 14666Mi 13% gke-xlou-cdm-ds-0f601751-mc5d 11612m 38% 3332Mi 2% gke-xlou-cdm-ds-0f601751-nj6q 14063m 47% 10838Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 6077m 20% 7349Mi 6% gke-xlou-cdm-frontend-a8771548-i82n 6929m 43% 6938Mi 11% 22:55:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:55:02 DEBUG --- status code --- 22:55:02 DEBUG --- status code --- 22:55:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:55:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:55:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:55:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:55:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:55:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:55:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:55:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:55:02 DEBUG --- status code --- 22:55:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:55:02 DEBUG --- status code --- 22:55:02 DEBUG --- status code --- 22:55:02 DEBUG --- status code --- 22:55:02 DEBUG --- status code --- 22:55:02 DEBUG --- http response --- 22:55:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:55:02 DEBUG --- status code --- 22:55:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:55:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:55:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:55:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:55:02 DEBUG --- stderr --- 22:55:02 DEBUG --- http response --- 22:55:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:55:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:55:02 DEBUG --- http response --- 22:55:02 DEBUG --- http response --- 22:55:02 DEBUG --- http response --- 22:55:02 DEBUG --- http response --- 22:55:02 DEBUG --- http response --- 22:55:02 DEBUG --- http response --- 22:55:02 DEBUG --- http response --- 22:55:02 DEBUG --- http response --- 22:55:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:55:02 DEBUG --- http response --- 22:55:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:55:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:55:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:55:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:55:02 INFO [http_cmd]: http status code OK 22:55:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:55:02 DEBUG --- http response --- 22:55:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:55:02 DEBUG --- http response --- 22:55:02 DEBUG --- http response --- 22:55:02 DEBUG --- http response --- 22:55:02 DEBUG --- http response --- 22:55:02 DEBUG 22:55:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:55:02 DEBUG --- http response --- 22:55:02 DEBUG --- http response --- 22:55:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:55:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:55:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:55:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:55:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:55:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:55:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:55:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:55:02 DEBUG --- http response --- 22:55:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:55:02 DEBUG --- http response --- 22:55:02 DEBUG --- http response --- 22:55:02 DEBUG --- http response --- 22:55:02 DEBUG --- http response --- 22:55:02 DEBUG --- status code --- 22:55:02 DEBUG ----- output ----- 22:55:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:55:02 DEBUG --- http response --- 22:55:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:55:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:55:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:55:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:55:02 DEBUG ----- output ----- 22:55:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:55:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:55:02 DEBUG ----- output ----- 22:55:02 DEBUG ----- output ----- 22:55:02 DEBUG ----- output ----- 22:55:02 DEBUG ----- output ----- 22:55:02 DEBUG ----- output ----- 22:55:02 DEBUG ----- output ----- 22:55:02 DEBUG ----- output ----- 22:55:02 DEBUG ----- output ----- 22:55:02 DEBUG ----- output ----- 22:55:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:55:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:55:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:55:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:55:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:55:02 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:55:02 DEBUG { 22:55:02 DEBUG ----- output ----- 22:55:02 DEBUG ----- output ----- 22:55:02 DEBUG ----- output ----- 22:55:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:55:02 DEBUG ----- output ----- 22:55:02 DEBUG ----- output ----- 22:55:02 DEBUG { 22:55:02 DEBUG ----- output ----- 22:55:02 DEBUG ----- output ----- 22:55:02 DEBUG { 22:55:02 DEBUG { 22:55:02 DEBUG { 22:55:02 DEBUG { 22:55:02 DEBUG { 22:55:02 DEBUG { 22:55:02 DEBUG { 22:55:02 DEBUG { 22:55:02 DEBUG { 22:55:02 DEBUG ----- output ----- 22:55:02 DEBUG ----- output ----- 22:55:02 DEBUG ----- output ----- 22:55:02 DEBUG ----- output ----- 22:55:02 DEBUG ----- output ----- 22:55:02 DEBUG --- http response --- 22:55:02 DEBUG "status": "success", 22:55:02 DEBUG { 22:55:02 DEBUG { 22:55:02 DEBUG { 22:55:02 DEBUG ----- output ----- 22:55:02 DEBUG { 22:55:02 DEBUG { 22:55:02 DEBUG "status": "success", 22:55:02 DEBUG { 22:55:02 DEBUG { 22:55:02 DEBUG "status": "success", 22:55:02 DEBUG "status": "success", 22:55:02 DEBUG "status": "success", 22:55:02 DEBUG "status": "success", 22:55:02 DEBUG "status": "success", 22:55:02 DEBUG "status": "success", 22:55:02 DEBUG "status": "success", 22:55:02 DEBUG "status": "success", 22:55:02 DEBUG "status": "success", 22:55:02 DEBUG { 22:55:02 DEBUG { 22:55:02 DEBUG { 22:55:02 DEBUG { 22:55:02 DEBUG { 22:55:02 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:55:02 DEBUG "data": { 22:55:02 DEBUG "status": "success", 22:55:02 DEBUG "status": "success", 22:55:02 DEBUG "status": "success", 22:55:02 DEBUG { 22:55:02 DEBUG "status": "success", 22:55:02 DEBUG "status": "success", 22:55:02 DEBUG "data": { 22:55:02 DEBUG "status": "success", 22:55:02 DEBUG "status": "success", 22:55:02 DEBUG "data": { 22:55:02 DEBUG "data": { 22:55:02 DEBUG "data": { 22:55:02 DEBUG "data": { 22:55:02 DEBUG "data": { 22:55:02 DEBUG "data": { 22:55:02 DEBUG "data": { 22:55:02 DEBUG "data": { 22:55:02 DEBUG "data": { 22:55:02 DEBUG "status": "success", 22:55:02 DEBUG "status": "success", 22:55:02 DEBUG "status": "success", 22:55:02 DEBUG "status": "success", 22:55:02 DEBUG "status": "success", 22:55:02 DEBUG ----- output ----- 22:55:02 DEBUG "resultType": "vector", 22:55:02 DEBUG "data": { 22:55:02 DEBUG "data": { 22:55:02 DEBUG "data": { 22:55:02 DEBUG "status": "success", 22:55:02 DEBUG "data": { 22:55:02 DEBUG "data": { 22:55:02 DEBUG "resultType": "vector", 22:55:02 DEBUG "data": { 22:55:02 DEBUG "data": { 22:55:02 DEBUG "resultType": "vector", 22:55:02 DEBUG "resultType": "vector", 22:55:02 DEBUG "resultType": "vector", 22:55:02 DEBUG "resultType": "vector", 22:55:02 DEBUG "resultType": "vector", 22:55:02 DEBUG "resultType": "vector", 22:55:02 DEBUG "resultType": "vector", 22:55:02 DEBUG "resultType": "vector", 22:55:02 DEBUG "resultType": "vector", 22:55:02 DEBUG "data": { 22:55:02 DEBUG "data": { 22:55:02 DEBUG "data": { 22:55:02 DEBUG "resultType": "vector", 22:55:02 DEBUG "data": { 22:55:02 DEBUG { 22:55:02 DEBUG "result": [ 22:55:02 DEBUG "resultType": "vector", 22:55:02 DEBUG "resultType": "vector", 22:55:02 DEBUG "resultType": "vector", 22:55:02 DEBUG "data": { 22:55:02 DEBUG "resultType": "vector", 22:55:02 DEBUG "resultType": "vector", 22:55:02 DEBUG "result": [ 22:55:02 DEBUG "resultType": "vector", 22:55:02 DEBUG "resultType": "vector", 22:55:02 DEBUG "result": [ 22:55:02 DEBUG "result": [ 22:55:02 DEBUG "result": [ 22:55:02 DEBUG "result": [ 22:55:02 DEBUG "result": [ 22:55:02 DEBUG "result": [ 22:55:02 DEBUG "result": [ 22:55:02 DEBUG "result": [ 22:55:02 DEBUG "result": [ 22:55:02 DEBUG "resultType": "vector", 22:55:02 DEBUG "resultType": "vector", 22:55:02 DEBUG "data": { 22:55:02 DEBUG "result": [ 22:55:02 DEBUG "resultType": "vector", 22:55:02 DEBUG "result": [ 22:55:02 DEBUG { 22:55:02 DEBUG "result": [ 22:55:02 DEBUG "result": [ 22:55:02 DEBUG "result": [ 22:55:02 DEBUG "resultType": "vector", 22:55:02 DEBUG "result": [ 22:55:02 DEBUG "result": [ 22:55:02 DEBUG { 22:55:02 DEBUG "result": [ 22:55:02 DEBUG "result": [ 22:55:02 DEBUG { 22:55:02 DEBUG { 22:55:02 DEBUG { 22:55:02 DEBUG { 22:55:02 DEBUG { 22:55:02 DEBUG { 22:55:02 DEBUG { 22:55:02 DEBUG { 22:55:02 DEBUG { 22:55:02 DEBUG "result": [ 22:55:02 DEBUG "result": [ 22:55:02 DEBUG "resultType": "vector", 22:55:02 DEBUG { 22:55:02 DEBUG { 22:55:02 DEBUG "status": "success", 22:55:02 DEBUG "metric": { 22:55:02 DEBUG { 22:55:02 DEBUG { 22:55:02 DEBUG { 22:55:02 DEBUG "result": [ 22:55:02 DEBUG { 22:55:02 DEBUG { 22:55:02 DEBUG "metric": { 22:55:02 DEBUG { 22:55:02 DEBUG { 22:55:02 DEBUG "metric": { 22:55:02 DEBUG "metric": { 22:55:02 DEBUG "metric": { 22:55:02 DEBUG "metric": { 22:55:02 DEBUG "metric": { 22:55:02 DEBUG "metric": { 22:55:02 DEBUG "metric": { 22:55:02 DEBUG "metric": { 22:55:02 DEBUG "metric": { 22:55:02 DEBUG { 22:55:02 DEBUG { 22:55:02 DEBUG "result": [ 22:55:02 DEBUG "metric": { 22:55:02 DEBUG "metric": { 22:55:02 DEBUG "data": { 22:55:02 DEBUG "pod": "ds-idrepo-2" 22:55:02 DEBUG "metric": { 22:55:02 DEBUG "metric": { 22:55:02 DEBUG "metric": { 22:55:02 DEBUG { 22:55:02 DEBUG "metric": { 22:55:02 DEBUG "metric": { 22:55:02 DEBUG "pod": "ds-idrepo-2" 22:55:02 DEBUG "metric": { 22:55:02 DEBUG "metric": { 22:55:02 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:55:02 DEBUG "instance": "10.142.0.107:9100" 22:55:02 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:55:02 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:55:02 DEBUG "pod": "am-555f4db595-fz8sn" 22:55:02 DEBUG "pod": "ds-idrepo-2" 22:55:02 DEBUG "pod": "am-555f4db595-fz8sn" 22:55:02 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:55:02 DEBUG "pod": "am-555f4db595-fz8sn" 22:55:02 DEBUG "metric": { 22:55:02 DEBUG "metric": { 22:55:02 DEBUG { 22:55:02 DEBUG "pod": "ds-idrepo-2" 22:55:02 DEBUG "instance": "10.142.0.107:9100" 22:55:02 DEBUG "resultType": "vector", 22:55:02 DEBUG }, 22:55:02 DEBUG "instance": "10.142.0.107:9100" 22:55:02 DEBUG "instance": "10.142.0.107:9100" 22:55:02 DEBUG "pod": "ds-idrepo-2" 22:55:02 DEBUG "metric": { 22:55:02 DEBUG "pod": "am-555f4db595-fz8sn" 22:55:02 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:55:02 DEBUG }, 22:55:02 DEBUG "instance": "10.142.0.107:9100" 22:55:02 DEBUG "pod": "ds-idrepo-2" 22:55:02 DEBUG }, 22:55:02 DEBUG }, 22:55:02 DEBUG }, 22:55:02 DEBUG }, 22:55:02 DEBUG }, 22:55:02 DEBUG }, 22:55:02 DEBUG }, 22:55:02 DEBUG }, 22:55:02 DEBUG }, 22:55:02 DEBUG "pod": "am-555f4db595-fz8sn" 22:55:02 DEBUG "pod": "am-555f4db595-fz8sn" 22:55:02 DEBUG "metric": { 22:55:02 DEBUG }, 22:55:02 DEBUG }, 22:55:02 DEBUG "result": [ 22:55:02 DEBUG "value": [ 22:55:02 DEBUG 22:55:02 DEBUG }, 22:55:02 DEBUG }, 22:55:02 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:55:02 DEBUG }, 22:55:02 DEBUG }, 22:55:02 DEBUG "value": [ 22:55:02 DEBUG }, 22:55:02 DEBUG }, 22:55:02 DEBUG "value": [ 22:55:02 DEBUG "value": [ 22:55:02 DEBUG "value": [ 22:55:02 DEBUG "value": [ 22:55:02 DEBUG "value": [ 22:55:02 DEBUG "value": [ 22:55:02 DEBUG "value": [ 22:55:02 DEBUG "value": [ 22:55:02 DEBUG "value": [ 22:55:02 DEBUG }, 22:55:02 DEBUG }, 22:55:02 DEBUG "pod": "am-555f4db595-fz8sn" 22:55:02 DEBUG "value": [ 22:55:02 DEBUG "value": [ 22:55:02 DEBUG { 22:55:02 DEBUG }, 22:55:02 DEBUG 22:55:02 DEBUG "value": [ 22:55:02 DEBUG "value": [ 22:55:02 DEBUG }, 22:55:02 DEBUG "value": [ 22:55:02 DEBUG "value": [ 22:55:02 DEBUG 22:55:02 DEBUG "value": [ 22:55:02 DEBUG 22:55:02 DEBUG "value": [ 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG "value": [ 22:55:02 DEBUG "value": [ 22:55:02 DEBUG }, 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG "metric": { 22:55:02 DEBUG "value": [ 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG "value": [ 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG "value": [ 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG "pod": "ds-idrepo-2" 22:55:02 DEBUG 22:55:02 DEBUG [...] 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG }, 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG [...] 22:55:02 DEBUG 22:55:02 DEBUG [...] 22:55:02 DEBUG 22:55:02 DEBUG [...] 22:55:02 DEBUG [...] 22:55:02 DEBUG [...] 22:55:02 DEBUG [...] 22:55:02 DEBUG [...] 22:55:02 DEBUG [...] 22:55:02 DEBUG [...] 22:55:02 DEBUG [...] 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG [...] 22:55:02 DEBUG [...] 22:55:02 DEBUG "value": [ 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG "pod": "ds-idrepo-0" 22:55:02 DEBUG 22:55:02 DEBUG [...] 22:55:02 DEBUG [...] 22:55:02 DEBUG 22:55:02 DEBUG [...] 22:55:02 DEBUG 22:55:02 DEBUG [...] 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG [...] 22:55:02 DEBUG [...] 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG [...] 22:55:02 DEBUG [...] 22:55:02 DEBUG }, 22:55:02 DEBUG [...] 22:55:02 DEBUG [...] 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG [...] 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG "value": [ 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG "instance": "10.142.0.43:9100" 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 1713304502, 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG "pod": "ds-idrepo-0" 22:55:02 DEBUG 22:55:02 DEBUG "pod": "ds-cts-2" 22:55:02 DEBUG 22:55:02 DEBUG "instance": "10.142.0.43:9100" 22:55:02 DEBUG "pod": "am-555f4db595-kzxsp" 22:55:02 DEBUG "pod": "am-555f4db595-kzxsp" 22:55:02 DEBUG "pod": "am-555f4db595-kzxsp" 22:55:02 DEBUG "pod": "ds-idrepo-0" 22:55:02 DEBUG "pod": "am-555f4db595-kzxsp" 22:55:02 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:55:02 DEBUG "pod": "am-555f4db595-kzxsp" 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG "pod": "ds-idrepo-0" 22:55:02 DEBUG 22:55:02 DEBUG }, 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG "0" 22:55:02 DEBUG 22:55:02 DEBUG 22:55:02 DEBUG "pod": "am-555f4db595-kzxsp" 22:55:02 DEBUG "pod": "am-555f4db595-kzxsp" 22:55:02 DEBUG }, 22:55:02 DEBUG "instance": "10.142.0.43:9100" 22:55:02 DEBUG }, 22:55:02 DEBUG "pod": "am-555f4db595-kzxsp" 22:55:02 DEBUG }, 22:55:02 DEBUG "value": [ 22:55:02 DEBUG }, 22:55:02 DEBUG }, 22:55:02 DEBUG }, 22:55:02 DEBUG }, 22:55:02 DEBUG }, 22:55:02 DEBUG }, 22:55:02 DEBUG "pod": "am-555f4db595-kzxsp" 22:55:02 DEBUG "pod": "am-555f4db595-kzxsp" 22:55:02 DEBUG 22:55:02 DEBUG }, 22:55:02 DEBUG [...] 22:55:02 DEBUG "value": [ 22:55:02 DEBUG "instance": "10.142.0.43:9100" 22:55:02 DEBUG "instance": "10.142.0.43:9100" 22:55:02 DEBUG ] 22:55:02 DEBUG "pod": "ds-idrepo-0" 22:55:02 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:55:02 DEBUG }, 22:55:02 DEBUG }, 22:55:02 DEBUG "value": [ 22:55:02 DEBUG }, 22:55:02 DEBUG "value": [ 22:55:02 DEBUG }, 22:55:02 DEBUG }, 22:55:02 DEBUG 1713304502, 22:55:02 DEBUG "value": [ 22:55:02 DEBUG "value": [ 22:55:02 DEBUG "value": [ 22:55:02 DEBUG "value": [ 22:55:02 DEBUG "value": [ 22:55:02 DEBUG "value": [ 22:55:02 DEBUG }, 22:55:02 DEBUG }, 22:55:02 DEBUG "pod": "am-555f4db595-kzxsp" 22:55:02 DEBUG "value": [ 22:55:02 DEBUG 22:55:02 DEBUG 1713304502, 22:55:02 DEBUG }, 22:55:02 DEBUG }, 22:55:02 DEBUG } 22:55:02 DEBUG }, 22:55:02 DEBUG }, 22:55:02 DEBUG "value": [ 22:55:02 DEBUG "value": [ 22:55:02 DEBUG 1713304502, 22:55:02 DEBUG "value": [ 22:55:02 DEBUG 1713304502, 22:55:02 DEBUG "value": [ 22:55:02 DEBUG "value": [ 22:55:02 DEBUG "19976651776" 22:55:02 DEBUG 1713304502, 22:55:02 DEBUG 1713304502, 22:55:02 DEBUG 1713304502, 22:55:02 DEBUG 1713304502, 22:55:02 DEBUG 1713304502, 22:55:02 DEBUG 1713304502, 22:55:02 DEBUG "value": [ 22:55:02 DEBUG "value": [ 22:55:02 DEBUG }, 22:55:02 DEBUG 1713304502, 22:55:02 DEBUG 22:55:02 DEBUG "0.0003333333333333262" 22:55:02 DEBUG "value": [ 22:55:02 DEBUG "value": [ 22:55:02 DEBUG ] 22:55:02 DEBUG "value": [ 22:55:02 DEBUG "value": [ 22:55:02 DEBUG 1713304502, 22:55:02 DEBUG 1713304502, 22:55:02 DEBUG "0" 22:55:02 DEBUG 1713304502, 22:55:02 DEBUG "0" 22:55:02 DEBUG 1713304502, 22:55:02 DEBUG 1713304502, 22:55:02 DEBUG ] 22:55:02 DEBUG "328101432.69834626" 22:55:02 DEBUG "0.027" 22:55:02 DEBUG "0" 22:55:02 DEBUG "2006.1333333333332" 22:55:02 DEBUG "1688539136" 22:55:02 DEBUG "2172.866666666667" 22:55:02 DEBUG 1713304502, 22:55:02 DEBUG 1713304502, 22:55:02 DEBUG "value": [ 22:55:02 DEBUG "0" 22:55:02 DEBUG 22:55:02 DEBUG ] 22:55:02 DEBUG 1713304502, 22:55:02 DEBUG 1713304502, 22:55:02 DEBUG } 22:55:02 DEBUG 1713304502, 22:55:02 DEBUG 1713304502, 22:55:02 DEBUG "3604314.5" 22:55:02 DEBUG "571.6279394320006" 22:55:02 DEBUG ] 22:55:02 DEBUG "49436079.9" 22:55:02 DEBUG ] 22:55:02 DEBUG "0" 22:55:02 DEBUG "13.733408624659855" 22:55:02 DEBUG } 22:55:02 DEBUG ] 22:55:02 DEBUG ] 22:55:02 DEBUG ] 22:55:02 DEBUG ] 22:55:02 DEBUG ] 22:55:02 DEBUG ] 22:55:02 DEBUG "0" 22:55:02 DEBUG "2174.5333333333333" 22:55:02 DEBUG 1713304502, 22:55:02 DEBUG ] 22:55:02 DEBUG "pod": "ds-idrepo-0" 22:55:02 DEBUG } 22:55:02 DEBUG "0.00046666666666605273" 22:55:02 DEBUG "49445268.23333333" 22:55:02 DEBUG } 22:55:02 DEBUG "0" 22:55:02 DEBUG "6.752255185830904" 22:55:02 DEBUG ] 22:55:02 DEBUG ] 22:55:02 DEBUG } 22:55:02 DEBUG ] 22:55:02 DEBUG } 22:55:02 DEBUG ] 22:55:02 DEBUG ] 22:55:02 DEBUG ] 22:55:02 DEBUG } 22:55:02 DEBUG } 22:55:02 DEBUG } 22:55:02 DEBUG } 22:55:02 DEBUG } 22:55:02 DEBUG } 22:55:02 DEBUG ] 22:55:02 DEBUG ] 22:55:02 DEBUG "2175.0666666666666" 22:55:02 DEBUG } 22:55:02 DEBUG }, 22:55:02 DEBUG ] 22:55:02 DEBUG ] 22:55:02 DEBUG ] 22:55:02 DEBUG ] 22:55:02 DEBUG ] 22:55:02 DEBUG } 22:55:02 DEBUG } 22:55:02 DEBUG ] 22:55:02 DEBUG } 22:55:02 DEBUG ] 22:55:02 DEBUG } 22:55:02 DEBUG } 22:55:02 DEBUG } 22:55:02 DEBUG ] 22:55:02 DEBUG ] 22:55:02 DEBUG ] 22:55:02 DEBUG ] 22:55:02 DEBUG ] 22:55:02 DEBUG ] 22:55:02 DEBUG } 22:55:02 DEBUG } 22:55:02 DEBUG ] 22:55:02 DEBUG ] 22:55:02 DEBUG "value": [ 22:55:02 DEBUG } 22:55:02 DEBUG } 22:55:02 DEBUG } 22:55:02 DEBUG } 22:55:02 DEBUG } 22:55:02 DEBUG ] 22:55:02 DEBUG ] 22:55:02 DEBUG } 22:55:02 DEBUG ] 22:55:02 DEBUG } 22:55:02 DEBUG ] 22:55:02 DEBUG ] 22:55:02 DEBUG } 22:55:02 DEBUG } 22:55:02 DEBUG } 22:55:02 DEBUG } 22:55:02 DEBUG } 22:55:02 DEBUG } 22:55:02 DEBUG } 22:55:02 DEBUG ] 22:55:02 DEBUG ] 22:55:02 DEBUG } 22:55:02 DEBUG } 22:55:02 DEBUG 1713304502, 22:55:02 DEBUG } 22:55:02 DEBUG ] 22:55:02 DEBUG ] 22:55:02 DEBUG ] 22:55:02 DEBUG ] 22:55:02 DEBUG } 22:55:02 DEBUG } 22:55:02 DEBUG } 22:55:02 DEBUG } 22:55:02 DEBUG } 22:55:02 DEBUG } 22:55:02 DEBUG } 22:55:02 DEBUG } 22:55:02 DEBUG } 22:55:02 DEBUG } 22:55:02 DEBUG } 22:55:02 DEBUG } 22:55:02 DEBUG } 22:55:02 DEBUG } 22:55:02 DEBUG } 22:55:02 DEBUG ] 22:55:02 DEBUG } 22:55:02 DEBUG "0" 22:55:02 DEBUG } 22:55:02 DEBUG } 22:55:02 DEBUG } 22:55:02 DEBUG } 22:55:02 DEBUG } 22:55:02 DEBUG } 22:55:02 DEBUG } 22:55:02 DEBUG } 22:55:02 DEBUG } 22:55:02 DEBUG } 22:55:02 DEBUG } 22:55:02 DEBUG } 22:55:02 DEBUG ] 22:55:02 DEBUG } 22:55:02 DEBUG } 22:55:02 DEBUG } 22:55:02 DEBUG } 22:55:02 DEBUG } 22:55:02 DEBUG } 22:55:02 DEBUG ] 22:55:02 DEBUG } 22:55:02 DEBUG } 22:55:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713304558" 22:55:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304558" 22:55:58 INFO [http_cmd]: http status code OK 22:55:58 DEBUG --- status code --- 22:55:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:55:58 DEBUG --- http response --- 22:55:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 22:55:58 ERROR [http_cmd]: ERROR 22:55:58 ERROR --- status code --- 22:55:58 ERROR http status code is 400 (expected [200, 401, 403, 404]) 22:55:58 ERROR --- http response --- 22:55:58 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 22:55:58 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 22:56:00 INFO 22:56:00 INFO [loop_until]: kubectl --namespace=xlou top pods 22:56:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:56:00 INFO [loop_until]: OK (rc = 0) 22:56:00 DEBUG --- stdout --- 22:56:00 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 9Mi am-555f4db595-4lbss 13506m 8681Mi am-555f4db595-fz8sn 13077m 8560Mi am-555f4db595-kzxsp 13268m 8966Mi ds-cts-0 19466m 13172Mi ds-cts-1 19541m 13058Mi ds-cts-2 19479m 13226Mi ds-idrepo-0 5792m 5030Mi ds-idrepo-1 5567m 2908Mi ds-idrepo-2 5583m 3220Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 3m 1207Mi idm-b85f9d978-rl76f 4m 1226Mi lodemon-bc756c647-cr7x6 2m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-7c9b667974-9mr2s 6838m 1522Mi 22:56:00 DEBUG --- stderr --- 22:56:00 DEBUG 22:56:02 INFO 22:56:02 INFO [loop_until]: kubectl --namespace=xlou top node 22:56:02 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:56:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304563" 22:56:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304563" 22:56:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304563" 22:56:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713304563" 22:56:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304563" 22:56:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304563" 22:56:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713304563" 22:56:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304563" 22:56:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713304563" 22:56:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304563" 22:56:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304563" 22:56:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304563" 22:56:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304563" 22:56:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304563" 22:56:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304563" 22:56:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304563" 22:56:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304563" 22:56:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304563" 22:56:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304563" 22:56:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304563" 22:56:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304563" 22:56:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304563" 22:56:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304563" 22:56:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304563" 22:56:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713304563" 22:56:03 INFO [http_cmd]: http status code OK 22:56:03 INFO [http_cmd]: http status code OK 22:56:03 INFO [http_cmd]: http status code OK 22:56:03 INFO [http_cmd]: http status code OK 22:56:03 INFO [http_cmd]: http status code OK 22:56:03 INFO [http_cmd]: http status code OK 22:56:03 INFO [http_cmd]: http status code OK 22:56:03 INFO [loop_until]: OK (rc = 0) 22:56:03 INFO [http_cmd]: http status code OK 22:56:03 INFO [http_cmd]: http status code OK 22:56:03 INFO [http_cmd]: http status code OK 22:56:03 INFO [http_cmd]: http status code OK 22:56:03 INFO [http_cmd]: http status code OK 22:56:03 INFO [http_cmd]: http status code OK 22:56:03 INFO [http_cmd]: http status code OK 22:56:03 INFO [http_cmd]: http status code OK 22:56:03 INFO [http_cmd]: http status code OK 22:56:03 INFO [http_cmd]: http status code OK 22:56:03 DEBUG --- status code --- 22:56:03 INFO [http_cmd]: http status code OK 22:56:03 INFO [http_cmd]: http status code OK 22:56:03 DEBUG --- status code --- 22:56:03 DEBUG --- status code --- 22:56:03 DEBUG --- status code --- 22:56:03 DEBUG --- status code --- 22:56:03 DEBUG --- status code --- 22:56:03 DEBUG --- status code --- 22:56:03 DEBUG --- stdout --- 22:56:03 DEBUG --- status code --- 22:56:03 DEBUG --- status code --- 22:56:03 DEBUG --- status code --- 22:56:03 DEBUG --- status code --- 22:56:03 DEBUG --- status code --- 22:56:03 DEBUG --- status code --- 22:56:03 DEBUG --- status code --- 22:56:03 INFO [http_cmd]: http status code OK 22:56:03 DEBUG --- status code --- 22:56:03 INFO [http_cmd]: http status code OK 22:56:03 DEBUG --- status code --- 22:56:03 DEBUG --- status code --- 22:56:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:56:03 DEBUG --- status code --- 22:56:03 DEBUG --- status code --- 22:56:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:56:03 INFO [http_cmd]: http status code OK 22:56:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:56:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:56:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:56:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:56:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:56:03 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 5084Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 19552m 65% 15733Mi 14% gke-xlou-cdm-default-pool-1b0769c8-nhlf 13238m 44% 10790Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 84m 0% 2650Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 5416m 18% 6566Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 20102m 67% 17702Mi 15% gke-xlou-cdm-default-pool-1b0769c8-xnmp 60m 0% 3159Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 13610m 45% 10459Mi 9% gke-xlou-cdm-ds-0f601751-hmm7 6045m 20% 5640Mi 5% gke-xlou-cdm-ds-0f601751-j488 19719m 66% 15133Mi 13% gke-xlou-cdm-ds-0f601751-mc5d 11646m 39% 3331Mi 2% gke-xlou-cdm-ds-0f601751-nj6q 14052m 47% 10848Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 6057m 20% 7359Mi 6% gke-xlou-cdm-frontend-a8771548-i82n 6940m 43% 6942Mi 11% 22:56:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:56:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:56:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:56:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:56:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:56:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:56:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:56:03 DEBUG --- status code --- 22:56:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:56:03 DEBUG --- status code --- 22:56:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:56:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:56:03 DEBUG --- http response --- 22:56:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:56:03 INFO [http_cmd]: http status code OK 22:56:03 INFO [http_cmd]: http status code OK 22:56:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:56:03 DEBUG --- http response --- 22:56:03 DEBUG --- status code --- 22:56:03 DEBUG --- http response --- 22:56:03 DEBUG --- http response --- 22:56:03 DEBUG --- http response --- 22:56:03 DEBUG --- http response --- 22:56:03 DEBUG --- http response --- 22:56:03 DEBUG --- stderr --- 22:56:03 DEBUG --- http response --- 22:56:03 DEBUG --- http response --- 22:56:03 DEBUG --- http response --- 22:56:03 DEBUG --- http response --- 22:56:03 DEBUG --- http response --- 22:56:03 DEBUG --- http response --- 22:56:03 INFO [http_cmd]: http status code OK 22:56:03 DEBUG --- http response --- 22:56:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:56:03 DEBUG --- http response --- 22:56:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:56:03 DEBUG --- http response --- 22:56:03 DEBUG --- http response --- 22:56:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:56:03 DEBUG --- http response --- 22:56:03 DEBUG --- status code --- 22:56:03 DEBUG --- status code --- 22:56:03 DEBUG --- http response --- 22:56:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:56:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:56:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:56:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:56:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:56:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:56:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:56:03 DEBUG 22:56:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:56:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:56:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:56:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:56:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:56:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:56:03 DEBUG --- status code --- 22:56:03 DEBUG --- http response --- 22:56:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:56:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:56:03 DEBUG --- http response --- 22:56:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:56:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:56:03 DEBUG ----- output ----- 22:56:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:56:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:56:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:56:03 DEBUG --- http response --- 22:56:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:56:03 DEBUG ----- output ----- 22:56:03 DEBUG ----- output ----- 22:56:03 DEBUG ----- output ----- 22:56:03 DEBUG ----- output ----- 22:56:03 DEBUG ----- output ----- 22:56:03 DEBUG ----- output ----- 22:56:03 DEBUG ----- output ----- 22:56:03 DEBUG ----- output ----- 22:56:03 DEBUG ----- output ----- 22:56:03 DEBUG ----- output ----- 22:56:03 DEBUG ----- output ----- 22:56:03 DEBUG ----- output ----- 22:56:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:56:03 DEBUG ----- output ----- 22:56:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:56:03 DEBUG ----- output ----- 22:56:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:56:03 DEBUG ----- output ----- 22:56:03 DEBUG ----- output ----- 22:56:03 DEBUG { 22:56:03 DEBUG ----- output ----- 22:56:03 DEBUG --- http response --- 22:56:03 DEBUG --- http response --- 22:56:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:56:03 DEBUG ----- output ----- 22:56:03 DEBUG { 22:56:03 DEBUG { 22:56:03 DEBUG { 22:56:03 DEBUG { 22:56:03 DEBUG { 22:56:03 DEBUG { 22:56:03 DEBUG { 22:56:03 DEBUG { 22:56:03 DEBUG { 22:56:03 DEBUG { 22:56:03 DEBUG { 22:56:03 DEBUG { 22:56:03 DEBUG --- http response --- 22:56:03 DEBUG { 22:56:03 DEBUG ----- output ----- 22:56:03 DEBUG { 22:56:03 DEBUG ----- output ----- 22:56:03 DEBUG { 22:56:03 DEBUG "status": "success", 22:56:03 DEBUG "status": "success", 22:56:03 DEBUG { 22:56:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:56:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:56:03 DEBUG ----- output ----- 22:56:03 DEBUG { 22:56:03 DEBUG "status": "success", 22:56:03 DEBUG "status": "success", 22:56:03 DEBUG "status": "success", 22:56:03 DEBUG "status": "success", 22:56:03 DEBUG "status": "success", 22:56:03 DEBUG "status": "success", 22:56:03 DEBUG "status": "success", 22:56:03 DEBUG "status": "success", 22:56:03 DEBUG "status": "success", 22:56:03 DEBUG "status": "success", 22:56:03 DEBUG "status": "success", 22:56:03 DEBUG "status": "success", 22:56:03 DEBUG "status": "success", 22:56:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:56:03 DEBUG { 22:56:03 DEBUG "status": "success", 22:56:03 DEBUG { 22:56:03 DEBUG { 22:56:03 DEBUG "data": { 22:56:03 DEBUG "data": { 22:56:03 DEBUG "status": "success", 22:56:03 DEBUG ----- output ----- 22:56:03 DEBUG ----- output ----- 22:56:03 DEBUG { 22:56:03 DEBUG "status": "success", 22:56:03 DEBUG "data": { 22:56:03 DEBUG "data": { 22:56:03 DEBUG "data": { 22:56:03 DEBUG "data": { 22:56:03 DEBUG "data": { 22:56:03 DEBUG "data": { 22:56:03 DEBUG "data": { 22:56:03 DEBUG "data": { 22:56:03 DEBUG "data": { 22:56:03 DEBUG "data": { 22:56:03 DEBUG "data": { 22:56:03 DEBUG "data": { 22:56:03 DEBUG "data": { 22:56:03 DEBUG ----- output ----- 22:56:03 DEBUG "status": "success", 22:56:03 DEBUG "data": { 22:56:03 DEBUG "status": "success", 22:56:03 DEBUG "status": "success", 22:56:03 DEBUG "resultType": "vector", 22:56:03 DEBUG "resultType": "vector", 22:56:03 DEBUG "data": { 22:56:03 DEBUG { 22:56:03 DEBUG { 22:56:03 DEBUG "status": "success", 22:56:03 DEBUG "data": { 22:56:03 DEBUG "resultType": "vector", 22:56:03 DEBUG "resultType": "vector", 22:56:03 DEBUG "resultType": "vector", 22:56:03 DEBUG "resultType": "vector", 22:56:03 DEBUG "resultType": "vector", 22:56:03 DEBUG "resultType": "vector", 22:56:03 DEBUG "resultType": "vector", 22:56:03 DEBUG "resultType": "vector", 22:56:03 DEBUG "resultType": "vector", 22:56:03 DEBUG "resultType": "vector", 22:56:03 DEBUG "resultType": "vector", 22:56:03 DEBUG "resultType": "vector", 22:56:03 DEBUG "resultType": "vector", 22:56:03 DEBUG { 22:56:03 DEBUG "data": { 22:56:03 DEBUG "resultType": "vector", 22:56:03 DEBUG "data": { 22:56:03 DEBUG "data": { 22:56:03 DEBUG "result": [ 22:56:03 DEBUG "result": [ 22:56:03 DEBUG "resultType": "vector", 22:56:03 DEBUG "status": "success", 22:56:03 DEBUG "status": "success", 22:56:03 DEBUG "data": { 22:56:03 DEBUG "resultType": "vector", 22:56:03 DEBUG "result": [ 22:56:03 DEBUG "result": [ 22:56:03 DEBUG "result": [ 22:56:03 DEBUG "result": [ 22:56:03 DEBUG "result": [ 22:56:03 DEBUG "result": [ 22:56:03 DEBUG "result": [ 22:56:03 DEBUG "result": [ 22:56:03 DEBUG "result": [ 22:56:03 DEBUG "result": [ 22:56:03 DEBUG "result": [ 22:56:03 DEBUG "result": [ 22:56:03 DEBUG "result": [ 22:56:03 DEBUG "status": "success", 22:56:03 DEBUG "resultType": "vector", 22:56:03 DEBUG "result": [ 22:56:03 DEBUG "resultType": "vector", 22:56:03 DEBUG "resultType": "vector", 22:56:03 DEBUG { 22:56:03 DEBUG { 22:56:03 DEBUG "result": [ 22:56:03 DEBUG "data": { 22:56:03 DEBUG "data": { 22:56:03 DEBUG "resultType": "vector", 22:56:03 DEBUG "result": [ 22:56:03 DEBUG { 22:56:03 DEBUG { 22:56:03 DEBUG { 22:56:03 DEBUG { 22:56:03 DEBUG { 22:56:03 DEBUG { 22:56:03 DEBUG { 22:56:03 DEBUG { 22:56:03 DEBUG { 22:56:03 DEBUG { 22:56:03 DEBUG { 22:56:03 DEBUG { 22:56:03 DEBUG { 22:56:03 DEBUG "data": { 22:56:03 DEBUG "result": [ 22:56:03 DEBUG { 22:56:03 DEBUG "result": [ 22:56:03 DEBUG "result": [ 22:56:03 DEBUG "metric": { 22:56:03 DEBUG "metric": { 22:56:03 DEBUG { 22:56:03 DEBUG "resultType": "vector", 22:56:03 DEBUG "resultType": "vector", 22:56:03 DEBUG "result": [ 22:56:03 DEBUG { 22:56:03 DEBUG "metric": { 22:56:03 DEBUG "metric": { 22:56:03 DEBUG "metric": { 22:56:03 DEBUG "metric": { 22:56:03 DEBUG "metric": { 22:56:03 DEBUG "metric": { 22:56:03 DEBUG "metric": { 22:56:03 DEBUG "metric": { 22:56:03 DEBUG "metric": { 22:56:03 DEBUG "metric": { 22:56:03 DEBUG "metric": { 22:56:03 DEBUG "metric": { 22:56:03 DEBUG "metric": { 22:56:03 DEBUG "resultType": "vector", 22:56:03 DEBUG { 22:56:03 DEBUG "metric": { 22:56:03 DEBUG { 22:56:03 DEBUG { 22:56:03 DEBUG "instance": "10.142.0.107:9100" 22:56:03 DEBUG "instance": "10.142.0.107:9100" 22:56:03 DEBUG "metric": { 22:56:03 DEBUG "result": [ 22:56:03 DEBUG "result": [ 22:56:03 DEBUG { 22:56:03 DEBUG "metric": { 22:56:03 DEBUG "pod": "ds-idrepo-2" 22:56:03 DEBUG "pod": "ds-idrepo-2" 22:56:03 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:56:03 DEBUG "pod": "am-555f4db595-fz8sn" 22:56:03 DEBUG "pod": "am-555f4db595-fz8sn" 22:56:03 DEBUG "instance": "10.142.0.107:9100" 22:56:03 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:56:03 DEBUG "pod": "ds-idrepo-2" 22:56:03 DEBUG "pod": "am-555f4db595-fz8sn" 22:56:03 DEBUG "pod": "am-555f4db595-fz8sn" 22:56:03 DEBUG "pod": "ds-idrepo-2" 22:56:03 [35mDEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:56:03 DEBUG "pod": "ds-idrepo-2" 22:56:03 DEBUG "result": [ 22:56:03 DEBUG "metric": { 22:56:03 DEBUG "pod": "am-555f4db595-fz8sn" 22:56:03 DEBUG "metric": { 22:56:03 DEBUG "metric": { 22:56:03 DEBUG }, 22:56:03 DEBUG }, 22:56:03 DEBUG "pod": "ds-idrepo-2" 22:56:03 DEBUG { 22:56:03 DEBUG { 22:56:03 DEBUG "metric": { 22:56:03 DEBUG "instance": "10.142.0.107:9100" 22:56:03 DEBUG }, 22:56:03 DEBUG }, 22:56:03 DEBUG }, 22:56:03 DEBUG }, 22:56:03 DEBUG }, 22:56:03 DEBUG }, 22:56:03 DEBUG }, 22:56:03 DEBUG }, 22:56:03 DEBUG }, 22:56:03 DEBUG }, 22:56:03 DEBUG }, 22:56:03 DEBUG }, 22:56:03 DEBUG }, 22:56:03 DEBUG { 22:56:03 DEBUG "instance": "10.142.0.107:9100" 22:56:03 DEBUG }, 22:56:03 DEBUG "pod": "ds-idrepo-2" 22:56:03 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:56:03 DEBUG "value": [ 22:56:03 DEBUG "value": [ 22:56:03 DEBUG }, 22:56:03 DEBUG "metric": { 22:56:03 DEBUG "metric": { 22:56:03 DEBUG "pod": "am-555f4db595-fz8sn" 22:56:03 DEBUG }, 22:56:03 DEBUG "value": [ 22:56:03 DEBUG "value": [ 22:56:03 DEBUG "value": [ 22:56:03 DEBUG "value": [ 22:56:03 DEBUG "value": [ 22:56:03 DEBUG "value": [ 22:56:03 DEBUG "value": [ 22:56:03 DEBUG "value": [ 22:56:03 DEBUG "value": [ 22:56:03 DEBUG "value": [ 22:56:03 DEBUG "value": [ 22:56:03 DEBUG "value": [ 22:56:03 DEBUG "value": [ 22:56:03 DEBUG "metric": { 22:56:03 DEBUG }, 22:56:03 DEBUG "value": [ 22:56:03 DEBUG }, 22:56:03 DEBUG }, 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG "value": [ 22:56:03 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:56:03 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:56:03 DEBUG }, 22:56:03 DEBUG "value": [ 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG "pod": "am-555f4db595-fz8sn" 22:56:03 DEBUG "value": [ 22:56:03 DEBUG 22:56:03 DEBUG "value": [ 22:56:03 DEBUG "value": [ 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG }, 22:56:03 DEBUG }, 22:56:03 DEBUG "value": [ 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG }, 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG "value": [ 22:56:03 DEBUG "value": [ 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG "value": [ 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG [...] 22:56:03 DEBUG [...] 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG [...] 22:56:03 DEBUG [...] 22:56:03 DEBUG [...] 22:56:03 DEBUG [...] 22:56:03 DEBUG [...] 22:56:03 DEBUG [...] 22:56:03 DEBUG [...] 22:56:03 DEBUG [...] 22:56:03 DEBUG [...] 22:56:03 DEBUG [...] 22:56:03 DEBUG [...] 22:56:03 DEBUG [...] 22:56:03 DEBUG [...] 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG [...] 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG [...] 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG [...] 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG [...] 22:56:03 DEBUG 22:56:03 DEBUG [...] 22:56:03 DEBUG [...] 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG [...] 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG [...] 22:56:03 DEBUG [...] 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG [...] 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG "instance": "10.142.0.43:9100" 22:56:03 DEBUG "instance": "10.142.0.43:9100" 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG "instance": "10.142.0.43:9100" 22:56:03 DEBUG 22:56:03 DEBUG "pod": "ds-cts-2" 22:56:03 DEBUG "pod": "idm-b85f9d978-j5pbm" 22:56:03 DEBUG "pod": "am-555f4db595-kzxsp" 22:56:03 DEBUG "pod": "am-555f4db595-kzxsp" 22:56:03 DEBUG "instance": "10.142.0.43:9100" 22:56:03 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:56:03 DEBUG "pod": "ds-idrepo-0" 22:56:03 DEBUG "pod": "am-555f4db595-kzxsp" 22:56:03 DEBUG "pod": "am-555f4db595-kzxsp" 22:56:03 DEBUG "pod": "ds-idrepo-0" 22:56:03 DEBUG "pod": "idm-b85f9d978-j5pbm" 22:56:03 DEBUG "pod": "ds-idrepo-0" 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG "pod": "am-555f4db595-kzxsp" 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG "pod": "idm-b85f9d978-j5pbm" 22:56:03 DEBUG }, 22:56:03 DEBUG "pod": "ds-idrepo-0" 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG }, 22:56:03 DEBUG "pod": "ds-idrepo-0" 22:56:03 DEBUG }, 22:56:03 DEBUG }, 22:56:03 DEBUG }, 22:56:03 DEBUG }, 22:56:03 DEBUG }, 22:56:03 DEBUG }, 22:56:03 DEBUG }, 22:56:03 DEBUG }, 22:56:03 DEBUG }, 22:56:03 DEBUG }, 22:56:03 DEBUG }, 22:56:03 DEBUG }, 22:56:03 DEBUG 22:56:03 DEBUG "instance": "10.142.0.43:9100" 22:56:03 DEBUG }, 22:56:03 DEBUG "pod": "ds-idrepo-0" 22:56:03 DEBUG }, 22:56:03 DEBUG }, 22:56:03 DEBUG "value": [ 22:56:03 DEBUG }, 22:56:03 DEBUG 22:56:03 DEBUG 22:56:03 DEBUG "pod": "am-555f4db595-kzxsp" 22:56:03 DEBUG "value": [ 22:56:03 DEBUG }, 22:56:03 DEBUG "value": [ 22:56:03 DEBUG "value": [ 22:56:03 DEBUG "value": [ 22:56:03 DEBUG "value": [ 22:56:03 DEBUG "value": [ 22:56:03 DEBUG "value": [ 22:56:03 DEBUG "value": [ 22:56:03 DEBUG "value": [ 22:56:03 DEBUG "value": [ 22:56:03 DEBUG "value": [ 22:56:03 DEBUG "value": [ 22:56:03 DEBUG "value": [ 22:56:03 DEBUG 22:56:03 DEBUG }, 22:56:03 DEBUG "value": [ 22:56:03 DEBUG }, 22:56:03 DEBUG "value": [ 22:56:03 DEBUG "value": [ 22:56:03 DEBUG 1713304563, 22:56:03 DEBUG "value": [ 22:56:03 DEBUG "pod": "idm-b85f9d978-j5pbm" 22:56:03 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:56:03 DEBUG }, 22:56:03 DEBUG 1713304563, 22:56:03 DEBUG "value": [ 22:56:03 DEBUG 1713304563, 22:56:03 DEBUG 1713304563, 22:56:03 DEBUG 1713304563, 22:56:03 DEBUG 1713304563, 22:56:03 DEBUG 1713304563, 22:56:03 DEBUG 1713304563, 22:56:03 DEBUG 1713304563, 22:56:03 DEBUG 1713304563, 22:56:03 DEBUG 1713304563, 22:56:03 DEBUG 1713304563, 22:56:03 DEBUG 1713304563, 22:56:03 DEBUG 1713304563, 22:56:03 DEBUG "pod": "am-555f4db595-kzxsp" 22:56:03 DEBUG "value": [ 22:56:03 DEBUG 1713304563, 22:56:03 DEBUG "value": [ 22:56:03 DEBUG 1713304563, 22:56:03 DEBUG 1713304563, 22:56:03 DEBUG "19977337856" 22:56:03 DEBUG 1713304563, 22:56:03 DEBUG }, 22:56:03 DEBUG }, 22:56:03 DEBUG "value": [ 22:56:03 DEBUG "49393834.06666666" 22:56:03 DEBUG 1713304563, 22:56:03 DEBUG "0" 22:56:03 DEBUG "37929419.90961987" 22:56:03 DEBUG "2003.5333333333333" 22:56:03 DEBUG "0.027" 22:56:03 DEBUG "0" 22:56:03 DEBUG "1690816512" 22:56:03 DEBUG "0" 22:56:03 DEBUG "2165.3333333333335" 22:56:03 DEBUG "3614378" 22:56:03 DEBUG "0" 22:56:03 DEBUG "488.5624016258167" 22:56:03 DEBUG "0" 22:56:03 DEBUG }, 22:56:03 DEBUG 1713304563, 22:56:03 DEBUG "0" 22:56:03 DEBUG 1713304563, 22:56:03 DEBUG "49384518.8" 22:56:03 DEBUG "0" 22:56:03 DEBUG ] 22:56:03 DEBUG "0" 22:56:03 DEBUG "value": [ 22:56:03 DEBUG "value": [ 22:56:03 DEBUG 1713304563, 22:56:03 DEBUG ] 22:56:03 DEBUG "0" 22:56:03 DEBUG ] 22:56:03 DEBUG ] 22:56:03 DEBUG ] 22:56:03 DEBUG ] 22:56:03 DEBUG ] 22:56:03 DEBUG ] 22:56:03 DEBUG ] 22:56:03 DEBUG ] 22:56:03 DEBUG ] 22:56:03 DEBUG ] 22:56:03 DEBUG ] 22:56:03 DEBUG ] 22:56:03 DEBUG "value": [ 22:56:03 DEBUG "0.0005000000000004927" 22:56:03 DEBUG ] 22:56:03 DEBUG "0" 22:56:03 DEBUG ] 22:56:03 DEBUG ] 22:56:03 DEBUG } 22:56:03 DEBUG ] 22:56:03 DEBUG 1713304563, 22:56:03 DEBUG 1713304563, 22:56:03 DEBUG "2163.8" 22:56:03 DEBUG } 22:56:03 DEBUG ] 22:56:03 DEBUG } 22:56:03 DEBUG } 22:56:03 DEBUG } 22:56:03 DEBUG } 22:56:03 DEBUG } 22:56:03 DEBUG } 22:56:03 DEBUG } 22:56:03 DEBUG } 22:56:03 DEBUG } 22:56:03 DEBUG } 22:56:03 DEBUG } 22:56:03 DEBUG } 22:56:03 DEBUG ] 22:56:03 DEBUG ] 22:56:03 DEBUG } 22:56:03 DEBUG ] 22:56:03 DEBUG } 22:56:03 DEBUG } 22:56:03 DEBUG ] 22:56:03 DEBUG } 22:56:03 DEBUG "0.0018813111522562908" 22:56:03 DEBUG "6.829114171258323" 22:56:03 DEBUG ] 22:56:03 DEBUG ] 22:56:03 DEBUG } 22:56:03 DEBUG ] 22:56:03 DEBUG ] 22:56:03 DEBUG ] 22:56:03 DEBUG ] 22:56:03 DEBUG ] 22:56:03 DEBUG ] 22:56:03 DEBUG ] 22:56:03 DEBUG ] 22:56:03 DEBUG ] 22:56:03 DEBUG ] 22:56:03 DEBUG ] 22:56:03 DEBUG 1713304563, 22:56:03 DEBUG } 22:56:03 DEBUG } 22:56:03 DEBUG ] 22:56:03 DEBUG } 22:56:03 DEBUG ] 22:56:03 DEBUG ] 22:56:03 DEBUG } 22:56:03 DEBUG ] 22:56:03 DEBUG ] 22:56:03 DEBUG } 22:56:03 DEBUG } 22:56:03 DEBUG } 22:56:03 DEBUG ] 22:56:03 DEBUG } 22:56:03 DEBUG } 22:56:03 DEBUG } 22:56:03 DEBUG } 22:56:03 DEBUG } 22:56:03 DEBUG } 22:56:03 DEBUG } 22:56:03 DEBUG } 22:56:03 DEBUG } 22:56:03 DEBUG } 22:56:03 DEBUG } 22:56:03 DEBUG "2164.6666666666665" 22:56:03 DEBUG } 22:56:03 DEBUG ] 22:56:03 DEBUG } 22:56:03 DEBUG ] 22:56:03 DEBUG } 22:56:03 DEBUG } 22:56:03 DEBUG } 22:56:03 DEBUG } 22:56:03 DEBUG ] 22:56:03 DEBUG ] 22:56:03 DEBUG ] 22:56:03 DEBUG } 22:56:03 DEBUG } 22:56:03 DEBUG } 22:56:03 DEBUG } 22:56:03 DEBUG } 22:56:03 DEBUG } 22:56:03 DEBUG } 22:56:03 DEBUG } 22:56:03 DEBUG } 22:56:03 DEBUG } 22:56:03 DEBUG } 22:56:03 DEBUG ] 22:56:03 DEBUG } 22:56:03 DEBUG } 22:56:03 DEBUG } 22:56:03 DEBUG } 22:56:03 DEBUG } 22:56:03 DEBUG } 22:56:03 DEBUG } 22:56:03 DEBUG } 22:56:03 DEBUG } 22:56:03 DEBUG } 22:56:03 DEBUG } 22:56:03 DEBUG } 22:56:03 DEBUG } 22:56:03 DEBUG } 22:56:03 DEBUG } 22:56:03 DEBUG ] 22:56:03 DEBUG } 22:56:03 DEBUG } 22:56:03 DEBUG ] 22:56:03 DEBUG } 22:56:03 DEBUG } 22:56:03 DEBUG } 22:56:03 DEBUG } 22:56:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304618" 22:56:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713304618" 22:56:58 INFO [http_cmd]: http status code OK 22:56:58 DEBUG --- status code --- 22:56:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:56:58 DEBUG --- http response --- 22:56:58 ERROR [http_cmd]: ERROR 22:56:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 22:56:58 ERROR --- status code --- 22:56:58 ERROR http status code is 400 (expected [200, 401, 403, 404]) 22:56:58 ERROR --- http response --- 22:56:58 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 22:56:58 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 22:57:00 INFO 22:57:00 INFO [loop_until]: kubectl --namespace=xlou top pods 22:57:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:57:00 INFO [loop_until]: OK (rc = 0) 22:57:00 DEBUG --- stdout --- 22:57:00 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 9Mi am-555f4db595-4lbss 12791m 8675Mi am-555f4db595-fz8sn 13194m 8562Mi am-555f4db595-kzxsp 13574m 8966Mi ds-cts-0 19592m 13009Mi ds-cts-1 18736m 13051Mi ds-cts-2 18584m 13452Mi ds-idrepo-0 5662m 5030Mi ds-idrepo-1 5557m 2907Mi ds-idrepo-2 5556m 3220Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 3m 1207Mi idm-b85f9d978-rl76f 3m 1225Mi lodemon-bc756c647-cr7x6 2m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-7c9b667974-9mr2s 6709m 1521Mi 22:57:00 DEBUG --- stderr --- 22:57:00 DEBUG 22:57:03 INFO 22:57:03 INFO [loop_until]: kubectl --namespace=xlou top node 22:57:03 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:57:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304623" 22:57:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304623" 22:57:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304623" 22:57:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304623" 22:57:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304623" 22:57:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713304623" 22:57:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304623" 22:57:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304623" 22:57:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304623" 22:57:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304623" 22:57:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713304623" 22:57:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713304623" 22:57:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304623" 22:57:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304623" 22:57:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304623" 22:57:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304623" 22:57:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304623" 22:57:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304623" 22:57:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304623" 22:57:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304623" 22:57:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304623" 22:57:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713304623" 22:57:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304623" 22:57:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304623" 22:57:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304623" 22:57:03 INFO [http_cmd]: http status code OK 22:57:03 INFO [loop_until]: OK (rc = 0) 22:57:03 INFO [http_cmd]: http status code OK 22:57:03 INFO [http_cmd]: http status code OK 22:57:03 INFO [http_cmd]: http status code OK 22:57:03 INFO [http_cmd]: http status code OK 22:57:03 INFO [http_cmd]: http status code OK 22:57:03 INFO [http_cmd]: http status code OK 22:57:03 INFO [http_cmd]: http status code OK 22:57:03 INFO [http_cmd]: http status code OK 22:57:03 INFO [http_cmd]: http status code OK 22:57:03 INFO [http_cmd]: http status code OK 22:57:03 INFO [http_cmd]: http status code OK 22:57:03 INFO [http_cmd]: http status code OK 22:57:03 INFO [http_cmd]: http status code OK 22:57:03 INFO [http_cmd]: http status code OK 22:57:03 DEBUG --- status code --- 22:57:03 INFO [http_cmd]: http status code OK 22:57:03 DEBUG --- stdout --- 22:57:03 INFO [http_cmd]: http status code OK 22:57:03 DEBUG --- status code --- 22:57:03 DEBUG --- status code --- 22:57:03 INFO [http_cmd]: http status code OK 22:57:03 INFO [http_cmd]: http status code OK 22:57:03 DEBUG --- status code --- 22:57:03 INFO [http_cmd]: http status code OK 22:57:03 INFO [http_cmd]: http status code OK 22:57:03 INFO [http_cmd]: http status code OK 22:57:03 DEBUG --- status code --- 22:57:03 DEBUG --- status code --- 22:57:03 DEBUG --- status code --- 22:57:03 DEBUG --- status code --- 22:57:03 DEBUG --- status code --- 22:57:03 DEBUG --- status code --- 22:57:03 DEBUG --- status code --- 22:57:03 DEBUG --- status code --- 22:57:03 DEBUG --- status code --- 22:57:03 INFO [http_cmd]: http status code OK 22:57:03 DEBUG --- status code --- 22:57:03 DEBUG --- status code --- 22:57:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:57:03 DEBUG --- status code --- 22:57:03 DEBUG --- status code --- 22:57:03 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 63m 0% 5083Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 19082m 63% 15642Mi 14% gke-xlou-cdm-default-pool-1b0769c8-nhlf 13374m 44% 10785Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 80m 0% 2640Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 5602m 18% 6565Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 20536m 68% 17484Mi 15% gke-xlou-cdm-default-pool-1b0769c8-xnmp 57m 0% 3144Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 13029m 43% 10479Mi 9% gke-xlou-cdm-ds-0f601751-hmm7 5984m 20% 5651Mi 5% gke-xlou-cdm-ds-0f601751-j488 18762m 62% 15062Mi 13% gke-xlou-cdm-ds-0f601751-mc5d 11523m 38% 3357Mi 3% gke-xlou-cdm-ds-0f601751-nj6q 13653m 45% 10845Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 5877m 19% 7351Mi 6% gke-xlou-cdm-frontend-a8771548-i82n 6637m 41% 6943Mi 11% 22:57:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:57:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:57:03 DEBUG --- status code --- 22:57:03 DEBUG --- status code --- 22:57:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:57:03 DEBUG --- status code --- 22:57:03 DEBUG --- status code --- 22:57:03 DEBUG --- status code --- 22:57:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:57:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:57:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:57:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:57:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:57:03 INFO [http_cmd]: http status code OK 22:57:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:57:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:57:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:57:03 DEBUG --- http response --- 22:57:03 INFO [http_cmd]: http status code OK 22:57:03 DEBUG --- status code --- 22:57:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:57:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:57:03 DEBUG --- http response --- 22:57:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:57:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:57:03 DEBUG --- stderr --- 22:57:03 DEBUG --- http response --- 22:57:03 DEBUG --- http response --- 22:57:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:57:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:57:03 DEBUG --- http response --- 22:57:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:57:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:57:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:57:03 DEBUG --- http response --- 22:57:03 DEBUG --- http response --- 22:57:03 DEBUG --- http response --- 22:57:03 DEBUG --- http response --- 22:57:03 DEBUG --- http response --- 22:57:03 DEBUG --- status code --- 22:57:03 DEBUG --- http response --- 22:57:03 DEBUG --- http response --- 22:57:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:57:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:57:03 DEBUG --- status code --- 22:57:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:57:03 DEBUG --- http response --- 22:57:03 DEBUG --- http response --- 22:57:03 DEBUG --- http response --- 22:57:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:57:03 DEBUG --- http response --- 22:57:03 DEBUG 22:57:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 22:57:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:57:03 DEBUG --- http response --- 22:57:03 DEBUG --- http response --- 22:57:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:57:03 DEBUG --- http response --- 22:57:03 DEBUG --- http response --- 22:57:03 DEBUG --- http response --- 22:57:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:57:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:57:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:57:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:57:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:57:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:57:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:57:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:57:03 DEBUG --- http response --- 22:57:03 DEBUG ----- output ----- 22:57:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:57:03 DEBUG --- http response --- 22:57:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:57:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:57:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 22:57:03 DEBUG ----- output ----- 22:57:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:57:03 DEBUG ----- output ----- 22:57:03 DEBUG ----- output ----- 22:57:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:57:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:57:03 DEBUG ----- output ----- 22:57:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:57:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:57:03 DEBUG ----- output ----- 22:57:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 22:57:03 DEBUG ----- output ----- 22:57:03 DEBUG ----- output ----- 22:57:03 DEBUG ----- output ----- 22:57:03 DEBUG { 22:57:03 DEBUG ----- output ----- 22:57:03 DEBUG ----- output ----- 22:57:03 DEBUG ----- output ----- 22:57:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:57:03 DEBUG { 22:57:03 DEBUG --- http response --- 22:57:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:57:03 DEBUG ----- output ----- 22:57:03 DEBUG ----- output ----- 22:57:03 DEBUG ----- output ----- 22:57:03 DEBUG { 22:57:03 DEBUG ----- output ----- 22:57:03 DEBUG { 22:57:03 DEBUG { 22:57:03 DEBUG ----- output ----- 22:57:03 DEBUG ----- output ----- 22:57:03 DEBUG { 22:57:03 DEBUG ----- output ----- 22:57:03 DEBUG ----- output ----- 22:57:03 DEBUG { 22:57:03 DEBUG ----- output ----- 22:57:03 DEBUG { 22:57:03 DEBUG { 22:57:03 DEBUG "status": "success", 22:57:03 DEBUG --- http response --- 22:57:03 DEBUG { 22:57:03 DEBUG { 22:57:03 DEBUG { 22:57:03 DEBUG ----- output ----- 22:57:03 DEBUG "status": "success", 22:57:03 DEBUG ----- output ----- 22:57:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:57:03 DEBUG { 22:57:03 DEBUG { 22:57:03 DEBUG { 22:57:03 DEBUG "status": "success", 22:57:03 DEBUG { 22:57:03 DEBUG "status": "success", 22:57:03 DEBUG "status": "success", 22:57:03 DEBUG { 22:57:03 DEBUG { 22:57:03 DEBUG "status": "success", 22:57:03 DEBUG { 22:57:03 DEBUG { 22:57:03 DEBUG "status": "success", 22:57:03 DEBUG { 22:57:03 DEBUG "status": "success", 22:57:03 DEBUG "status": "success", 22:57:03 DEBUG "data": { 22:57:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:57:03 DEBUG "status": "success", 22:57:03 DEBUG "status": "success", 22:57:03 DEBUG "status": "success", 22:57:03 DEBUG { 22:57:03 DEBUG "data": { 22:57:03 DEBUG { 22:57:03 DEBUG ----- output ----- 22:57:03 DEBUG "status": "success", 22:57:03 DEBUG "status": "success", 22:57:03 DEBUG "status": "success", 22:57:03 DEBUG "data": { 22:57:03 DEBUG "status": "success", 22:57:03 DEBUG "data": { 22:57:03 DEBUG "data": { 22:57:03 DEBUG "status": "success", 22:57:03 DEBUG "status": "success", 22:57:03 DEBUG "data": { 22:57:03 DEBUG "status": "success", 22:57:03 DEBUG "status": "success", 22:57:03 DEBUG "data": { 22:57:03 DEBUG "status": "success", 22:57:03 DEBUG "data": { 22:57:03 DEBUG "data": { 22:57:03 DEBUG "resultType": "vector", 22:57:03 DEBUG ----- output ----- 22:57:03 DEBUG "data": { 22:57:03 DEBUG "data": { 22:57:03 DEBUG "data": { 22:57:03 DEBUG "status": "success", 22:57:03 DEBUG "resultType": "vector", 22:57:03 DEBUG "status": "success", 22:57:03 DEBUG { 22:57:03 DEBUG "data": { 22:57:03 DEBUG "data": { 22:57:03 DEBUG "data": { 22:57:03 DEBUG "resultType": "vector", 22:57:03 DEBUG "data": { 22:57:03 DEBUG "resultType": "vector", 22:57:03 DEBUG "resultType": "vector", 22:57:03 DEBUG "data": { 22:57:03 DEBUG "data": { 22:57:03 DEBUG "resultType": "vector", 22:57:03 DEBUG "data": { 22:57:03 DEBUG "data": { 22:57:03 DEBUG "resultType": "vector", 22:57:03 DEBUG "data": { 22:57:03 DEBUG "resultType": "vector", 22:57:03 DEBUG "resultType": "vector", 22:57:03 DEBUG "result": [ 22:57:03 DEBUG { 22:57:03 DEBUG "resultType": "vector", 22:57:03 DEBUG "resultType": "vector", 22:57:03 DEBUG "resultType": "vector", 22:57:03 DEBUG "data": { 22:57:03 DEBUG "result": [ 22:57:03 DEBUG "data": { 22:57:03 DEBUG "status": "success", 22:57:03 DEBUG "resultType": "vector", 22:57:03 DEBUG "resultType": "vector", 22:57:03 DEBUG "resultType": "vector", 22:57:03 DEBUG "result": [ 22:57:03 DEBUG "resultType": "vector", 22:57:03 DEBUG "result": [ 22:57:03 DEBUG "result": [ 22:57:03 DEBUG "resultType": "vector", 22:57:03 DEBUG "resultType": "vector", 22:57:03 DEBUG "result": [ 22:57:03 DEBUG "resultType": "vector", 22:57:03 DEBUG "resultType": "vector", 22:57:03 DEBUG "result": [ 22:57:03 DEBUG "resultType": "vector", 22:57:03 DEBUG "result": [ 22:57:03 DEBUG "result": [ 22:57:03 DEBUG { 22:57:03 DEBUG "status": "success", 22:57:03 DEBUG "result": [ 22:57:03 DEBUG "result": [ 22:57:03 DEBUG "result": [ 22:57:03 DEBUG "resultType": "vector", 22:57:03 DEBUG { 22:57:03 DEBUG "resultType": "vector", 22:57:03 DEBUG "data": { 22:57:03 DEBUG "result": [ 22:57:03 DEBUG "result": [ 22:57:03 DEBUG "result": [ 22:57:03 DEBUG { 22:57:03 DEBUG "result": [ 22:57:03 DEBUG { 22:57:03 DEBUG { 22:57:03 DEBUG "result": [ 22:57:03 DEBUG "result": [ 22:57:03 DEBUG { 22:57:03 DEBUG "result": [ 22:57:03 DEBUG "result": [ 22:57:03 DEBUG { 22:57:03 DEBUG "result": [ 22:57:03 DEBUG { 22:57:03 DEBUG { 22:57:03 DEBUG "metric": { 22:57:03 DEBUG "data": { 22:57:03 DEBUG { 22:57:03 DEBUG { 22:57:03 DEBUG { 22:57:03 DEBUG "result": [ 22:57:03 DEBUG "metric": { 22:57:03 DEBUG "result": [ 22:57:03 DEBUG "resultType": "vector", 22:57:03 DEBUG { 22:57:03 DEBUG { 22:57:03 DEBUG { 22:57:03 DEBUG "metric": { 22:57:03 DEBUG { 22:57:03 DEBUG "metric": { 22:57:03 DEBUG "metric": { 22:57:03 DEBUG { 22:57:03 DEBUG { 22:57:03 DEBUG "metric": { 22:57:03 DEBUG { 22:57:03 DEBUG { 22:57:03 DEBUG "metric": { 22:57:03 DEBUG { 22:57:03 DEBUG "metric": { 22:57:03 DEBUG "metric": { 22:57:03 DEBUG "pod": "am-555f4db595-fz8sn" 22:57:03 DEBUG "resultType": "vector", 22:57:03 DEBUG "metric": { 22:57:03 DEBUG "metric": { 22:57:03 DEBUG "metric": { 22:57:03 DEBUG { 22:57:03 DEBUG "instance": "10.142.0.107:9100" 22:57:03 DEBUG { 22:57:03 DEBUG "result": [ 22:57:03 DEBUG "metric": { 22:57:03 DEBUG "metric": { 22:57:03 DEBUG "metric": { 22:57:03 DEBUG "pod": "am-555f4db595-fz8sn" 22:57:03 DEBUG "metric": { 22:57:03 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:57:03 DEBUG "instance": "10.142.0.107:9100" 22:57:03 DEBUG "metric": { 22:57:03 DEBUG "metric": { 22:57:03 DEBUG "pod": "ds-idrepo-2" 22:57:03 DEBUG "metric": { 22:57:03 DEBUG "metric": { 22:57:03 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:57:03 DEBUG "metric": { 22:57:03 DEBUG "instance": "10.142.0.107:9100" 22:57:03 DEBUG "pod": "ds-idrepo-2" 22:57:03 DEBUG }, 22:57:03 DEBUG "result": [ 22:57:03 DEBUG "pod": "am-555f4db595-fz8sn" 22:57:03 DEBUG "pod": "ds-idrepo-2" 22:57:03 DEBUG "pod": "ds-idrepo-2" 22:57:03 DEBUG "metric": { 22:57:03 DEBUG }, 22:57:03 DEBUG "metric": { 22:57:03 DEBUG { 22:57:03 DEBUG "pod": "am-555f4db595-fz8sn" 22:57:03 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:57:03 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:57:03 DEBUG }, 22:57:03 DEBUG "pod": "ds-idrepo-2" 22:57:03 DEBUG }, 22:57:03 DEBUG }, 22:57:03 DEBUG "pod": "ds-idrepo-2" 22:57:03 DEBUG "pod": "am-555f4db595-fz8sn" 22:57:03 DEBUG }, 22:57:03 DEBUG "pod": "am-555f4db595-fz8sn" 22:57:03 DEBUG "instance": "10.142.0.107:9100" 22:57:03 DEBUG }, 22:57:03 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:57:03 DEBUG }, 22:57:03 DEBUG }, 22:57:03 DEBUG "value": [ 22:57:03 DEBUG { 22:57:03 DEBUG }, 22:57:03 DEBUG }, 22:57:03 DEBUG }, 22:57:03 DEBUG "value": [ 22:57:03 DEBUG "value": [ 22:57:03 DEBUG "pod": "am-555f4db595-fz8sn" 22:57:03 DEBUG "metric": { 22:57:03 DEBUG }, 22:57:03 DEBUG }, 22:57:03 DEBUG }, 22:57:03 DEBUG "value": [ 22:57:03 DEBUG }, 22:57:03 DEBUG "value": [ 22:57:03 DEBUG "value": [ 22:57:03 DEBUG }, 22:57:03 DEBUG }, 22:57:03 DEBUG "value": [ 22:57:03 DEBUG }, 22:57:03 DEBUG }, 22:57:03 DEBUG "value": [ 22:57:03 DEBUG }, 22:57:03 DEBUG "value": [ 22:57:03 DEBUG "value": [ 22:57:03 DEBUG 22:57:03 DEBUG "metric": { 22:57:03 DEBUG "value": [ 22:57:03 DEBUG "value": [ 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG "instance": "10.142.0.107:9100" 22:57:03 DEBUG }, 22:57:03 DEBUG "pod": "ds-idrepo-2" 22:57:03 DEBUG "value": [ 22:57:03 DEBUG "value": [ 22:57:03 DEBUG "value": [ 22:57:03 DEBUG 22:57:03 DEBUG "value": [ 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG "value": [ 22:57:03 DEBUG "value": [ 22:57:03 DEBUG 22:57:03 DEBUG "value": [ 22:57:03 DEBUG "value": [ 22:57:03 DEBUG 22:57:03 DEBUG "value": [ 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG }, 22:57:03 DEBUG "value": [ 22:57:03 DEBUG }, 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG }, 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG "value": [ 22:57:03 DEBUG 22:57:03 DEBUG "value": [ 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG [...] 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG [...] 22:57:03 DEBUG "value": [ 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG [...] 22:57:03 DEBUG [...] 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG [...] 22:57:03 DEBUG [...] 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG [...] 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG [...] 22:57:03 DEBUG 22:57:03 DEBUG [...] 22:57:03 DEBUG [...] 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG [...] 22:57:03 DEBUG [...] 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG [...] 22:57:03 DEBUG [...] 22:57:03 DEBUG [...] 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG [...] 22:57:03 DEBUG [...] 22:57:03 DEBUG 22:57:03 DEBUG [...] 22:57:03 DEBUG [...] 22:57:03 DEBUG 22:57:03 DEBUG [...] 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG [...] 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG [...] 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG "pod": "ds-idrepo-0" 22:57:03 DEBUG [...] 22:57:03 DEBUG 22:57:03 DEBUG [...] 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG "pod": "am-555f4db595-kzxsp" 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG "pod": "am-555f4db595-kzxsp" 22:57:03 DEBUG [...] 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG }, 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG }, 22:57:03 DEBUG "pod": "am-555f4db595-kzxsp" 22:57:03 DEBUG "instance": "10.142.0.43:9100" 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG "pod": "ds-idrepo-0" 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG "pod": "am-555f4db595-kzxsp" 22:57:03 DEBUG 22:57:03 DEBUG "instance": "10.142.0.43:9100" 22:57:03 DEBUG "pod": "ds-cts-2" 22:57:03 DEBUG }, 22:57:03 DEBUG 22:57:03 DEBUG "pod": "am-555f4db595-kzxsp" 22:57:03 DEBUG "pod": "ds-idrepo-0" 22:57:03 DEBUG "value": [ 22:57:03 DEBUG "instance": "10.142.0.43:9100" 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG 22:57:03 DEBUG "pod": "am-555f4db595-kzxsp" 22:57:03 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:57:03 DEBUG "pod": "am-555f4db595-kzxsp" 22:57:03 DEBUG 22:57:03 DEBUG "value": [ 22:57:03 DEBUG }, 22:57:03 DEBUG }, 22:57:03 DEBUG "pod": "ds-idrepo-0" 22:57:03 DEBUG "pod": "am-555f4db595-kzxsp" 22:57:03 DEBUG }, 22:57:03 DEBUG "pod": "am-555f4db595-kzxsp" 22:57:03 DEBUG "instance": "10.142.0.43:9100" 22:57:03 DEBUG }, 22:57:03 DEBUG "pod": "am-555f4db595-kzxsp" 22:57:03 DEBUG }, 22:57:03 DEBUG }, 22:57:03 DEBUG "value": [ 22:57:03 DEBUG 22:57:03 DEBUG }, 22:57:03 DEBUG }, 22:57:03 DEBUG 1713304623, 22:57:03 DEBUG }, 22:57:03 DEBUG 22:57:03 DEBUG "pod": "am-555f4db595-kzxsp" 22:57:03 DEBUG 22:57:03 DEBUG }, 22:57:03 DEBUG }, 22:57:03 DEBUG }, 22:57:03 DEBUG "pod": "ds-idrepo-0" 22:57:03 DEBUG 1713304623, 22:57:03 DEBUG "value": [ 22:57:03 DEBUG "value": [ 22:57:03 DEBUG }, 22:57:03 DEBUG }, 22:57:03 DEBUG "value": [ 22:57:03 DEBUG }, 22:57:03 DEBUG }, 22:57:03 DEBUG "value": [ 22:57:03 DEBUG }, 22:57:03 DEBUG "value": [ 22:57:03 DEBUG "value": [ 22:57:03 DEBUG 1713304623, 22:57:03 DEBUG 22:57:03 DEBUG "value": [ 22:57:03 DEBUG "value": [ 22:57:03 DEBUG "0" 22:57:03 DEBUG "value": [ 22:57:03 DEBUG "instance": "10.142.0.43:9100" 22:57:03 DEBUG }, 22:57:03 DEBUG "pod": "ds-idrepo-0" 22:57:03 DEBUG "value": [ 22:57:03 DEBUG "value": [ 22:57:03 DEBUG "value": [ 22:57:03 DEBUG }, 22:57:03 DEBUG "3624165.5" 22:57:03 DEBUG 1713304623, 22:57:03 DEBUG 1713304623, 22:57:03 DEBUG "value": [ 22:57:03 DEBUG "value": [ 22:57:03 DEBUG 1713304623, 22:57:03 DEBUG "value": [ 22:57:03 DEBUG "value": [ 22:57:03 DEBUG 1713304623, 22:57:03 DEBUG "value": [ 22:57:03 DEBUG 1713304623, 22:57:03 DEBUG 1713304623, 22:57:03 DEBUG "0.027" 22:57:03 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:57:03 DEBUG 1713304623, 22:57:03 DEBUG 1713304623, 22:57:03 DEBUG ] 22:57:03 DEBUG 1713304623, 22:57:03 DEBUG }, 22:57:03 DEBUG "value": [ 22:57:03 DEBUG }, 22:57:03 DEBUG 1713304623, 22:57:03 DEBUG 1713304623, 22:57:03 DEBUG 1713304623, 22:57:03 DEBUG "value": [ 22:57:03 DEBUG ] 22:57:03 DEBUG "277328109.9658487" 22:57:03 DEBUG "0" 22:57:03 DEBUG 1713304623, 22:57:03 DEBUG 1713304623, 22:57:03 DEBUG "0" 22:57:03 DEBUG 1713304623, 22:57:03 DEBUG 1713304623, 22:57:03 DEBUG "483.2586242654685" 22:57:03 DEBUG 1713304623, 22:57:03 DEBUG "49050041.766666666" 22:57:03 DEBUG "0" 22:57:03 DEBUG ] 22:57:03 DEBUG }, 22:57:03 DEBUG "0" 22:57:03 DEBUG "0" 22:57:03 DEBUG } 22:57:03 DEBUG "49040468.53333333" 22:57:03 DEBUG "value": [ 22:57:03 DEBUG 1713304623, 22:57:03 DEBUG "value": [ 22:57:03 DEBUG "1999.8666666666666" 22:57:03 DEBUG "1693495296" 22:57:03 DEBUG "0" 22:57:03 DEBUG 1713304623, 22:57:03 DEBUG } 22:57:03 DEBUG ] 22:57:03 DEBUG ] 22:57:03 DEBUG "0" 22:57:03 DEBUG "2157.4666666666667" 22:57:03 DEBUG ] 22:57:03 DEBUG "2154.8333333333335" 22:57:03 DEBUG "0.0004999999999995453" 22:57:03 DEBUG ] 22:57:03 DEBUG "9.159944293955231" 22:57:03 DEBUG ] 22:57:03 DEBUG ] 22:57:03 DEBUG } 22:57:03 DEBUG "value": [ 22:57:03 DEBUG ] 22:57:03 DEBUG ] 22:57:03 DEBUG ] 22:57:03 DEBUG ] 22:57:03 DEBUG 1713304623, 22:57:03 DEBUG "2154.6666666666665" 22:57:03 DEBUG 1713304623, 22:57:03 DEBUG ] 22:57:03 DEBUG ] 22:57:03 DEBUG ] 22:57:03 DEBUG "0" 22:57:03 DEBUG ] 22:57:03 DEBUG } 22:57:03 DEBUG } 22:57:03 DEBUG ] 22:57:03 DEBUG } 22:57:03 DEBUG } 22:57:03 DEBUG ] 22:57:03 DEBUG ] 22:57:03 DEBUG } 22:57:03 DEBUG ] 22:57:03 DEBUG } 22:57:03 DEBUG } 22:57:03 DEBUG ] 22:57:03 DEBUG 1713304623, 22:57:03 DEBUG } 22:57:03 DEBUG } 22:57:03 DEBUG } 22:57:03 DEBUG } 22:57:03 DEBUG "19977970688" 22:57:03 DEBUG ] 22:57:03 DEBUG "0" 22:57:03 DEBUG } 22:57:03 DEBUG } 22:57:03 DEBUG } 22:57:03 DEBUG ] 22:57:03 DEBUG } 22:57:03 DEBUG ] 22:57:03 DEBUG ] 22:57:03 DEBUG ] 22:57:03 DEBUG ] 22:57:03 DEBUG } 22:57:03 DEBUG } 22:57:03 DEBUG ] 22:57:03 DEBUG } 22:57:03 DEBUG ] 22:57:03 DEBUG ] 22:57:03 DEBUG } 22:57:03 DEBUG "6.754845813526126" 22:57:03 DEBUG ] 22:57:03 DEBUG ] 22:57:03 DEBUG ] 22:57:03 DEBUG } 22:57:03 DEBUG ] 22:57:03 DEBUG ] 22:57:03 DEBUG } 22:57:03 DEBUG ] 22:57:03 DEBUG ] 22:57:03 DEBUG ] 22:57:03 DEBUG ] 22:57:03 DEBUG } 22:57:03 DEBUG } 22:57:03 DEBUG } 22:57:03 DEBUG } 22:57:03 DEBUG } 22:57:03 DEBUG } 22:57:03 DEBUG ] 22:57:03 DEBUG ] 22:57:03 DEBUG } 22:57:03 DEBUG ] 22:57:03 DEBUG } 22:57:03 DEBUG } 22:57:03 DEBUG } 22:57:03 DEBUG ] 22:57:03 DEBUG } 22:57:03 DEBUG } 22:57:03 DEBUG } 22:57:03 DEBUG } 22:57:03 DEBUG } 22:57:03 DEBUG } 22:57:03 DEBUG ] 22:57:03 DEBUG } 22:57:03 DEBUG } 22:57:03 DEBUG } 22:57:03 DEBUG } 22:57:03 DEBUG } 22:57:03 DEBUG } 22:57:03 DEBUG } 22:57:03 DEBUG } 22:57:03 DEBUG } 22:57:03 DEBUG } 22:57:03 DEBUG } 22:57:03 DEBUG } 22:57:03 DEBUG } 22:57:03 DEBUG } 22:57:03 DEBUG } 22:57:03 DEBUG } 22:57:03 DEBUG ] 22:57:03 DEBUG } 22:57:03 DEBUG } 22:57:03 DEBUG ] 22:57:03 DEBUG } 22:57:03 DEBUG ] 22:57:03 DEBUG } 22:57:03 DEBUG } 22:57:03 DEBUG ] 22:57:03 DEBUG } 22:57:03 DEBUG } 22:57:03 DEBUG } 22:57:03 DEBUG ] 22:57:03 DEBUG } 22:57:03 DEBUG } 22:57:03 DEBUG } 22:57:03 DEBUG } 22:57:03 DEBUG } 22:57:03 DEBUG } 22:57:03 DEBUG } 22:57:03 DEBUG } 22:57:03 DEBUG } 22:57:03 DEBUG } 22:57:03 DEBUG } 22:57:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304678" 22:57:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713304678" 22:57:58 ERROR [http_cmd]: ERROR 22:57:58 ERROR --- status code --- 22:57:58 ERROR http status code is 400 (expected [200, 401, 403, 404]) 22:57:58 ERROR --- http response --- 22:57:58 INFO [http_cmd]: http status code OK 22:57:58 DEBUG --- status code --- 22:57:58 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 22:57:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:57:58 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 22:57:58 DEBUG --- http response --- 22:57:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 22:58:00 INFO 22:58:00 INFO [loop_until]: kubectl --namespace=xlou top pods 22:58:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:58:00 INFO [loop_until]: OK (rc = 0) 22:58:00 DEBUG --- stdout --- 22:58:00 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 9Mi am-555f4db595-4lbss 13336m 8682Mi am-555f4db595-fz8sn 13358m 8564Mi am-555f4db595-kzxsp 13506m 8976Mi ds-cts-0 19657m 12853Mi ds-cts-1 18655m 13282Mi ds-cts-2 18735m 13435Mi ds-idrepo-0 5853m 5030Mi ds-idrepo-1 5747m 2908Mi ds-idrepo-2 5550m 3220Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 3m 1207Mi idm-b85f9d978-rl76f 4m 1226Mi lodemon-bc756c647-cr7x6 2m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-7c9b667974-9mr2s 6842m 1528Mi 22:58:00 DEBUG --- stderr --- 22:58:00 DEBUG 22:58:03 INFO 22:58:03 INFO [loop_until]: kubectl --namespace=xlou top node 22:58:03 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:58:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304683" 22:58:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713304683" 22:58:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304683" 22:58:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304683" 22:58:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304683" 22:58:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304683" 22:58:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304683" 22:58:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304683" 22:58:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304683" 22:58:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713304683" 22:58:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304683" 22:58:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304683" 22:58:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304683" 22:58:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304683" 22:58:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304683" 22:58:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713304683" 22:58:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304683" 22:58:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304683" 22:58:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304683" 22:58:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304683" 22:58:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304683" 22:58:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713304683" 22:58:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304683" 22:58:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304683" 22:58:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304683" 22:58:03 INFO [http_cmd]: http status code OK 22:58:03 INFO [loop_until]: OK (rc = 0) 22:58:03 INFO [http_cmd]: http status code OK 22:58:03 INFO [http_cmd]: http status code OK 22:58:03 INFO [http_cmd]: http status code OK 22:58:03 INFO [http_cmd]: http status code OK 22:58:03 INFO [http_cmd]: http status code OK 22:58:03 INFO [http_cmd]: http status code OK 22:58:03 INFO [http_cmd]: http status code OK 22:58:03 INFO [http_cmd]: http status code OK 22:58:03 INFO [http_cmd]: http status code OK 22:58:03 INFO [http_cmd]: http status code OK 22:58:03 INFO [http_cmd]: http status code OK 22:58:03 INFO [http_cmd]: http status code OK 22:58:03 INFO [http_cmd]: http status code OK 22:58:03 INFO [http_cmd]: http status code OK 22:58:03 INFO [http_cmd]: http status code OK 22:58:03 INFO [http_cmd]: http status code OK 22:58:03 DEBUG --- status code --- 22:58:03 INFO [http_cmd]: http status code OK 22:58:03 DEBUG --- stdout --- 22:58:03 DEBUG --- status code --- 22:58:03 DEBUG --- status code --- 22:58:03 DEBUG --- status code --- 22:58:03 DEBUG --- status code --- 22:58:03 DEBUG --- status code --- 22:58:03 DEBUG --- status code --- 22:58:03 DEBUG --- status code --- 22:58:03 DEBUG --- status code --- 22:58:03 DEBUG --- status code --- 22:58:03 DEBUG --- status code --- 22:58:03 DEBUG --- status code --- 22:58:03 INFO [http_cmd]: http status code OK 22:58:03 INFO [http_cmd]: http status code OK 22:58:03 DEBUG --- status code --- 22:58:03 INFO [http_cmd]: http status code OK 22:58:03 DEBUG --- status code --- 22:58:03 DEBUG --- status code --- 22:58:03 DEBUG --- status code --- 22:58:03 DEBUG --- status code --- 22:58:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:58:03 DEBUG --- status code --- 22:58:03 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 66m 0% 5083Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 18851m 63% 15905Mi 14% gke-xlou-cdm-default-pool-1b0769c8-nhlf 13048m 43% 10812Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 84m 0% 2643Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 5687m 19% 6576Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 20622m 69% 17373Mi 15% gke-xlou-cdm-default-pool-1b0769c8-xnmp 58m 0% 3136Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 13241m 44% 10477Mi 9% gke-xlou-cdm-ds-0f601751-hmm7 6104m 20% 5656Mi 5% gke-xlou-cdm-ds-0f601751-j488 18946m 63% 15502Mi 13% gke-xlou-cdm-ds-0f601751-mc5d 11636m 38% 3387Mi 3% gke-xlou-cdm-ds-0f601751-nj6q 14087m 47% 10842Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 6091m 20% 7356Mi 6% gke-xlou-cdm-frontend-a8771548-i82n 6953m 43% 6941Mi 11% 22:58:03 DEBUG --- stderr --- 22:58:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:58:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:58:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:58:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:58:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:58:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:58:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:58:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:58:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:58:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:58:03 DEBUG --- status code --- 22:58:03 DEBUG --- status code --- 22:58:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:58:03 DEBUG --- status code --- 22:58:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:58:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:58:03 DEBUG --- http response --- 22:58:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:58:03 DEBUG --- http response --- 22:58:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:58:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:58:03 DEBUG --- http response --- 22:58:03 DEBUG 22:58:03 DEBUG --- http response --- 22:58:03 DEBUG --- http response --- 22:58:03 DEBUG --- http response --- 22:58:03 DEBUG --- http response --- 22:58:03 DEBUG --- http response --- 22:58:03 DEBUG --- http response --- 22:58:03 DEBUG --- http response --- 22:58:03 DEBUG --- http response --- 22:58:03 DEBUG --- http response --- 22:58:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:58:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:58:03 DEBUG --- http response --- 22:58:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:58:03 DEBUG --- http response --- 22:58:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:58:03 DEBUG --- http response --- 22:58:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:58:03 INFO [http_cmd]: http status code OK 22:58:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:58:03 DEBUG --- http response --- 22:58:03 INFO [http_cmd]: http status code OK 22:58:03 INFO [http_cmd]: http status code OK 22:58:03 INFO [http_cmd]: http status code OK 22:58:03 DEBUG --- http response --- 22:58:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:58:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:58:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:58:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:58:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:58:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:58:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:58:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:58:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:58:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:58:03 DEBUG --- http response --- 22:58:03 DEBUG --- http response --- 22:58:03 DEBUG --- http response --- 22:58:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:58:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:58:03 DEBUG --- http response --- 22:58:03 DEBUG ----- output ----- 22:58:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:58:03 DEBUG --- status code --- 22:58:03 DEBUG ----- output ----- 22:58:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:58:03 DEBUG --- status code --- 22:58:03 DEBUG --- status code --- 22:58:03 DEBUG --- status code --- 22:58:03 DEBUG ----- output ----- 22:58:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:58:03 DEBUG ----- output ----- 22:58:03 DEBUG ----- output ----- 22:58:03 DEBUG ----- output ----- 22:58:03 DEBUG ----- output ----- 22:58:03 DEBUG ----- output ----- 22:58:03 DEBUG ----- output ----- 22:58:03 DEBUG ----- output ----- 22:58:03 DEBUG ----- output ----- 22:58:03 DEBUG ----- output ----- 22:58:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:58:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:58:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:58:03 DEBUG ----- output ----- 22:58:03 DEBUG ----- output ----- 22:58:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:58:03 DEBUG { 22:58:03 DEBUG ----- output ----- 22:58:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:58:03 DEBUG { 22:58:03 DEBUG ----- output ----- 22:58:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:58:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:58:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:58:03 DEBUG { 22:58:03 DEBUG ----- output ----- 22:58:03 DEBUG { 22:58:03 DEBUG { 22:58:03 DEBUG { 22:58:03 DEBUG { 22:58:03 DEBUG { 22:58:03 DEBUG { 22:58:03 DEBUG { 22:58:03 DEBUG { 22:58:03 DEBUG { 22:58:03 DEBUG ----- output ----- 22:58:03 DEBUG ----- output ----- 22:58:03 DEBUG ----- output ----- 22:58:03 DEBUG { 22:58:03 DEBUG { 22:58:03 DEBUG ----- output ----- 22:58:03 DEBUG "status": "success", 22:58:03 DEBUG { 22:58:03 DEBUG --- http response --- 22:58:03 DEBUG "status": "success", 22:58:03 DEBUG { 22:58:03 DEBUG --- http response --- 22:58:03 DEBUG --- http response --- 22:58:03 DEBUG --- http response --- 22:58:03 DEBUG "status": "success", 22:58:03 DEBUG { 22:58:03 DEBUG "status": "success", 22:58:03 DEBUG "status": "success", 22:58:03 DEBUG "status": "success", 22:58:03 DEBUG "status": "success", 22:58:03 DEBUG "status": "success", 22:58:03 DEBUG "status": "success", 22:58:03 DEBUG "status": "success", 22:58:03 DEBUG "data": { 22:58:03 DEBUG "status": "success", 22:58:03 DEBUG { 22:58:03 DEBUG { 22:58:03 DEBUG { 22:58:03 DEBUG "status": "success", 22:58:03 DEBUG "status": "success", 22:58:03 DEBUG { 22:58:03 DEBUG "data": { 22:58:03 DEBUG "status": "success", 22:58:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:58:03 DEBUG "data": { 22:58:03 DEBUG "status": "success", 22:58:03 DEBUG "data": { 22:58:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:58:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:58:03 DEBUG "data": { 22:58:03 DEBUG "status": "success", 22:58:03 DEBUG "data": { 22:58:03 DEBUG "data": { 22:58:03 DEBUG "data": { 22:58:03 DEBUG "data": { 22:58:03 DEBUG "data": { 22:58:03 DEBUG "data": { 22:58:03 DEBUG "resultType": "vector", 22:58:03 DEBUG "status": "success", 22:58:03 DEBUG "data": { 22:58:03 DEBUG "status": "success", 22:58:03 DEBUG "status": "success", 22:58:03 DEBUG "status": "success", 22:58:03 DEBUG "data": { 22:58:03 DEBUG "data": { 22:58:03 DEBUG "status": "success", 22:58:03 DEBUG "resultType": "vector", 22:58:03 DEBUG "data": { 22:58:03 DEBUG "resultType": "vector", 22:58:03 DEBUG "result": [ 22:58:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:58:03 DEBUG "resultType": "vector", 22:58:03 DEBUG ----- output ----- 22:58:03 DEBUG ----- output ----- 22:58:03 DEBUG "resultType": "vector", 22:58:03 DEBUG "data": { 22:58:03 DEBUG "resultType": "vector", 22:58:03 DEBUG "resultType": "vector", 22:58:03 DEBUG "resultType": "vector", 22:58:03 DEBUG "resultType": "vector", 22:58:03 DEBUG "resultType": "vector", 22:58:03 DEBUG "resultType": "vector", 22:58:03 DEBUG "result": [ 22:58:03 DEBUG "data": { 22:58:03 DEBUG "resultType": "vector", 22:58:03 DEBUG "data": { 22:58:03 DEBUG "data": { 22:58:03 DEBUG "data": { 22:58:03 DEBUG "resultType": "vector", 22:58:03 DEBUG "resultType": "vector", 22:58:03 DEBUG "data": { 22:58:03 DEBUG "result": [ 22:58:03 DEBUG ----- output ----- 22:58:03 DEBUG "resultType": "vector", 22:58:03 DEBUG { 22:58:03 DEBUG ----- output ----- 22:58:03 DEBUG "result": [ 22:58:03 DEBUG { 22:58:03 DEBUG { 22:58:03 DEBUG "result": [ 22:58:03 DEBUG "resultType": "vector", 22:58:03 DEBUG "result": [ 22:58:03 DEBUG "result": [ 22:58:03 DEBUG "result": [ 22:58:03 DEBUG "result": [ 22:58:03 DEBUG "result": [ 22:58:03 DEBUG "result": [ 22:58:03 DEBUG { 22:58:03 DEBUG "resultType": "vector", 22:58:03 DEBUG "result": [ 22:58:03 DEBUG "resultType": "vector", 22:58:03 DEBUG "resultType": "vector", 22:58:03 DEBUG "resultType": "vector", 22:58:03 DEBUG "result": [ 22:58:03 DEBUG "result": [ 22:58:03 DEBUG "resultType": "vector", 22:58:03 DEBUG { 22:58:03 DEBUG { 22:58:03 DEBUG "result": [ 22:58:03 DEBUG "metric": { 22:58:03 DEBUG { 22:58:03 DEBUG { 22:58:03 DEBUG "status": "success", 22:58:03 DEBUG "status": "success", 22:58:03 DEBUG { 22:58:03 DEBUG "result": [ 22:58:03 DEBUG { 22:58:03 DEBUG { 22:58:03 DEBUG { 22:58:03 DEBUG { 22:58:03 DEBUG { 22:58:03 DEBUG { 22:58:03 DEBUG "metric": { 22:58:03 DEBUG "result": [ 22:58:03 DEBUG { 22:58:03 DEBUG "result": [ 22:58:03 DEBUG "result": [ 22:58:03 DEBUG "result": [ 22:58:03 DEBUG { 22:58:03 DEBUG { 22:58:03 DEBUG "result": [ 22:58:03 DEBUG "metric": { 22:58:03 DEBUG "status": "success", 22:58:03 DEBUG { 22:58:03 DEBUG "pod": "ds-cts-1" 22:58:03 DEBUG "status": "success", 22:58:03 DEBUG "metric": { 22:58:03 DEBUG "data": { 22:58:03 DEBUG "data": { 22:58:03 DEBUG "metric": { 22:58:03 DEBUG { 22:58:03 DEBUG "metric": { 22:58:03 DEBUG "metric": { 22:58:03 DEBUG "metric": { 22:58:03 DEBUG "metric": { 22:58:03 DEBUG "metric": { 22:58:03 DEBUG "metric": { 22:58:03 DEBUG "pod": "ds-cts-1" 22:58:03 DEBUG { 22:58:03 DEBUG "metric": { 22:58:03 DEBUG { 22:58:03 DEBUG { 22:58:03 DEBUG { 22:58:03 DEBUG "metric": { 22:58:03 DEBUG "metric": { 22:58:03 DEBUG { 22:58:03 DEBUG "pod": "am-555f4db595-fz8sn" 22:58:03 DEBUG "data": { 22:58:03 DEBUG "metric": { 22:58:03 DEBUG }, 22:58:03 DEBUG "data": { 22:58:03 DEBUG "pod": "am-555f4db595-fz8sn" 22:58:03 DEBUG "resultType": "vector", 22:58:03 DEBUG "resultType": "vector", 22:58:03 DEBUG "pod": "ds-idrepo-2" 22:58:03 DEBUG "metric": { 22:58:03 DEBUG "pod": "ds-idrepo-2" 22:58:03 DEBUG "pod": "ds-cts-1" 22:58:03 DEBUG "pod": "am-555f4db595-fz8sn" 22:58:03 DEBUG "pod": "ds-idrepo-2" 22:58:03 DEBUG "instance": "10.142.0.107:9100" 22:58:03 DEBUG "pod": "ds-idrepo-2" 22:58:03 DEBUG }, 22:58:03 DEBUG "metric": { 22:58:03 DEBUG "instance": "10.142.0.107:9100" 22:58:03 DEBUG "metric": { 22:58:03 DEBUG "metric": { 22:58:03 DEBUG "metric": { 22:58:03 DEBUG "pod": "am-555f4db595-fz8sn" 22:58:03 DEBUG "pod": "ds-cts-1" 22:58:03 DEBUG "metric": { 22:58:03 DEBUG }, 22:58:03 DEBUG "resultType": "vector", 22:58:03 DEBUG "pod": "am-555f4db595-fz8sn" 22:58:03 DEBUG "value": [ 22:58:03 DEBUG "resultType": "vector", 22:58:03 DEBUG }, 22:58:03 DEBUG "result": [ 22:58:03 DEBUG "result": [ 22:58:03 DEBUG }, 22:58:03 DEBUG "pod": "ds-idrepo-2" 22:58:03 DEBUG }, 22:58:03 DEBUG }, 22:58:03 DEBUG }, 22:58:03 DEBUG }, 22:58:03 DEBUG }, 22:58:03 DEBUG }, 22:58:03 DEBUG "value": [ 22:58:03 DEBUG "instance": "10.142.0.107:9100" 22:58:03 DEBUG }, 22:58:03 DEBUG "pod": "am-555f4db595-fz8sn" 22:58:03 DEBUG "pod": "ds-idrepo-2" 22:58:03 DEBUG "instance": "10.142.0.107:9100" 22:58:03 DEBUG }, 22:58:03 DEBUG }, 22:58:03 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:58:03 DEBUG "value": [ 22:58:03 DEBUG "result": [ 22:58:03 DEBUG }, 22:58:03 DEBUG 22:58:03 DEBUG "result": [ 22:58:03 DEBUG "value": [ 22:58:03 DEBUG { 22:58:03 DEBUG { 22:58:03 DEBUG "value": [ 22:58:03 DEBUG }, 22:58:03 DEBUG "value": [ 22:58:03 DEBUG "value": [ 22:58:03 DEBUG "value": [ 22:58:03 DEBUG "value": [ 22:58:03 DEBUG "value": [ 22:58:03 DEBUG "value": [ 22:58:03 DEBUG 22:58:03 DEBUG }, 22:58:03 DEBUG "value": [ 22:58:03 DEBUG }, 22:58:03 DEBUG }, 22:58:03 DEBUG }, 22:58:03 DEBUG "value": [ 22:58:03 DEBUG "value": [ 22:58:03 DEBUG }, 22:58:03 DEBUG 22:58:03 DEBUG { 22:58:03 DEBUG "value": [ 22:58:03 DEBUG 22:58:03 DEBUG { 22:58:03 DEBUG 22:58:03 DEBUG "metric": { 22:58:03 DEBUG "metric": { 22:58:03 DEBUG 22:58:03 DEBUG "value": [ 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG "value": [ 22:58:03 DEBUG 22:58:03 DEBUG "value": [ 22:58:03 DEBUG "value": [ 22:58:03 DEBUG "value": [ 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG "value": [ 22:58:03 DEBUG 22:58:03 DEBUG "metric": { 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG "metric": { 22:58:03 DEBUG 22:58:03 DEBUG "pod": "am-555f4db595-fz8sn" 22:58:03 DEBUG "instance": "10.142.0.107:9100" 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:58:03 DEBUG 22:58:03 DEBUG [...] 22:58:03 DEBUG "pod": "ds-idrepo-2" 22:58:03 DEBUG 22:58:03 DEBUG }, 22:58:03 DEBUG }, 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG [...] 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG [...] 22:58:03 DEBUG }, 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG }, 22:58:03 DEBUG [...] 22:58:03 DEBUG "value": [ 22:58:03 DEBUG "value": [ 22:58:03 DEBUG [...] 22:58:03 DEBUG 22:58:03 DEBUG [...] 22:58:03 DEBUG [...] 22:58:03 DEBUG [...] 22:58:03 DEBUG [...] 22:58:03 DEBUG [...] 22:58:03 DEBUG [...] 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG [...] 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG [...] 22:58:03 DEBUG [...] 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG "value": [ 22:58:03 DEBUG [...] 22:58:03 DEBUG 22:58:03 DEBUG "value": [ 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG [...] 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG [...] 22:58:03 DEBUG 22:58:03 DEBUG [...] 22:58:03 DEBUG [...] 22:58:03 DEBUG [...] 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG [...] 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG "pod": "am-555f4db595-kzxsp" 22:58:03 DEBUG 22:58:03 DEBUG [...] 22:58:03 DEBUG }, 22:58:03 DEBUG 22:58:03 DEBUG "pod": "am-555f4db595-kzxsp" 22:58:03 DEBUG [...] 22:58:03 DEBUG [...] 22:58:03 DEBUG "pod": "ds-idrepo-0" 22:58:03 DEBUG 22:58:03 DEBUG "pod": "ds-idrepo-0" 22:58:03 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:58:03 DEBUG "pod": "am-555f4db595-kzxsp" 22:58:03 DEBUG "pod": "ds-idrepo-0" 22:58:03 DEBUG "instance": "10.142.0.43:9100" 22:58:03 DEBUG "pod": "ds-cts-2" 22:58:03 DEBUG }, 22:58:03 DEBUG 22:58:03 DEBUG "instance": "10.142.0.43:9100" 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG "pod": "am-555f4db595-kzxsp" 22:58:03 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:58:03 DEBUG 22:58:03 DEBUG }, 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG "value": [ 22:58:03 DEBUG 1713304683, 22:58:03 DEBUG }, 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG }, 22:58:03 DEBUG "pod": "ds-idrepo-0" 22:58:03 DEBUG }, 22:58:03 DEBUG }, 22:58:03 DEBUG }, 22:58:03 DEBUG }, 22:58:03 DEBUG }, 22:58:03 DEBUG }, 22:58:03 DEBUG "value": [ 22:58:03 DEBUG "instance": "10.142.0.43:9100" 22:58:03 DEBUG }, 22:58:03 DEBUG "pod": "am-555f4db595-kzxsp" 22:58:03 DEBUG "pod": "ds-idrepo-0" 22:58:03 DEBUG "instance": "10.142.0.43:9100" 22:58:03 DEBUG }, 22:58:03 DEBUG }, 22:58:03 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:58:03 DEBUG "value": [ 22:58:03 DEBUG "pod": "am-555f4db595-kzxsp" 22:58:03 DEBUG 22:58:03 DEBUG [...] 22:58:03 DEBUG "0.000026505513146735527" 22:58:03 DEBUG "value": [ 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG "value": [ 22:58:03 DEBUG }, 22:58:03 DEBUG "value": [ 22:58:03 DEBUG "value": [ 22:58:03 DEBUG "value": [ 22:58:03 DEBUG "value": [ 22:58:03 DEBUG "value": [ 22:58:03 DEBUG "value": [ 22:58:03 DEBUG 1713304683, 22:58:03 DEBUG }, 22:58:03 DEBUG "value": [ 22:58:03 DEBUG }, 22:58:03 DEBUG }, 22:58:03 DEBUG }, 22:58:03 DEBUG "value": [ 22:58:03 DEBUG "value": [ 22:58:03 DEBUG }, 22:58:03 DEBUG 1713304683, 22:58:03 DEBUG }, 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG ] 22:58:03 DEBUG } 22:58:03 DEBUG 1713304683, 22:58:03 DEBUG 22:58:03 DEBUG 1713304683, 22:58:03 DEBUG "value": [ 22:58:03 DEBUG 1713304683, 22:58:03 DEBUG 1713304683, 22:58:03 DEBUG 1713304683, 22:58:03 DEBUG 1713304683, 22:58:03 DEBUG 1713304683, 22:58:03 DEBUG 1713304683, 22:58:03 DEBUG "0" 22:58:03 DEBUG "value": [ 22:58:03 DEBUG 1713304683, 22:58:03 DEBUG "value": [ 22:58:03 DEBUG "value": [ 22:58:03 DEBUG "value": [ 22:58:03 DEBUG 1713304683, 22:58:03 DEBUG 1713304683, 22:58:03 DEBUG "value": [ 22:58:03 DEBUG "0" 22:58:03 DEBUG "value": [ 22:58:03 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:58:03 DEBUG 22:58:03 DEBUG 22:58:03 DEBUG ] 22:58:03 DEBUG "2167.866666666667" 22:58:03 DEBUG "instance": "10.142.0.43:9100" 22:58:03 DEBUG "0" 22:58:03 DEBUG 1713304683, 22:58:03 DEBUG "0" 22:58:03 DEBUG "0" 22:58:03 DEBUG "3634138.5" 22:58:03 DEBUG "0" 22:58:03 DEBUG "0" 22:58:03 DEBUG "0" 22:58:03 DEBUG ] 22:58:03 DEBUG 1713304683, 22:58:03 DEBUG "49223503.76666666" 22:58:03 DEBUG 1713304683, 22:58:03 DEBUG 1713304683, 22:58:03 DEBUG 1713304683, 22:58:03 DEBUG "1990.3999999999999" 22:58:03 DEBUG "0" 22:58:03 DEBUG 1713304683, 22:58:03 DEBUG ] 22:58:03 DEBUG 1713304683, 22:58:03 DEBUG }, 22:58:03 DEBUG 22:58:03 DEBUG "pod": "am-555f4db595-kzxsp" 22:58:03 DEBUG } 22:58:03 DEBUG ] 22:58:03 DEBUG }, 22:58:03 DEBUG ] 22:58:03 DEBUG "0" 22:58:03 DEBUG ] 22:58:03 DEBUG ] 22:58:03 DEBUG ] 22:58:03 DEBUG ] 22:58:03 DEBUG ] 22:58:03 DEBUG ] 22:58:03 DEBUG } 22:58:03 DEBUG "49214617.86666667" 22:58:03 DEBUG ] 22:58:03 DEBUG "2168.5666666666666" 22:58:03 DEBUG "0" 22:58:03 DEBUG "19978738688" 22:58:03 DEBUG ] 22:58:03 DEBUG ] 22:58:03 DEBUG "1693863936" 22:58:03 DEBUG } 22:58:03 DEBUG "0.027" 22:58:03 DEBUG "value": [ 22:58:03 DEBUG "pod": "ds-idrepo-0" 22:58:03 DEBUG }, 22:58:03 DEBUG } 22:58:03 DEBUG } 22:58:03 DEBUG "value": [ 22:58:03 DEBUG } 22:58:03 DEBUG ] 22:58:03 DEBUG } 22:58:03 DEBUG } 22:58:03 DEBUG } 22:58:03 DEBUG } 22:58:03 DEBUG } 22:58:03 DEBUG } 22:58:03 DEBUG ] 22:58:03 DEBUG ] 22:58:03 DEBUG } 22:58:03 DEBUG ] 22:58:03 DEBUG ] 22:58:03 DEBUG ] 22:58:03 DEBUG } 22:58:03 DEBUG } 22:58:03 DEBUG ] 22:58:03 DEBUG ] 22:58:03 DEBUG ] 22:58:03 DEBUG 1713304683, 22:58:03 DEBUG }, 22:58:03 DEBUG "value": [ 22:58:03 DEBUG ] 22:58:03 DEBUG 1713304683, 22:58:03 DEBUG ] 22:58:03 DEBUG } 22:58:03 DEBUG ] 22:58:03 DEBUG ] 22:58:03 DEBUG ] 22:58:03 DEBUG ] 22:58:03 DEBUG ] 22:58:03 DEBUG ] 22:58:03 DEBUG } 22:58:03 DEBUG } 22:58:03 DEBUG ] 22:58:03 DEBUG } 22:58:03 DEBUG } 22:58:03 DEBUG } 22:58:03 DEBUG ] 22:58:03 DEBUG ] 22:58:03 DEBUG } 22:58:03 DEBUG } 22:58:03 DEBUG } 22:58:03 DEBUG "6.799892727872301" 22:58:03 DEBUG "value": [ 22:58:03 DEBUG 1713304683, 22:58:03 DEBUG } 22:58:03 DEBUG "0.0005666666666665302" 22:58:03 DEBUG } 22:58:03 DEBUG ] 22:58:03 DEBUG } 22:58:03 DEBUG } 22:58:03 DEBUG } 22:58:03 DEBUG } 22:58:03 DEBUG } 22:58:03 DEBUG } 22:58:03 DEBUG } 22:58:03 DEBUG } 22:58:03 DEBUG } 22:58:03 DEBUG ] 22:58:03 DEBUG ] 22:58:03 DEBUG ] 22:58:03 DEBUG } 22:58:03 DEBUG } 22:58:03 DEBUG ] 22:58:03 DEBUG } 22:58:03 DEBUG ] 22:58:03 DEBUG ] 22:58:03 DEBUG 1713304683, 22:58:03 DEBUG "2166.5666666666666" 22:58:03 DEBUG } 22:58:03 DEBUG ] 22:58:03 DEBUG } 22:58:03 DEBUG } 22:58:03 DEBUG } 22:58:03 DEBUG } 22:58:03 DEBUG } 22:58:03 DEBUG } 22:58:03 DEBUG } 22:58:03 DEBUG } 22:58:03 DEBUG ] 22:58:03 DEBUG } 22:58:03 DEBUG } 22:58:03 DEBUG } 22:58:03 DEBUG } 22:58:03 DEBUG } 22:58:03 DEBUG } 22:58:03 DEBUG } 22:58:03 DEBUG } 22:58:03 DEBUG "0" 22:58:03 DEBUG ] 22:58:03 DEBUG } 22:58:03 DEBUG } 22:58:03 DEBUG } 22:58:03 DEBUG } 22:58:03 DEBUG } 22:58:03 DEBUG } 22:58:03 DEBUG } 22:58:03 DEBUG } 22:58:03 DEBUG ] 22:58:03 DEBUG ] 22:58:03 DEBUG } 22:58:03 DEBUG ] 22:58:03 DEBUG } 22:58:03 DEBUG } 22:58:03 DEBUG } 22:58:03 DEBUG ] 22:58:03 DEBUG } 22:58:03 DEBUG } 22:58:03 DEBUG ] 22:58:03 DEBUG } 22:58:03 DEBUG } 22:58:03 DEBUG } 22:58:03 DEBUG } 22:58:03 DEBUG } 22:58:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304738" 22:58:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713304738" 22:58:58 ERROR [http_cmd]: ERROR 22:58:58 INFO [http_cmd]: http status code OK 22:58:58 ERROR --- status code --- 22:58:58 DEBUG --- status code --- 22:58:58 ERROR http status code is 400 (expected [200, 401, 403, 404]) 22:58:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:58:58 ERROR --- http response --- 22:58:58 DEBUG --- http response --- 22:58:58 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 22:58:58 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 22:58:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 22:59:00 INFO 22:59:00 INFO [loop_until]: kubectl --namespace=xlou top pods 22:59:00 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:59:01 INFO [loop_until]: OK (rc = 0) 22:59:01 DEBUG --- stdout --- 22:59:01 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 9Mi am-555f4db595-4lbss 13116m 8677Mi am-555f4db595-fz8sn 12951m 8561Mi am-555f4db595-kzxsp 13099m 8970Mi ds-cts-0 19520m 12680Mi ds-cts-1 19333m 13156Mi ds-cts-2 19186m 13508Mi ds-idrepo-0 5809m 5030Mi ds-idrepo-1 5600m 2908Mi ds-idrepo-2 5379m 3220Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 4m 1208Mi idm-b85f9d978-rl76f 4m 1225Mi lodemon-bc756c647-cr7x6 2m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-7c9b667974-9mr2s 6735m 1526Mi 22:59:01 DEBUG --- stderr --- 22:59:01 DEBUG 22:59:03 INFO 22:59:03 INFO [loop_until]: kubectl --namespace=xlou top node 22:59:03 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 22:59:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304743" 22:59:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304743" 22:59:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304743" 22:59:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304743" 22:59:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304743" 22:59:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304743" 22:59:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304743" 22:59:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304743" 22:59:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713304743" 22:59:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304743" 22:59:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304743" 22:59:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304743" 22:59:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713304743" 22:59:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304743" 22:59:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304743" 22:59:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304743" 22:59:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304743" 22:59:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713304743" 22:59:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304743" 22:59:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304743" 22:59:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304743" 22:59:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304743" 22:59:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304743" 22:59:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304743" 22:59:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713304743" 22:59:03 INFO [loop_until]: OK (rc = 0) 22:59:03 INFO [http_cmd]: http status code OK 22:59:03 INFO [http_cmd]: http status code OK 22:59:03 INFO [http_cmd]: http status code OK 22:59:03 INFO [http_cmd]: http status code OK 22:59:03 INFO [http_cmd]: http status code OK 22:59:03 INFO [http_cmd]: http status code OK 22:59:03 INFO [http_cmd]: http status code OK 22:59:03 INFO [http_cmd]: http status code OK 22:59:03 INFO [http_cmd]: http status code OK 22:59:03 INFO [http_cmd]: http status code OK 22:59:03 INFO [http_cmd]: http status code OK 22:59:03 INFO [http_cmd]: http status code OK 22:59:03 INFO [http_cmd]: http status code OK 22:59:03 INFO [http_cmd]: http status code OK 22:59:03 INFO [http_cmd]: http status code OK 22:59:03 INFO [http_cmd]: http status code OK 22:59:03 DEBUG --- stdout --- 22:59:03 DEBUG --- status code --- 22:59:03 INFO [http_cmd]: http status code OK 22:59:03 INFO [http_cmd]: http status code OK 22:59:03 INFO [http_cmd]: http status code OK 22:59:03 DEBUG --- status code --- 22:59:03 INFO [http_cmd]: http status code OK 22:59:03 DEBUG --- status code --- 22:59:03 INFO [http_cmd]: http status code OK 22:59:03 DEBUG --- status code --- 22:59:03 INFO [http_cmd]: http status code OK 22:59:03 DEBUG --- status code --- 22:59:03 DEBUG --- status code --- 22:59:03 DEBUG --- status code --- 22:59:03 DEBUG --- status code --- 22:59:03 DEBUG --- status code --- 22:59:03 DEBUG --- status code --- 22:59:03 DEBUG --- status code --- 22:59:03 DEBUG --- status code --- 22:59:03 DEBUG --- status code --- 22:59:03 DEBUG --- status code --- 22:59:03 DEBUG --- status code --- 22:59:03 DEBUG --- status code --- 22:59:03 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 5083Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 18535m 62% 15596Mi 13% gke-xlou-cdm-default-pool-1b0769c8-nhlf 13229m 44% 10800Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 85m 0% 2649Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 5706m 19% 6565Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 20580m 68% 17178Mi 15% gke-xlou-cdm-default-pool-1b0769c8-xnmp 62m 0% 3127Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 13587m 45% 10469Mi 9% gke-xlou-cdm-ds-0f601751-hmm7 5999m 20% 5651Mi 5% gke-xlou-cdm-ds-0f601751-j488 18249m 61% 15416Mi 13% gke-xlou-cdm-ds-0f601751-mc5d 11197m 37% 3320Mi 2% gke-xlou-cdm-ds-0f601751-nj6q 13689m 45% 10834Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 5973m 20% 7357Mi 6% gke-xlou-cdm-frontend-a8771548-i82n 6907m 43% 6940Mi 11% 22:59:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:59:03 DEBUG --- status code --- 22:59:03 DEBUG --- status code --- 22:59:03 DEBUG --- status code --- 22:59:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:59:03 DEBUG --- status code --- 22:59:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:59:03 INFO [http_cmd]: http status code OK 22:59:03 DEBUG --- status code --- 22:59:03 INFO [http_cmd]: http status code OK 22:59:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:59:03 DEBUG --- status code --- 22:59:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:59:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:59:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:59:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:59:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:59:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:59:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:59:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:59:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:59:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:59:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:59:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:59:03 DEBUG --- stderr --- 22:59:03 DEBUG --- http response --- 22:59:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:59:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:59:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:59:03 DEBUG --- http response --- 22:59:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:59:03 DEBUG --- http response --- 22:59:03 DEBUG --- status code --- 22:59:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:59:03 DEBUG --- status code --- 22:59:03 DEBUG --- http response --- 22:59:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:59:03 DEBUG --- http response --- 22:59:03 DEBUG --- http response --- 22:59:03 DEBUG --- http response --- 22:59:03 DEBUG --- http response --- 22:59:03 DEBUG --- http response --- 22:59:03 DEBUG --- http response --- 22:59:03 DEBUG --- http response --- 22:59:03 DEBUG --- http response --- 22:59:03 DEBUG --- http response --- 22:59:03 DEBUG --- http response --- 22:59:03 DEBUG --- http response --- 22:59:03 DEBUG --- http response --- 22:59:03 DEBUG 22:59:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:59:03 DEBUG --- http response --- 22:59:03 DEBUG --- http response --- 22:59:03 DEBUG --- http response --- 22:59:03 INFO [http_cmd]: http status code OK 22:59:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:59:03 DEBUG --- http response --- 22:59:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 22:59:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:59:03 DEBUG --- http response --- 22:59:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:59:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 22:59:03 DEBUG --- http response --- 22:59:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:59:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:59:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:59:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 22:59:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:59:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:59:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:59:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:59:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:59:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:59:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:59:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:59:03 DEBUG ----- output ----- 22:59:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:59:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:59:03 DEBUG --- status code --- 22:59:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:59:03 DEBUG ----- output ----- 22:59:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:59:03 DEBUG ----- output ----- 22:59:03 DEBUG --- http response --- 22:59:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 22:59:03 DEBUG --- http response --- 22:59:03 DEBUG ----- output ----- 22:59:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 22:59:03 DEBUG ----- output ----- 22:59:03 DEBUG ----- output ----- 22:59:03 DEBUG ----- output ----- 22:59:03 DEBUG ----- output ----- 22:59:03 DEBUG ----- output ----- 22:59:03 DEBUG ----- output ----- 22:59:03 DEBUG ----- output ----- 22:59:03 DEBUG ----- output ----- 22:59:03 DEBUG ----- output ----- 22:59:03 DEBUG ----- output ----- 22:59:03 DEBUG ----- output ----- 22:59:03 DEBUG ----- output ----- 22:59:03 DEBUG { 22:59:03 DEBUG ----- output ----- 22:59:03 DEBUG ----- output ----- 22:59:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:59:03 DEBUG ----- output ----- 22:59:03 DEBUG { 22:59:03 DEBUG ----- output ----- 22:59:03 DEBUG { 22:59:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 22:59:03 DEBUG ----- output ----- 22:59:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:59:03 DEBUG { 22:59:03 DEBUG ----- output ----- 22:59:03 DEBUG { 22:59:03 DEBUG { 22:59:03 DEBUG { 22:59:03 DEBUG { 22:59:03 DEBUG { 22:59:03 DEBUG { 22:59:03 DEBUG { 22:59:03 DEBUG { 22:59:03 DEBUG { 22:59:03 DEBUG { 22:59:03 DEBUG { 22:59:03 DEBUG { 22:59:03 DEBUG "status": "success", 22:59:03 DEBUG { 22:59:03 DEBUG { 22:59:03 DEBUG --- http response --- 22:59:03 DEBUG { 22:59:03 DEBUG "status": "success", 22:59:03 DEBUG { 22:59:03 DEBUG "status": "success", 22:59:03 DEBUG ----- output ----- 22:59:03 DEBUG { 22:59:03 DEBUG ----- output ----- 22:59:03 DEBUG "status": "success", 22:59:03 DEBUG { 22:59:03 DEBUG "status": "success", 22:59:03 DEBUG "status": "success", 22:59:03 DEBUG "status": "success", 22:59:03 DEBUG "status": "success", 22:59:03 DEBUG "status": "success", 22:59:03 DEBUG "status": "success", 22:59:03 DEBUG "status": "success", 22:59:03 DEBUG "status": "success", 22:59:03 DEBUG "status": "success", 22:59:03 DEBUG "status": "success", 22:59:03 DEBUG "status": "success", 22:59:03 DEBUG "status": "success", 22:59:03 DEBUG "data": { 22:59:03 DEBUG "status": "success", 22:59:03 DEBUG "status": "success", 22:59:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 22:59:03 DEBUG "status": "success", 22:59:03 DEBUG "data": { 22:59:03 DEBUG "status": "success", 22:59:03 DEBUG "data": { 22:59:03 DEBUG { 22:59:03 DEBUG "status": "success", 22:59:03 DEBUG { 22:59:03 DEBUG "data": { 22:59:03 DEBUG "status": "success", 22:59:03 DEBUG "data": { 22:59:03 DEBUG "data": { 22:59:03 DEBUG "data": { 22:59:03 DEBUG "data": { 22:59:03 DEBUG "data": { 22:59:03 DEBUG "data": { 22:59:03 DEBUG "data": { 22:59:03 DEBUG "data": { 22:59:03 DEBUG "data": { 22:59:03 DEBUG "data": { 22:59:03 DEBUG "data": { 22:59:03 DEBUG "data": { 22:59:03 DEBUG "resultType": "vector", 22:59:03 DEBUG "data": { 22:59:03 DEBUG "data": { 22:59:03 DEBUG ----- output ----- 22:59:03 DEBUG "data": { 22:59:03 DEBUG "resultType": "vector", 22:59:03 DEBUG "data": { 22:59:03 DEBUG "resultType": "vector", 22:59:03 DEBUG "status": "success", 22:59:03 DEBUG "data": { 22:59:03 DEBUG "status": "success", 22:59:03 DEBUG "resultType": "vector", 22:59:03 DEBUG "data": { 22:59:03 DEBUG "resultType": "vector", 22:59:03 DEBUG "resultType": "vector", 22:59:03 DEBUG "resultType": "vector", 22:59:03 DEBUG "resultType": "vector", 22:59:03 DEBUG "resultType": "vector", 22:59:03 DEBUG "resultType": "vector", 22:59:03 DEBUG "resultType": "vector", 22:59:03 DEBUG "resultType": "vector", 22:59:03 DEBUG "resultType": "vector", 22:59:03 DEBUG "resultType": "vector", 22:59:03 DEBUG "resultType": "vector", 22:59:03 DEBUG "resultType": "vector", 22:59:03 DEBUG "result": [ 22:59:03 DEBUG "resultType": "vector", 22:59:03 DEBUG "resultType": "vector", 22:59:03 DEBUG { 22:59:03 DEBUG "resultType": "vector", 22:59:03 DEBUG "result": [ 22:59:03 DEBUG "resultType": "vector", 22:59:03 DEBUG "result": [ 22:59:03 DEBUG "data": { 22:59:03 DEBUG "resultType": "vector", 22:59:03 DEBUG "data": { 22:59:03 DEBUG "result": [ 22:59:03 DEBUG "resultType": "vector", 22:59:03 DEBUG "result": [ 22:59:03 DEBUG "result": [ 22:59:03 DEBUG "result": [ 22:59:03 DEBUG "result": [ 22:59:03 DEBUG "result": [ 22:59:03 DEBUG "result": [ 22:59:03 DEBUG "result": [ 22:59:03 DEBUG "result": [ 22:59:03 DEBUG "result": [ 22:59:03 DEBUG "result": [ 22:59:03 DEBUG "result": [ 22:59:03 DEBUG "result": [ 22:59:03 DEBUG { 22:59:03 DEBUG "result": [ 22:59:03 DEBUG "result": [ 22:59:03 DEBUG "status": "success", 22:59:03 DEBUG "result": [ 22:59:03 DEBUG { 22:59:03 DEBUG "result": [ 22:59:03 DEBUG { 22:59:03 DEBUG "resultType": "vector", 22:59:03 DEBUG "result": [ 22:59:03 DEBUG "resultType": "vector", 22:59:03 DEBUG { 22:59:03 DEBUG "result": [ 22:59:03 DEBUG { 22:59:03 DEBUG { 22:59:03 DEBUG { 22:59:03 DEBUG { 22:59:03 DEBUG { 22:59:03 DEBUG { 22:59:03 DEBUG { 22:59:03 DEBUG { 22:59:03 DEBUG { 22:59:03 DEBUG { 22:59:03 DEBUG { 22:59:03 DEBUG { 22:59:03 DEBUG "metric": { 22:59:03 DEBUG { 22:59:03 DEBUG { 22:59:03 DEBUG "data": { 22:59:03 DEBUG { 22:59:03 DEBUG "metric": { 22:59:03 DEBUG { 22:59:03 DEBUG "metric": { 22:59:03 DEBUG "result": [ 22:59:03 DEBUG { 22:59:03 DEBUG "result": [ 22:59:03 DEBUG "metric": { 22:59:03 DEBUG { 22:59:03 DEBUG "metric": { 22:59:03 DEBUG "metric": { 22:59:03 DEBUG "metric": { 22:59:03 DEBUG "metric": { 22:59:03 DEBUG "metric": { 22:59:03 DEBUG "metric": { 22:59:03 DEBUG "metric": { 22:59:03 DEBUG "metric": { 22:59:03 DEBUG "metric": { 22:59:03 DEBUG "metric": { 22:59:03 DEBUG "metric": { 22:59:03 DEBUG "metric": { 22:59:03 DEBUG "pod": "am-555f4db595-fz8sn" 22:59:03 DEBUG "metric": { 22:59:03 DEBUG "metric": { 22:59:03 DEBUG "resultType": "vector", 22:59:03 DEBUG "metric": { 22:59:03 DEBUG "instance": "10.142.0.107:9100" 22:59:03 DEBUG "metric": { 22:59:03 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:59:03 DEBUG { 22:59:03 DEBUG "metric": { 22:59:03 DEBUG { 22:59:03 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:59:03 DEBUG "metric": { 22:59:03 DEBUG "pod": "ds-idrepo-2" 22:59:03 DEBUG "pod": "am-555f4db595-fz8sn" 22:59:03 DEBUG "instance": "10.142.0.107:9100" 22:59:03 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:59:03 DEBUG "pod": "am-555f4db595-fz8sn" 22:59:03 DEBUG "pod": "ds-idrepo-2" 22:59:03 DEBUG "pod": "ds-idrepo-2" 22:59:03 DEBUG "pod": "am-555f4db595-fz8sn" 22:59:03 DEBUG "pod": "ds-idrepo-2" 22:59:03 DEBUG "pod": "ds-idrepo-2" 22:59:03 DEBUG "pod": "ds-idrepo-2" 22:59:03 DEBUG "pod": "am-555f4db595-fz8sn" 22:59:03 DEBUG }, 22:59:03 DEBUG "instance": "10.142.0.107:9100" 22:59:03 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:59:03 DEBUG "result": [ 22:59:03 DEBUG "instance": "10.142.0.107:9100" 22:59:03 DEBUG }, 22:59:03 DEBUG "pod": "am-555f4db595-fz8sn" 22:59:03 DEBUG }, 22:59:03 DEBUG "metric": { 22:59:03 DEBUG "pod": "am-555f4db595-fz8sn" 22:59:03 DEBUG "metric": { 22:59:03 DEBUG }, 22:59:03 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 22:59:03 DEBUG }, 22:59:03 DEBUG }, 22:59:03 DEBUG }, 22:59:03 DEBUG }, 22:59:03 DEBUG "value": [ 22:59:03 DEBUG }, 22:59:03 DEBUG }, 22:59:03 DEBUG }, 22:59:03 DEBUG }, 22:59:03 DEBUG }, 22:59:03 DEBUG }, 22:59:03 DEBUG }, 22:59:03 DEBUG "value": [ 22:59:03 DEBUG }, 22:59:03 DEBUG }, 22:59:03 DEBUG { 22:59:03 DEBUG }, 22:59:03 DEBUG "value": [ 22:59:03 DEBUG }, 22:59:03 DEBUG "value": [ 22:59:03 DEBUG "pod": "ds-idrepo-2" 22:59:03 DEBUG }, 22:59:03 DEBUG "instance": "10.142.0.107:9100" 22:59:03 DEBUG "value": [ 22:59:03 DEBUG }, 22:59:03 DEBUG "value": [ 22:59:03 DEBUG "value": [ 22:59:03 DEBUG "value": [ 22:59:03 DEBUG }, 22:59:03 DEBUG 22:59:03 DEBUG "value": [ 22:59:03 DEBUG "value": [ 22:59:03 DEBUG "value": [ 22:59:03 DEBUG "value": [ 22:59:03 DEBUG "value": [ 22:59:03 DEBUG "value": [ 22:59:03 DEBUG "value": [ 22:59:03 DEBUG 22:59:03 DEBUG "value": [ 22:59:03 DEBUG "value": [ 22:59:03 DEBUG "metric": { 22:59:03 DEBUG "value": [ 22:59:03 DEBUG 22:59:03 DEBUG "value": [ 22:59:03 DEBUG 22:59:03 DEBUG }, 22:59:03 DEBUG "value": [ 22:59:03 DEBUG }, 22:59:03 DEBUG 22:59:03 DEBUG "value": [ 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG "value": [ 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG "value": [ 22:59:03 DEBUG 22:59:03 DEBUG "value": [ 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG }, 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG [...] 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG [...] 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG "value": [ 22:59:03 DEBUG 22:59:03 DEBUG [...] 22:59:03 DEBUG 22:59:03 DEBUG [...] 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG [...] 22:59:03 DEBUG 22:59:03 DEBUG [...] 22:59:03 DEBUG [...] 22:59:03 DEBUG [...] 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG [...] 22:59:03 DEBUG [...] 22:59:03 DEBUG [...] 22:59:03 DEBUG [...] 22:59:03 DEBUG [...] 22:59:03 DEBUG [...] 22:59:03 DEBUG [...] 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG [...] 22:59:03 DEBUG 22:59:03 DEBUG [...] 22:59:03 DEBUG 22:59:03 DEBUG [...] 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG [...] 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG [...] 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG [...] 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG [...] 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG [...] 22:59:03 DEBUG 22:59:03 DEBUG [...] 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG "pod": "am-555f4db595-kzxsp" 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG "pod": "ds-cts-2" 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG "pod": "ds-cts-2" 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG }, 22:59:03 DEBUG 22:59:03 DEBUG [...] 22:59:03 DEBUG 22:59:03 DEBUG "instance": "10.142.0.43:9100" 22:59:03 DEBUG }, 22:59:03 DEBUG "pod": "ds-cts-2" 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG "pod": "ds-cts-2" 22:59:03 DEBUG 22:59:03 DEBUG 22:59:03 DEBUG "pod": "am-555f4db595-kzxsp" 22:59:03 DEBUG }, 22:59:03 DEBUG 22:59:03 DEBUG }, 22:59:03 DEBUG "pod": "ds-idrepo-0" 22:59:03 DEBUG "pod": "ds-idrepo-0" 22:59:03 DEBUG "pod": "am-555f4db595-kzxsp" 22:59:03 DEBUG "pod": "ds-cts-2" 22:59:03 DEBUG "pod": "ds-idrepo-0" 22:59:03 DEBUG "pod": "ds-idrepo-0" 22:59:03 DEBUG "pod": "am-555f4db595-kzxsp" 22:59:03 DEBUG 22:59:03 DEBUG "value": [ 22:59:03 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:59:03 DEBUG 22:59:03 DEBUG "instance": "10.142.0.43:9100" 22:59:03 DEBUG 22:59:03 DEBUG "value": [ 22:59:03 DEBUG }, 22:59:03 DEBUG 22:59:03 DEBUG "pod": "am-555f4db595-kzxsp" 22:59:03 DEBUG 22:59:03 DEBUG }, 22:59:03 DEBUG "pod": "ds-idrepo-0" 22:59:03 DEBUG "instance": "10.142.0.43:9100" 22:59:03 DEBUG }, 22:59:03 DEBUG "value": [ 22:59:03 DEBUG "pod": "am-555f4db595-kzxsp" 22:59:03 DEBUG "value": [ 22:59:03 DEBUG }, 22:59:03 DEBUG }, 22:59:03 DEBUG }, 22:59:03 DEBUG }, 22:59:03 DEBUG }, 22:59:03 DEBUG }, 22:59:03 DEBUG }, 22:59:03 DEBUG "instance": "10.142.0.43:9100" 22:59:03 DEBUG 1713304743, 22:59:03 DEBUG }, 22:59:03 DEBUG 22:59:03 DEBUG }, 22:59:03 DEBUG "pod": "am-555f4db595-kzxsp" 22:59:03 DEBUG 1713304743, 22:59:03 DEBUG "value": [ 22:59:03 DEBUG "pod": "ds-idrepo-0" 22:59:03 DEBUG }, 22:59:03 DEBUG "instance": "10.142.0.43:9100" 22:59:03 DEBUG "value": [ 22:59:03 DEBUG }, 22:59:03 DEBUG }, 22:59:03 DEBUG "value": [ 22:59:03 DEBUG 1713304743, 22:59:03 DEBUG }, 22:59:03 DEBUG 1713304743, 22:59:03 DEBUG "value": [ 22:59:03 DEBUG "value": [ 22:59:03 DEBUG "value": [ 22:59:03 DEBUG "value": [ 22:59:03 DEBUG "value": [ 22:59:03 DEBUG 1713304743, 22:59:03 DEBUG "value": [ 22:59:03 DEBUG }, 22:59:03 DEBUG "2161.9" 22:59:03 DEBUG "value": [ 22:59:03 DEBUG 22:59:03 DEBUG "value": [ 22:59:03 DEBUG }, 22:59:03 DEBUG "49245356.53333333" 22:59:03 DEBUG 1713304743, 22:59:03 DEBUG }, 22:59:03 DEBUG "value": [ 22:59:03 DEBUG }, 22:59:03 DEBUG 1713304743, 22:59:03 DEBUG "value": [ 22:59:03 DEBUG "value": [ 22:59:03 DEBUG 1713304743, 22:59:03 DEBUG "69347719.99497299" 22:59:03 DEBUG "value": [ 22:59:03 DEBUG "57167177.57948975" 22:59:03 DEBUG 1713304743, 22:59:03 DEBUG 1713304743, 22:59:03 DEBUG 1713304743, 22:59:03 DEBUG 1713304743, 22:59:03 DEBUG "0" 22:59:03 DEBUG "value": [ 22:59:03 DEBUG 1713304743, 22:59:03 DEBUG "value": [ 22:59:03 DEBUG ] 22:59:03 DEBUG 1713304743, 22:59:03 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 22:59:03 DEBUG }, 22:59:03 DEBUG "value": [ 22:59:03 DEBUG ] 22:59:03 DEBUG "19.220304668845312" 22:59:03 DEBUG "value": [ 22:59:03 DEBUG 1713304743, 22:59:03 DEBUG "value": [ 22:59:03 DEBUG "14726450.672363956" 22:59:03 DEBUG 1713304743, 22:59:03 DEBUG 1713304743, 22:59:03 DEBUG "0" 22:59:03 DEBUG ] 22:59:03 DEBUG 1713304743, 22:59:03 DEBUG ] 22:59:03 DEBUG "0" 22:59:03 DEBUG "0" 22:59:03 DEBUG "0.027" 22:59:03 DEBUG "0" 22:59:03 DEBUG ] 22:59:03 DEBUG 1713304743, 22:59:03 DEBUG "0" 22:59:03 DEBUG 1713304743, 22:59:03 DEBUG } 22:59:03 DEBUG "1694412800" 22:59:03 DEBUG "value": [ 22:59:03 DEBUG 1713304743, 22:59:03 DEBUG } 22:59:03 DEBUG 1713304743, 22:59:03 DEBUG ] 22:59:03 DEBUG 1713304743, 22:59:03 DEBUG "2166.3" 22:59:03 DEBUG 1713304743, 22:59:03 DEBUG ] 22:59:03 DEBUG "0" 22:59:03 DEBUG "0" 22:59:03 DEBUG ] 22:59:03 DEBUG } 22:59:03 DEBUG "3644546" 22:59:03 DEBUG } 22:59:03 DEBUG ] 22:59:03 DEBUG ] 22:59:03 DEBUG ] 22:59:03 DEBUG ] 22:59:03 DEBUG } 22:59:03 DEBUG "1987.6333333333332" 22:59:03 DEBUG ] 22:59:03 DEBUG "19979658240" 22:59:03 DEBUG ] 22:59:03 DEBUG ] 22:59:03 DEBUG 1713304743, 22:59:03 DEBUG "2162.4666666666667" 22:59:03 DEBUG ] 22:59:03 DEBUG "49254513.966666654" 22:59:03 DEBUG } 22:59:03 DEBUG "0" 22:59:03 DEBUG ] 22:59:03 DEBUG "0.00046666666666700016" 22:59:03 DEBUG } 22:59:03 DEBUG ] 22:59:03 DEBUG ] 22:59:03 DEBUG } 22:59:03 DEBUG ] 22:59:03 DEBUG ] 22:59:03 DEBUG ] 22:59:03 DEBUG } 22:59:03 DEBUG } 22:59:03 DEBUG } 22:59:03 DEBUG } 22:59:03 DEBUG ] 22:59:03 DEBUG ] 22:59:03 DEBUG } 22:59:03 DEBUG ] 22:59:03 DEBUG } 22:59:03 DEBUG } 22:59:03 DEBUG "6.763841341469098" 22:59:03 DEBUG ] 22:59:03 DEBUG } 22:59:03 DEBUG ] 22:59:03 DEBUG ] 22:59:03 DEBUG ] 22:59:03 DEBUG } 22:59:03 DEBUG ] 22:59:03 DEBUG ] 22:59:03 DEBUG } 22:59:03 DEBUG } 22:59:03 DEBUG ] 22:59:03 DEBUG } 22:59:03 DEBUG } 22:59:03 DEBUG } 22:59:03 DEBUG ] 22:59:03 DEBUG ] 22:59:03 DEBUG ] 22:59:03 DEBUG ] 22:59:03 DEBUG } 22:59:03 DEBUG } 22:59:03 DEBUG ] 22:59:03 DEBUG } 22:59:03 DEBUG } 22:59:03 DEBUG ] 22:59:03 DEBUG ] 22:59:03 DEBUG } 22:59:03 DEBUG } 22:59:03 DEBUG } 22:59:03 DEBUG } 22:59:03 DEBUG } 22:59:03 DEBUG ] 22:59:03 DEBUG } 22:59:03 DEBUG } 22:59:03 DEBUG ] 22:59:03 DEBUG ] 22:59:03 DEBUG } 22:59:03 DEBUG } 22:59:03 DEBUG ] 22:59:03 DEBUG } 22:59:03 DEBUG } 22:59:03 DEBUG } 22:59:03 DEBUG } 22:59:03 DEBUG } 22:59:03 DEBUG } 22:59:03 DEBUG ] 22:59:03 DEBUG } 22:59:03 DEBUG ] 22:59:03 DEBUG } 22:59:03 DEBUG } 22:59:03 DEBUG ] 22:59:03 DEBUG ] 22:59:03 DEBUG } 22:59:03 DEBUG ] 22:59:03 DEBUG } 22:59:03 DEBUG ] 22:59:03 DEBUG } 22:59:03 DEBUG } 22:59:03 DEBUG } 22:59:03 DEBUG } 22:59:03 DEBUG } 22:59:03 DEBUG } 22:59:03 DEBUG } 22:59:03 DEBUG } 22:59:03 DEBUG } 22:59:03 DEBUG } 22:59:03 DEBUG } 22:59:03 DEBUG } 22:59:03 DEBUG } 22:59:03 DEBUG ] 22:59:03 DEBUG } 22:59:03 DEBUG } 22:59:03 DEBUG } 22:59:03 DEBUG } 22:59:03 DEBUG } 22:59:03 DEBUG } 22:59:03 DEBUG } 22:59:03 DEBUG } 22:59:03 DEBUG } 22:59:03 DEBUG } 22:59:03 DEBUG } 22:59:03 DEBUG } 22:59:03 DEBUG } 22:59:03 DEBUG } 22:59:03 DEBUG } 22:59:03 DEBUG } 22:59:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713304798" 22:59:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304798" 22:59:58 ERROR [http_cmd]: ERROR 22:59:58 ERROR --- status code --- 22:59:58 ERROR http status code is 400 (expected [200, 401, 403, 404]) 22:59:58 ERROR --- http response --- 22:59:58 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 22:59:58 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 22:59:58 INFO [http_cmd]: http status code OK 22:59:58 DEBUG --- status code --- 22:59:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 22:59:58 DEBUG --- http response --- 22:59:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 23:00:01 INFO 23:00:01 INFO [loop_until]: kubectl --namespace=xlou top pods 23:00:01 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 23:00:01 INFO [loop_until]: OK (rc = 0) 23:00:01 DEBUG --- stdout --- 23:00:01 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 9Mi am-555f4db595-4lbss 13376m 8682Mi am-555f4db595-fz8sn 12775m 8563Mi am-555f4db595-kzxsp 13739m 8978Mi ds-cts-0 19431m 12999Mi ds-cts-1 18449m 12832Mi ds-cts-2 18572m 12967Mi ds-idrepo-0 5708m 5030Mi ds-idrepo-1 5631m 2908Mi ds-idrepo-2 5565m 3220Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 3m 1207Mi idm-b85f9d978-rl76f 4m 1226Mi lodemon-bc756c647-cr7x6 2m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-7c9b667974-9mr2s 6634m 1529Mi 23:00:01 DEBUG --- stderr --- 23:00:01 DEBUG 23:00:03 INFO 23:00:03 INFO [loop_until]: kubectl --namespace=xlou top node 23:00:03 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 23:00:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304803" 23:00:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304803" 23:00:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304803" 23:00:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304803" 23:00:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304803" 23:00:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713304803" 23:00:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304803" 23:00:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304803" 23:00:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304803" 23:00:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713304803" 23:00:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304803" 23:00:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304803" 23:00:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713304803" 23:00:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713304803" 23:00:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304803" 23:00:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304803" 23:00:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304803" 23:00:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304803" 23:00:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304803" 23:00:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304803" 23:00:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304803" 23:00:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304803" 23:00:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304803" 23:00:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304803" 23:00:03 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304803" 23:00:03 INFO [loop_until]: OK (rc = 0) 23:00:03 INFO [http_cmd]: http status code OK 23:00:03 INFO [http_cmd]: http status code OK 23:00:03 INFO [http_cmd]: http status code OK 23:00:03 INFO [http_cmd]: http status code OK 23:00:03 INFO [http_cmd]: http status code OK 23:00:03 INFO [http_cmd]: http status code OK 23:00:03 INFO [http_cmd]: http status code OK 23:00:03 INFO [http_cmd]: http status code OK 23:00:03 INFO [http_cmd]: http status code OK 23:00:03 INFO [http_cmd]: http status code OK 23:00:03 INFO [http_cmd]: http status code OK 23:00:03 INFO [http_cmd]: http status code OK 23:00:03 INFO [http_cmd]: http status code OK 23:00:03 INFO [http_cmd]: http status code OK 23:00:03 INFO [http_cmd]: http status code OK 23:00:03 INFO [http_cmd]: http status code OK 23:00:03 INFO [http_cmd]: http status code OK 23:00:03 INFO [http_cmd]: http status code OK 23:00:03 DEBUG --- stdout --- 23:00:03 INFO [http_cmd]: http status code OK 23:00:03 INFO [http_cmd]: http status code OK 23:00:03 DEBUG --- status code --- 23:00:03 DEBUG --- status code --- 23:00:03 DEBUG --- status code --- 23:00:03 DEBUG --- status code --- 23:00:03 DEBUG --- status code --- 23:00:03 DEBUG --- status code --- 23:00:03 DEBUG --- status code --- 23:00:03 DEBUG --- status code --- 23:00:03 DEBUG --- status code --- 23:00:03 DEBUG --- status code --- 23:00:03 INFO [http_cmd]: http status code OK 23:00:03 DEBUG --- status code --- 23:00:03 DEBUG --- status code --- 23:00:03 INFO [http_cmd]: http status code OK 23:00:03 DEBUG --- status code --- 23:00:03 DEBUG --- status code --- 23:00:03 DEBUG --- status code --- 23:00:03 DEBUG --- status code --- 23:00:03 DEBUG --- status code --- 23:00:03 DEBUG --- status code --- 23:00:03 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 61m 0% 5097Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 18512m 62% 15377Mi 13% gke-xlou-cdm-default-pool-1b0769c8-nhlf 13075m 43% 10834Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 84m 0% 2650Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 5678m 19% 6567Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 20563m 68% 17426Mi 15% gke-xlou-cdm-default-pool-1b0769c8-xnmp 60m 0% 3137Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 13488m 45% 10482Mi 9% gke-xlou-cdm-ds-0f601751-hmm7 5980m 20% 5644Mi 5% gke-xlou-cdm-ds-0f601751-j488 18892m 63% 15005Mi 13% gke-xlou-cdm-ds-0f601751-mc5d 11679m 39% 3410Mi 3% gke-xlou-cdm-ds-0f601751-nj6q 14018m 46% 10849Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 6094m 20% 7345Mi 6% gke-xlou-cdm-frontend-a8771548-i82n 6969m 43% 6941Mi 11% 23:00:03 DEBUG --- status code --- 23:00:03 DEBUG --- status code --- 23:00:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:00:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:00:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:00:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:00:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:00:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:00:03 INFO [http_cmd]: http status code OK 23:00:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:00:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:00:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:00:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:00:03 DEBUG --- status code --- 23:00:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:00:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:00:03 DEBUG --- status code --- 23:00:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:00:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:00:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:00:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:00:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:00:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:00:03 DEBUG --- stderr --- 23:00:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:00:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:00:03 DEBUG --- http response --- 23:00:03 DEBUG --- http response --- 23:00:03 DEBUG --- http response --- 23:00:03 DEBUG --- http response --- 23:00:03 DEBUG --- http response --- 23:00:03 DEBUG --- http response --- 23:00:03 DEBUG --- status code --- 23:00:03 DEBUG --- http response --- 23:00:03 DEBUG --- http response --- 23:00:03 DEBUG --- http response --- 23:00:03 DEBUG --- http response --- 23:00:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:00:03 INFO [http_cmd]: http status code OK 23:00:03 INFO [http_cmd]: http status code OK 23:00:03 DEBUG --- http response --- 23:00:03 DEBUG --- http response --- 23:00:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:00:03 DEBUG --- http response --- 23:00:03 DEBUG --- http response --- 23:00:03 DEBUG --- http response --- 23:00:03 DEBUG --- http response --- 23:00:03 DEBUG --- http response --- 23:00:03 DEBUG --- http response --- 23:00:03 DEBUG 23:00:03 DEBUG --- http response --- 23:00:03 DEBUG --- http response --- 23:00:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:00:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 23:00:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:00:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:00:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:00:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 23:00:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:00:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 23:00:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 23:00:03 DEBUG ----- output ----- 23:00:03 DEBUG --- http response --- 23:00:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 23:00:03 DEBUG --- status code --- 23:00:03 DEBUG --- status code --- 23:00:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:00:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 23:00:03 DEBUG --- http response --- 23:00:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:00:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:00:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:00:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:00:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:00:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:00:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:00:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:00:03 DEBUG ----- output ----- 23:00:03 DEBUG ----- output ----- 23:00:03 DEBUG ----- output ----- 23:00:03 DEBUG ----- output ----- 23:00:03 DEBUG ----- output ----- 23:00:03 DEBUG ----- output ----- 23:00:03 DEBUG --- http response --- 23:00:03 DEBUG ----- output ----- 23:00:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 23:00:03 DEBUG { 23:00:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:00:03 DEBUG ----- output ----- 23:00:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:00:03 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:00:03 DEBUG ----- output ----- 23:00:03 DEBUG ----- output ----- 23:00:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:00:03 DEBUG ----- output ----- 23:00:03 DEBUG ----- output ----- 23:00:03 DEBUG ----- output ----- 23:00:03 DEBUG ----- output ----- 23:00:03 DEBUG ----- output ----- 23:00:03 DEBUG ----- output ----- 23:00:03 DEBUG ----- output ----- 23:00:03 DEBUG ----- output ----- 23:00:03 DEBUG { 23:00:03 DEBUG { 23:00:03 DEBUG { 23:00:03 DEBUG { 23:00:03 DEBUG { 23:00:03 DEBUG { 23:00:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:00:03 DEBUG { 23:00:03 DEBUG ----- output ----- 23:00:03 DEBUG "status": "success", 23:00:03 DEBUG ----- output ----- 23:00:03 DEBUG { 23:00:03 DEBUG --- http response --- 23:00:03 DEBUG --- http response --- 23:00:03 DEBUG { 23:00:03 DEBUG { 23:00:03 DEBUG ----- output ----- 23:00:03 DEBUG { 23:00:03 DEBUG { 23:00:03 DEBUG { 23:00:03 DEBUG { 23:00:03 DEBUG { 23:00:03 DEBUG { 23:00:03 DEBUG { 23:00:03 DEBUG { 23:00:03 DEBUG "status": "success", 23:00:03 DEBUG "status": "success", 23:00:03 DEBUG "status": "success", 23:00:03 DEBUG "status": "success", 23:00:03 DEBUG "status": "success", 23:00:03 DEBUG "status": "success", 23:00:03 DEBUG ----- output ----- 23:00:03 DEBUG "status": "success", 23:00:03 DEBUG { 23:00:03 DEBUG "data": { 23:00:03 DEBUG { 23:00:03 DEBUG "status": "success", 23:00:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:00:03 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 23:00:03 DEBUG "status": "success", 23:00:03 DEBUG "status": "success", 23:00:03 DEBUG { 23:00:03 DEBUG "status": "success", 23:00:03 DEBUG "status": "success", 23:00:03 DEBUG "status": "success", 23:00:03 DEBUG "status": "success", 23:00:03 DEBUG "status": "success", 23:00:03 DEBUG "status": "success", 23:00:03 DEBUG "status": "success", 23:00:03 DEBUG "status": "success", 23:00:03 DEBUG "data": { 23:00:03 DEBUG "data": { 23:00:03 DEBUG "data": { 23:00:03 DEBUG "data": { 23:00:03 DEBUG "data": { 23:00:03 DEBUG "data": { 23:00:03 DEBUG { 23:00:03 DEBUG "data": { 23:00:03 DEBUG "status": "success", 23:00:03 DEBUG "resultType": "vector", 23:00:03 DEBUG "status": "success", 23:00:03 DEBUG "data": { 23:00:03 DEBUG ----- output ----- 23:00:03 DEBUG ----- output ----- 23:00:03 DEBUG "data": { 23:00:03 DEBUG "data": { 23:00:03 DEBUG "status": "success", 23:00:03 DEBUG "data": { 23:00:03 DEBUG "data": { 23:00:03 DEBUG "data": { 23:00:03 DEBUG "data": { 23:00:03 DEBUG "data": { 23:00:03 DEBUG "data": { 23:00:03 DEBUG "data": { 23:00:03 DEBUG "data": { 23:00:03 DEBUG "resultType": "vector", 23:00:03 DEBUG "resultType": "vector", 23:00:03 DEBUG "resultType": "vector", 23:00:03 DEBUG "resultType": "vector", 23:00:03 DEBUG "resultType": "vector", 23:00:03 DEBUG "resultType": "vector", 23:00:03 DEBUG "status": "success", 23:00:03 DEBUG "resultType": "vector", 23:00:03 DEBUG "data": { 23:00:03 DEBUG "result": [ 23:00:03 DEBUG "data": { 23:00:03 DEBUG "resultType": "vector", 23:00:03 DEBUG { 23:00:03 DEBUG { 23:00:03 DEBUG "resultType": "vector", 23:00:03 DEBUG "resultType": "vector", 23:00:03 DEBUG "data": { 23:00:03 DEBUG "resultType": "vector", 23:00:03 DEBUG "resultType": "vector", 23:00:03 DEBUG "resultType": "vector", 23:00:03 DEBUG "resultType": "vector", 23:00:03 DEBUG "resultType": "vector", 23:00:03 DEBUG "resultType": "vector", 23:00:03 DEBUG "resultType": "vector", 23:00:03 DEBUG "resultType": "vector", 23:00:03 DEBUG "result": [ 23:00:03 DEBUG "result": [ 23:00:03 DEBUG "result": [ 23:00:03 DEBUG "result": [ 23:00:03 DEBUG "result": [ 23:00:03 DEBUG "result": [ 23:00:03 DEBUG "data": { 23:00:03 DEBUG "result": [ 23:00:03 DEBUG "resultType": "vector", 23:00:03 DEBUG { 23:00:03 DEBUG "resultType": "vector", 23:00:03 DEBUG "result": [ 23:00:03 DEBUG "status": "success", 23:00:03 DEBUG "status": "success", 23:00:03 DEBUG "result": [ 23:00:03 DEBUG "result": [ 23:00:03 DEBUG "resultType": "vector", 23:00:03 DEBUG "result": [ 23:00:03 DEBUG "result": [ 23:00:03 DEBUG "result": [ 23:00:03 DEBUG "result": [ 23:00:03 DEBUG "result": [ 23:00:03 DEBUG "result": [ 23:00:03 DEBUG "result": [ 23:00:03 DEBUG "result": [ 23:00:03 DEBUG { 23:00:03 DEBUG { 23:00:03 DEBUG { 23:00:03 DEBUG { 23:00:03 DEBUG { 23:00:03 DEBUG { 23:00:03 DEBUG "resultType": "vector", 23:00:03 DEBUG { 23:00:03 DEBUG "result": [ 23:00:03 DEBUG { 23:00:03 DEBUG "result": [ 23:00:03 DEBUG { 23:00:03 DEBUG "data": { 23:00:03 DEBUG "data": { 23:00:03 DEBUG { 23:00:03 DEBUG { 23:00:03 DEBUG "result": [ 23:00:03 DEBUG { 23:00:03 DEBUG { 23:00:03 DEBUG { 23:00:03 DEBUG { 23:00:03 DEBUG { 23:00:03 DEBUG { 23:00:03 DEBUG { 23:00:03 DEBUG { 23:00:03 DEBUG "metric": { 23:00:03 DEBUG "metric": { 23:00:03 DEBUG "metric": { 23:00:03 DEBUG "metric": { 23:00:03 DEBUG "metric": { 23:00:03 DEBUG "metric": { 23:00:03 DEBUG "result": [ 23:00:03 DEBUG "metric": { 23:00:03 DEBUG "metric": { 23:00:03 DEBUG "metric": { 23:00:03 DEBUG { 23:00:03 DEBUG "metric": { 23:00:03 DEBUG "resultType": "vector", 23:00:03 DEBUG "resultType": "vector", 23:00:03 DEBUG "metric": { 23:00:03 DEBUG "metric": { 23:00:03 DEBUG { 23:00:03 DEBUG "metric": { 23:00:03 DEBUG "metric": { 23:00:03 DEBUG "metric": { 23:00:03 DEBUG "metric": { 23:00:03 DEBUG "metric": { 23:00:03 DEBUG "metric": { 23:00:03 DEBUG "metric": { 23:00:03 DEBUG "metric": { 23:00:03 DEBUG "pod": "am-555f4db595-fz8sn" 23:00:03 DEBUG "pod": "ds-cts-1" 23:00:03 DEBUG "instance": "10.142.0.107:9100" 23:00:03 DEBUG "pod": "ds-idrepo-2" 23:00:03 DEBUG "pod": "am-555f4db595-fz8sn" 23:00:03 DEBUG "pod": "ds-cts-1" 23:00:03 DEBUG { 23:00:03 DEBUG "pod": "ds-cts-1" 23:00:03 DEBUG "pod": "ds-cts-1" 23:00:03 DEBUG "pod": "ds-idrepo-2" 23:00:03 DEBUG "metric": { 23:00:03 DEBUG "instance": "10.142.0.107:9100" 23:00:03 DEBUG "result": [ 23:00:03 DEBUG "result": [ 23:00:03 DEBUG "pod": "ds-idrepo-2" 23:00:03 DEBUG "pod": "ds-idrepo-2" 23:00:03 DEBUG "metric": { 23:00:03 DEBUG "instance": "10.142.0.107:9100" 23:00:03 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 23:00:03 DEBUG "pod": "ds-idrepo-2" 23:00:03 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 23:00:03 DEBUG "pod": "am-555f4db595-fz8sn" 23:00:03 DEBUG "pod": "am-555f4db595-fz8sn" 23:00:03 DEBUG "instance": "10.142.0.107:9100" 23:00:03 DEBUG "pod": "am-555f4db595-fz8sn" 23:00:03 DEBUG }, 23:00:03 DEBUG }, 23:00:03 DEBUG }, 23:00:03 DEBUG }, 23:00:03 DEBUG }, 23:00:03 DEBUG }, 23:00:03 DEBUG "metric": { 23:00:03 DEBUG }, 23:00:03 DEBUG }, 23:00:03 DEBUG }, 23:00:03 DEBUG "pod": "ds-idrepo-2" 23:00:03 DEBUG }, 23:00:03 DEBUG { 23:00:03 DEBUG { 23:00:03 DEBUG }, 23:00:03 DEBUG }, 23:00:03 DEBUG "pod": "am-555f4db595-fz8sn" 23:00:03 DEBUG }, 23:00:03 DEBUG }, 23:00:03 DEBUG }, 23:00:03 DEBUG }, 23:00:03 DEBUG }, 23:00:03 DEBUG }, 23:00:03 DEBUG }, 23:00:03 DEBUG }, 23:00:03 DEBUG "value": [ 23:00:03 DEBUG "value": [ 23:00:03 DEBUG "value": [ 23:00:03 DEBUG "value": [ 23:00:03 DEBUG "value": [ 23:00:03 DEBUG "value": [ 23:00:03 DEBUG "pod": "am-555f4db595-fz8sn" 23:00:03 DEBUG "value": [ 23:00:03 DEBUG "value": [ 23:00:03 DEBUG "value": [ 23:00:03 DEBUG }, 23:00:03 DEBUG "value": [ 23:00:03 DEBUG "metric": { 23:00:03 DEBUG "metric": { 23:00:03 DEBUG "value": [ 23:00:03 DEBUG "value": [ 23:00:03 DEBUG }, 23:00:03 DEBUG "value": [ 23:00:03 DEBUG "value": [ 23:00:03 DEBUG "value": [ 23:00:03 DEBUG "value": [ 23:00:03 DEBUG "value": [ 23:00:03 DEBUG "value": [ 23:00:03 DEBUG "value": [ 23:00:03 DEBUG "value": [ 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG }, 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG "value": [ 23:00:03 DEBUG 23:00:03 DEBUG "instance": "10.142.0.107:9100" 23:00:03 DEBUG "pod": "ds-idrepo-2" 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG "value": [ 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG "value": [ 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG }, 23:00:03 DEBUG }, 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG "value": [ 23:00:03 DEBUG "value": [ 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG [...] 23:00:03 DEBUG [...] 23:00:03 DEBUG [...] 23:00:03 DEBUG [...] 23:00:03 DEBUG [...] 23:00:03 DEBUG [...] 23:00:03 DEBUG 23:00:03 DEBUG [...] 23:00:03 DEBUG [...] 23:00:03 DEBUG [...] 23:00:03 DEBUG 23:00:03 DEBUG [...] 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG [...] 23:00:03 DEBUG [...] 23:00:03 DEBUG 23:00:03 DEBUG [...] 23:00:03 DEBUG [...] 23:00:03 DEBUG [...] 23:00:03 DEBUG [...] 23:00:03 DEBUG [...] 23:00:03 DEBUG [...] 23:00:03 DEBUG [...] 23:00:03 DEBUG [...] 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG [...] 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG [...] 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG [...] 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG [...] 23:00:03 DEBUG [...] 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG "pod": "am-555f4db595-kzxsp" 23:00:03 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 23:00:03 DEBUG "instance": "10.142.0.43:9100" 23:00:03 DEBUG "pod": "ds-idrepo-0" 23:00:03 DEBUG "pod": "am-555f4db595-kzxsp" 23:00:03 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 23:00:03 DEBUG 23:00:03 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 23:00:03 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 23:00:03 DEBUG "pod": "ds-idrepo-0" 23:00:03 DEBUG 23:00:03 DEBUG "instance": "10.142.0.43:9100" 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG "pod": "ds-idrepo-0" 23:00:03 DEBUG "pod": "ds-idrepo-0" 23:00:03 DEBUG 23:00:03 DEBUG "instance": "10.142.0.43:9100" 23:00:03 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 23:00:03 DEBUG "pod": "ds-idrepo-0" 23:00:03 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 23:00:03 DEBUG "pod": "am-555f4db595-kzxsp" 23:00:03 DEBUG "pod": "am-555f4db595-kzxsp" 23:00:03 DEBUG "instance": "10.142.0.43:9100" 23:00:03 DEBUG "pod": "am-555f4db595-kzxsp" 23:00:03 DEBUG }, 23:00:03 DEBUG }, 23:00:03 DEBUG }, 23:00:03 DEBUG }, 23:00:03 DEBUG }, 23:00:03 DEBUG }, 23:00:03 DEBUG 23:00:03 DEBUG }, 23:00:03 DEBUG }, 23:00:03 DEBUG }, 23:00:03 DEBUG "pod": "ds-cts-2" 23:00:03 DEBUG }, 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG }, 23:00:03 DEBUG }, 23:00:03 DEBUG "pod": "am-555f4db595-kzxsp" 23:00:03 DEBUG }, 23:00:03 DEBUG }, 23:00:03 DEBUG }, 23:00:03 DEBUG }, 23:00:03 DEBUG }, 23:00:03 DEBUG }, 23:00:03 DEBUG }, 23:00:03 DEBUG }, 23:00:03 DEBUG "value": [ 23:00:03 DEBUG "value": [ 23:00:03 DEBUG "value": [ 23:00:03 DEBUG "value": [ 23:00:03 DEBUG "value": [ 23:00:03 DEBUG "value": [ 23:00:03 DEBUG "pod": "am-555f4db595-kzxsp" 23:00:03 DEBUG "value": [ 23:00:03 DEBUG "value": [ 23:00:03 DEBUG 1713304803, 23:00:03 DEBUG }, 23:00:03 DEBUG "value": [ 23:00:03 DEBUG 23:00:03 DEBUG 23:00:03 DEBUG "value": [ 23:00:03 DEBUG "value": [ 23:00:03 DEBUG }, 23:00:03 DEBUG "value": [ 23:00:03 DEBUG "value": [ 23:00:03 DEBUG "value": [ 23:00:03 DEBUG "value": [ 23:00:03 DEBUG "value": [ 23:00:03 DEBUG "value": [ 23:00:03 DEBUG "value": [ 23:00:03 DEBUG "value": [ 23:00:03 DEBUG 1713304803, 23:00:03 DEBUG 1713304803, 23:00:03 DEBUG 1713304803, 23:00:03 DEBUG 1713304803, 23:00:03 DEBUG 1713304803, 23:00:03 DEBUG 1713304803, 23:00:03 DEBUG }, 23:00:03 DEBUG 1713304803, 23:00:03 DEBUG "value": [ 23:00:03 DEBUG "0.000026660247344518598" 23:00:03 DEBUG "value": [ 23:00:03 DEBUG 1713304803, 23:00:03 DEBUG "instance": "10.142.0.43:9100" 23:00:03 DEBUG "pod": "ds-idrepo-0" 23:00:03 DEBUG 1713304803, 23:00:03 DEBUG 1713304803, 23:00:03 DEBUG "value": [ 23:00:03 DEBUG 1713304803, 23:00:03 DEBUG 1713304803, 23:00:03 DEBUG 1713304803, 23:00:03 DEBUG 1713304803, 23:00:03 DEBUG 1713304803, 23:00:03 DEBUG 1713304803, 23:00:03 DEBUG 1713304803, 23:00:03 DEBUG 1713304803, 23:00:03 DEBUG "2162.866666666667" 23:00:03 DEBUG "0" 23:00:03 DEBUG "49298283.13333333" 23:00:03 DEBUG "0" 23:00:03 DEBUG "0.027" 23:00:03 DEBUG "0" 23:00:03 DEBUG "value": [ 23:00:03 DEBUG "0" 23:00:03 DEBUG 1713304803, 23:00:03 DEBUG ] 23:00:03 DEBUG 1713304803, 23:00:03 DEBUG "0.00046666666666700016" 23:00:03 DEBUG }, 23:00:03 DEBUG }, 23:00:03 DEBUG "0" 23:00:03 DEBUG "0" 23:00:03 DEBUG 1713304803, 23:00:03 DEBUG "49320074.39999999" 23:00:03 DEBUG "1698177024" 23:00:03 DEBUG "0" 23:00:03 DEBUG "6.861304476920942" 23:00:03 DEBUG "0" 23:00:03 DEBUG "3654303" 23:00:03 DEBUG "19980223488" 23:00:03 DEBUG "2001.7666666666667" 23:00:03 DEBUG ] 23:00:03 DEBUG ] 23:00:03 DEBUG ] 23:00:03 DEBUG ] 23:00:03 DEBUG ] 23:00:03 DEBUG ] 23:00:03 DEBUG 1713304803, 23:00:03 DEBUG ] 23:00:03 DEBUG "0" 23:00:03 DEBUG } 23:00:03 DEBUG "0" 23:00:03 DEBUG ] 23:00:03 DEBUG "value": [ 23:00:03 DEBUG "value": [ 23:00:03 DEBUG ] 23:00:03 DEBUG ] 23:00:03 DEBUG "2162.4" 23:00:03 DEBUG ] 23:00:03 DEBUG } 23:00:03 DEBUG ] 23:00:03 DEBUG ] 23:00:03 DEBUG ] 23:00:03 DEBUG ] 23:00:03 DEBUG ] 23:00:03 DEBUG ] 23:00:03 DEBUG } 23:00:03 DEBUG } 23:00:03 DEBUG } 23:00:03 DEBUG } 23:00:03 DEBUG } 23:00:03 DEBUG } 23:00:03 DEBUG "2162.9666666666667" 23:00:03 DEBUG } 23:00:03 DEBUG ] 23:00:03 DEBUG ] 23:00:03 DEBUG ] 23:00:03 DEBUG } 23:00:03 DEBUG 1713304803, 23:00:03 DEBUG 1713304803, 23:00:03 DEBUG } 23:00:03 DEBUG } 23:00:03 DEBUG ] 23:00:03 DEBUG ] 23:00:03 DEBUG ] 23:00:03 DEBUG } 23:00:03 DEBUG } 23:00:03 DEBUG } 23:00:03 DEBUG } 23:00:03 DEBUG } 23:00:03 DEBUG } 23:00:03 DEBUG ] 23:00:03 DEBUG ] 23:00:03 DEBUG ] 23:00:03 DEBUG ] 23:00:03 DEBUG ] 23:00:03 DEBUG ] 23:00:03 DEBUG ] 23:00:03 DEBUG ] 23:00:03 DEBUG } 23:00:03 DEBUG } 23:00:03 DEBUG } 23:00:03 DEBUG ] 23:00:03 DEBUG "0" 23:00:03 DEBUG "0" 23:00:03 DEBUG ] 23:00:03 DEBUG ] 23:00:03 DEBUG } 23:00:03 DEBUG } 23:00:03 DEBUG } 23:00:03 DEBUG ] 23:00:03 DEBUG } 23:00:03 DEBUG ] 23:00:03 DEBUG ] 23:00:03 DEBUG ] 23:00:03 DEBUG ] 23:00:03 DEBUG } 23:00:03 DEBUG } 23:00:03 DEBUG } 23:00:03 DEBUG } 23:00:03 DEBUG } 23:00:03 DEBUG } 23:00:03 DEBUG } 23:00:03 DEBUG } 23:00:03 DEBUG ] 23:00:03 DEBUG } 23:00:03 DEBUG ] 23:00:03 DEBUG } 23:00:03 DEBUG ] 23:00:03 DEBUG ] 23:00:03 DEBUG } 23:00:03 DEBUG } 23:00:03 DEBUG ] 23:00:03 DEBUG ] 23:00:03 DEBUG } 23:00:03 DEBUG } 23:00:03 DEBUG } 23:00:03 DEBUG ] 23:00:03 DEBUG } 23:00:03 DEBUG } 23:00:03 DEBUG } 23:00:03 DEBUG } 23:00:03 DEBUG } 23:00:03 DEBUG } 23:00:03 DEBUG } 23:00:03 DEBUG } 23:00:03 DEBUG } 23:00:03 DEBUG ] 23:00:03 DEBUG } 23:00:03 DEBUG } 23:00:04 DEBUG } 23:00:04 DEBUG } 23:00:04 DEBUG } 23:00:04 DEBUG } 23:00:04 DEBUG } 23:00:04 DEBUG } 23:00:04 DEBUG } 23:00:04 DEBUG } 23:00:04 DEBUG } 23:00:04 DEBUG } 23:00:04 DEBUG } 23:00:04 DEBUG } 23:00:04 DEBUG } 23:00:04 DEBUG } 23:00:04 DEBUG } 23:00:04 DEBUG } 23:00:04 DEBUG ] 23:00:04 DEBUG ] 23:00:04 DEBUG } 23:00:04 DEBUG } 23:00:04 DEBUG } 23:00:04 DEBUG } 23:00:04 DEBUG } 23:00:04 DEBUG } 23:00:04 DEBUG } 23:00:04 DEBUG } 23:00:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713304858" 23:00:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304858" 23:00:58 ERROR [http_cmd]: ERROR 23:00:58 INFO [http_cmd]: http status code OK 23:00:58 ERROR --- status code --- 23:00:58 DEBUG --- status code --- 23:00:58 ERROR http status code is 400 (expected [200, 401, 403, 404]) 23:00:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:00:58 ERROR --- http response --- 23:00:58 DEBUG --- http response --- 23:00:58 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 23:00:58 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 23:00:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 23:01:01 INFO 23:01:01 INFO [loop_until]: kubectl --namespace=xlou top pods 23:01:01 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 23:01:01 INFO [loop_until]: OK (rc = 0) 23:01:01 DEBUG --- stdout --- 23:01:01 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 9Mi am-555f4db595-4lbss 12913m 8682Mi am-555f4db595-fz8sn 13135m 8562Mi am-555f4db595-kzxsp 14036m 8976Mi ds-cts-0 19369m 12786Mi ds-cts-1 20858m 13071Mi ds-cts-2 18845m 13134Mi ds-idrepo-0 5805m 5030Mi ds-idrepo-1 5575m 2907Mi ds-idrepo-2 5393m 3220Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 4m 1207Mi idm-b85f9d978-rl76f 3m 1225Mi lodemon-bc756c647-cr7x6 2m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-7c9b667974-9mr2s 6449m 1528Mi 23:01:01 DEBUG --- stderr --- 23:01:01 DEBUG 23:01:03 INFO 23:01:03 INFO [loop_until]: kubectl --namespace=xlou top node 23:01:03 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 23:01:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304864" 23:01:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304864" 23:01:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304864" 23:01:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304864" 23:01:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304864" 23:01:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304864" 23:01:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304864" 23:01:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304864" 23:01:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304864" 23:01:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304864" 23:01:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304864" 23:01:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713304864" 23:01:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304864" 23:01:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304864" 23:01:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304864" 23:01:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713304864" 23:01:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304864" 23:01:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713304864" 23:01:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304864" 23:01:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713304864" 23:01:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304864" 23:01:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304864" 23:01:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304864" 23:01:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304864" 23:01:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304864" 23:01:04 INFO [http_cmd]: http status code OK 23:01:04 INFO [http_cmd]: http status code OK 23:01:04 INFO [loop_until]: OK (rc = 0) 23:01:04 INFO [http_cmd]: http status code OK 23:01:04 DEBUG --- status code --- 23:01:04 INFO [http_cmd]: http status code OK 23:01:04 INFO [http_cmd]: http status code OK 23:01:04 INFO [http_cmd]: http status code OK 23:01:04 INFO [http_cmd]: http status code OK 23:01:04 INFO [http_cmd]: http status code OK 23:01:04 INFO [http_cmd]: http status code OK 23:01:04 INFO [http_cmd]: http status code OK 23:01:04 INFO [http_cmd]: http status code OK 23:01:04 INFO [http_cmd]: http status code OK 23:01:04 INFO [http_cmd]: http status code OK 23:01:04 INFO [http_cmd]: http status code OK 23:01:04 DEBUG --- status code --- 23:01:04 INFO [http_cmd]: http status code OK 23:01:04 INFO [http_cmd]: http status code OK 23:01:04 DEBUG --- status code --- 23:01:04 INFO [http_cmd]: http status code OK 23:01:04 INFO [http_cmd]: http status code OK 23:01:04 INFO [http_cmd]: http status code OK 23:01:04 DEBUG --- stdout --- 23:01:04 INFO [http_cmd]: http status code OK 23:01:04 INFO [http_cmd]: http status code OK 23:01:04 INFO [http_cmd]: http status code OK 23:01:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:01:04 DEBUG --- status code --- 23:01:04 DEBUG --- status code --- 23:01:04 DEBUG --- status code --- 23:01:04 DEBUG --- status code --- 23:01:04 DEBUG --- status code --- 23:01:04 DEBUG --- status code --- 23:01:04 DEBUG --- status code --- 23:01:04 DEBUG --- status code --- 23:01:04 DEBUG --- status code --- 23:01:04 DEBUG --- status code --- 23:01:04 DEBUG --- status code --- 23:01:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:01:04 DEBUG --- status code --- 23:01:04 DEBUG --- status code --- 23:01:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:01:04 DEBUG --- status code --- 23:01:04 DEBUG --- status code --- 23:01:04 DEBUG --- status code --- 23:01:04 INFO [http_cmd]: http status code OK 23:01:04 INFO [http_cmd]: http status code OK 23:01:04 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 5090Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 19664m 65% 15882Mi 14% gke-xlou-cdm-default-pool-1b0769c8-nhlf 13157m 44% 10804Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 90m 0% 2647Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 5731m 19% 6571Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 20443m 68% 17261Mi 15% gke-xlou-cdm-default-pool-1b0769c8-xnmp 60m 0% 3152Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 13336m 44% 10472Mi 9% gke-xlou-cdm-ds-0f601751-hmm7 6037m 20% 5661Mi 5% gke-xlou-cdm-ds-0f601751-j488 18979m 63% 15064Mi 13% gke-xlou-cdm-ds-0f601751-mc5d 11605m 38% 3382Mi 3% gke-xlou-cdm-ds-0f601751-nj6q 13768m 46% 10850Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 6084m 20% 7359Mi 6% gke-xlou-cdm-frontend-a8771548-i82n 6981m 43% 6942Mi 11% 23:01:04 DEBUG --- status code --- 23:01:04 DEBUG --- status code --- 23:01:04 DEBUG --- status code --- 23:01:04 DEBUG --- http response --- 23:01:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:01:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:01:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:01:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:01:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:01:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:01:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:01:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:01:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:01:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:01:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:01:04 DEBUG --- http response --- 23:01:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:01:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:01:04 DEBUG --- http response --- 23:01:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:01:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:01:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:01:04 DEBUG --- status code --- 23:01:04 DEBUG --- status code --- 23:01:04 DEBUG --- stderr --- 23:01:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:01:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:01:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:01:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:01:04 DEBUG --- http response --- 23:01:04 DEBUG --- http response --- 23:01:04 DEBUG --- http response --- 23:01:04 DEBUG --- http response --- 23:01:04 DEBUG --- http response --- 23:01:04 DEBUG --- http response --- 23:01:04 DEBUG --- http response --- 23:01:04 DEBUG --- http response --- 23:01:04 DEBUG --- http response --- 23:01:04 DEBUG --- http response --- 23:01:04 INFO [http_cmd]: http status code OK 23:01:04 DEBUG --- http response --- 23:01:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:01:04 DEBUG --- http response --- 23:01:04 DEBUG --- http response --- 23:01:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:01:04 DEBUG --- http response --- 23:01:04 DEBUG --- http response --- 23:01:04 DEBUG --- http response --- 23:01:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:01:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:01:04 DEBUG 23:01:04 DEBUG --- http response --- 23:01:04 DEBUG --- http response --- 23:01:04 DEBUG --- http response --- 23:01:04 DEBUG ----- output ----- 23:01:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:01:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:01:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:01:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:01:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:01:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 23:01:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:01:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:01:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:01:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:01:04 DEBUG --- status code --- 23:01:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:01:04 DEBUG ----- output ----- 23:01:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 23:01:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 23:01:04 DEBUG ----- output ----- 23:01:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:01:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:01:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:01:04 DEBUG --- http response --- 23:01:04 DEBUG --- http response --- 23:01:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:01:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:01:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:01:04 DEBUG { 23:01:04 DEBUG ----- output ----- 23:01:04 DEBUG ----- output ----- 23:01:04 DEBUG ----- output ----- 23:01:04 DEBUG ----- output ----- 23:01:04 DEBUG ----- output ----- 23:01:04 DEBUG ----- output ----- 23:01:04 DEBUG ----- output ----- 23:01:04 DEBUG ----- output ----- 23:01:04 DEBUG ----- output ----- 23:01:04 DEBUG ----- output ----- 23:01:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:01:04 DEBUG ----- output ----- 23:01:04 DEBUG { 23:01:04 DEBUG ----- output ----- 23:01:04 DEBUG ----- output ----- 23:01:04 DEBUG { 23:01:04 DEBUG ----- output ----- 23:01:04 DEBUG ----- output ----- 23:01:04 DEBUG ----- output ----- 23:01:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:01:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:01:04 DEBUG ----- output ----- 23:01:04 DEBUG ----- output ----- 23:01:04 DEBUG ----- output ----- 23:01:04 DEBUG "status": "success", 23:01:04 DEBUG { 23:01:04 DEBUG { 23:01:04 DEBUG { 23:01:04 DEBUG { 23:01:04 DEBUG { 23:01:04 DEBUG { 23:01:04 DEBUG { 23:01:04 DEBUG { 23:01:04 DEBUG { 23:01:04 DEBUG { 23:01:04 DEBUG --- http response --- 23:01:04 DEBUG { 23:01:04 DEBUG "status": "success", 23:01:04 DEBUG { 23:01:04 DEBUG { 23:01:04 DEBUG "status": "success", 23:01:04 DEBUG { 23:01:04 DEBUG { 23:01:04 DEBUG { 23:01:04 DEBUG ----- output ----- 23:01:04 DEBUG ----- output ----- 23:01:04 DEBUG { 23:01:04 DEBUG { 23:01:04 DEBUG { 23:01:04 DEBUG "data": { 23:01:04 DEBUG "status": "success", 23:01:04 DEBUG "status": "success", 23:01:04 DEBUG "status": "success", 23:01:04 DEBUG "status": "success", 23:01:04 DEBUG "status": "success", 23:01:04 DEBUG "status": "success", 23:01:04 DEBUG "status": "success", 23:01:04 DEBUG "status": "success", 23:01:04 DEBUG "status": "success", 23:01:04 DEBUG "status": "success", 23:01:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 23:01:04 DEBUG "status": "success", 23:01:04 DEBUG "data": { 23:01:04 DEBUG "status": "success", 23:01:04 DEBUG "status": "success", 23:01:04 DEBUG "data": { 23:01:04 DEBUG "status": "success", 23:01:04 DEBUG "status": "success", 23:01:04 DEBUG "status": "success", 23:01:04 DEBUG { 23:01:04 DEBUG { 23:01:04 DEBUG "status": "success", 23:01:04 DEBUG "status": "success", 23:01:04 DEBUG "status": "success", 23:01:04 DEBUG "resultType": "vector", 23:01:04 DEBUG "data": { 23:01:04 DEBUG "data": { 23:01:04 DEBUG "data": { 23:01:04 DEBUG "data": { 23:01:04 DEBUG "data": { 23:01:04 DEBUG "data": { 23:01:04 DEBUG "data": { 23:01:04 DEBUG "data": { 23:01:04 DEBUG "data": { 23:01:04 DEBUG "data": { 23:01:04 DEBUG ----- output ----- 23:01:04 DEBUG "data": { 23:01:04 DEBUG "resultType": "vector", 23:01:04 DEBUG "data": { 23:01:04 DEBUG "data": { 23:01:04 DEBUG "resultType": "vector", 23:01:04 DEBUG "data": { 23:01:04 DEBUG "data": { 23:01:04 DEBUG "data": { 23:01:04 DEBUG "status": "success", 23:01:04 DEBUG "status": "success", 23:01:04 DEBUG "data": { 23:01:04 DEBUG "data": { 23:01:04 DEBUG "data": { 23:01:04 DEBUG "result": [ 23:01:04 DEBUG "resultType": "vector", 23:01:04 DEBUG "resultType": "vector", 23:01:04 DEBUG "resultType": "vector", 23:01:04 DEBUG "resultType": "vector", 23:01:04 DEBUG "resultType": "vector", 23:01:04 DEBUG "resultType": "vector", 23:01:04 DEBUG "resultType": "vector", 23:01:04 DEBUG "resultType": "vector", 23:01:04 DEBUG "resultType": "vector", 23:01:04 DEBUG "resultType": "vector", 23:01:04 DEBUG { 23:01:04 DEBUG "resultType": "vector", 23:01:04 DEBUG "result": [ 23:01:04 DEBUG "resultType": "vector", 23:01:04 DEBUG "resultType": "vector", 23:01:04 DEBUG "result": [ 23:01:04 DEBUG "resultType": "vector", 23:01:04 DEBUG "resultType": "vector", 23:01:04 DEBUG "resultType": "vector", 23:01:04 DEBUG "data": { 23:01:04 DEBUG "data": { 23:01:04 DEBUG "resultType": "vector", 23:01:04 DEBUG "resultType": "vector", 23:01:04 DEBUG "resultType": "vector", 23:01:04 DEBUG { 23:01:04 DEBUG "result": [ 23:01:04 DEBUG "result": [ 23:01:04 DEBUG "result": [ 23:01:04 DEBUG "result": [ 23:01:04 DEBUG "result": [ 23:01:04 DEBUG "result": [ 23:01:04 DEBUG "result": [ 23:01:04 DEBUG "result": [ 23:01:04 DEBUG "result": [ 23:01:04 DEBUG "result": [ 23:01:04 DEBUG "status": "success", 23:01:04 DEBUG "result": [ 23:01:04 DEBUG { 23:01:04 DEBUG "result": [ 23:01:04 DEBUG "result": [ 23:01:04 DEBUG { 23:01:04 DEBUG "result": [ 23:01:04 DEBUG "result": [ 23:01:04 DEBUG "result": [ 23:01:04 DEBUG "resultType": "vector", 23:01:04 DEBUG "resultType": "vector", 23:01:04 DEBUG "result": [ 23:01:04 DEBUG "result": [ 23:01:04 DEBUG "result": [ 23:01:04 DEBUG "metric": { 23:01:04 DEBUG { 23:01:04 DEBUG { 23:01:04 DEBUG { 23:01:04 DEBUG { 23:01:04 DEBUG { 23:01:04 DEBUG { 23:01:04 DEBUG { 23:01:04 DEBUG { 23:01:04 DEBUG { 23:01:04 DEBUG { 23:01:04 DEBUG "data": { 23:01:04 DEBUG { 23:01:04 DEBUG "metric": { 23:01:04 DEBUG { 23:01:04 DEBUG { 23:01:04 DEBUG "metric": { 23:01:04 DEBUG { 23:01:04 DEBUG { 23:01:04 DEBUG { 23:01:04 DEBUG "result": [ 23:01:04 DEBUG "result": [ 23:01:04 DEBUG { 23:01:04 DEBUG { 23:01:04 DEBUG { 23:01:04 DEBUG "instance": "10.142.0.107:9100" 23:01:04 DEBUG "metric": { 23:01:04 DEBUG "metric": { 23:01:04 DEBUG "metric": { 23:01:04 DEBUG "metric": { 23:01:04 DEBUG "metric": { 23:01:04 DEBUG "metric": { 23:01:04 DEBUG "metric": { 23:01:04 DEBUG "metric": { 23:01:04 DEBUG "metric": { 23:01:04 DEBUG "metric": { 23:01:04 DEBUG "resultType": "vector", 23:01:04 DEBUG "metric": { 23:01:04 DEBUG "pod": "ds-idrepo-2" 23:01:04 DEBUG "metric": { 23:01:04 DEBUG "metric": { 23:01:04 DEBUG "pod": "ds-idrepo-2" 23:01:04 DEBUG "metric": { 23:01:04 DEBUG "metric": { 23:01:04 DEBUG "metric": { 23:01:04 DEBUG { 23:01:04 DEBUG { 23:01:04 DEBUG "metric": { 23:01:04 DEBUG "metric": { 23:01:04 DEBUG "metric": { 23:01:04 DEBUG }, 23:01:04 DEBUG "instance": "10.142.0.107:9100" 23:01:04 DEBUG "pod": "ds-idrepo-2" 23:01:04 DEBUG "pod": "ds-idrepo-2" 23:01:04 DEBUG "instance": "10.142.0.107:9100" 23:01:04 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 23:01:04 DEBUG "pod": "ds-idrepo-2" 23:01:04 DEBUG "pod": "am-555f4db595-fz8sn" 23:01:04 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 23:01:04 DEBUG "pod": "am-555f4db595-fz8sn" 23:01:04 DEBUG "instance": "10.142.0.107:9100" 23:01:04 DEBUG "result": [ 23:01:04 DEBUG "pod": "am-555f4db595-fz8sn" 23:01:04 DEBUG }, 23:01:04 DEBUG "pod": "ds-idrepo-2" 23:01:04 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 23:01:04 DEBUG }, 23:01:04 DEBUG "pod": "am-555f4db595-fz8sn" 23:01:04 DEBUG "pod": "am-555f4db595-fz8sn" 23:01:04 DEBUG "pod": "am-555f4db595-fz8sn" 23:01:04 DEBUG "metric": { 23:01:04 DEBUG "metric": { 23:01:04 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 23:01:04 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 23:01:04 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 23:01:04 DEBUG "value": [ 23:01:04 DEBUG }, 23:01:04 DEBUG }, 23:01:04 DEBUG }, 23:01:04 DEBUG }, 23:01:04 DEBUG }, 23:01:04 DEBUG }, 23:01:04 DEBUG }, 23:01:04 DEBUG }, 23:01:04 DEBUG }, 23:01:04 DEBUG }, 23:01:04 DEBUG { 23:01:04 DEBUG }, 23:01:04 DEBUG "value": [ 23:01:04 DEBUG }, 23:01:04 DEBUG }, 23:01:04 DEBUG "value": [ 23:01:04 DEBUG }, 23:01:04 DEBUG }, 23:01:04 DEBUG }, 23:01:04 DEBUG "pod": "am-555f4db595-fz8sn" 23:01:04 DEBUG "instance": "10.142.0.107:9100" 23:01:04 DEBUG }, 23:01:04 DEBUG }, 23:01:04 DEBUG }, 23:01:04 DEBUG 23:01:04 DEBUG "value": [ 23:01:04 DEBUG "value": [ 23:01:04 DEBUG "value": [ 23:01:04 DEBUG "value": [ 23:01:04 DEBUG "value": [ 23:01:04 DEBUG "value": [ 23:01:04 DEBUG "value": [ 23:01:04 DEBUG "value": [ 23:01:04 DEBUG "value": [ 23:01:04 DEBUG "value": [ 23:01:04 DEBUG "metric": { 23:01:04 DEBUG "value": [ 23:01:04 DEBUG 23:01:04 DEBUG "value": [ 23:01:04 DEBUG "value": [ 23:01:04 DEBUG 23:01:04 DEBUG "value": [ 23:01:04 DEBUG "value": [ 23:01:04 DEBUG "value": [ 23:01:04 DEBUG }, 23:01:04 DEBUG }, 23:01:04 DEBUG "value": [ 23:01:04 DEBUG "value": [ 23:01:04 DEBUG "value": [ 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG "pod": "ds-idrepo-2" 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG "value": [ 23:01:04 DEBUG "value": [ 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG }, 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG [...] 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG "value": [ 23:01:04 DEBUG 23:01:04 DEBUG [...] 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG [...] 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG [...] 23:01:04 DEBUG [...] 23:01:04 DEBUG [...] 23:01:04 DEBUG [...] 23:01:04 DEBUG [...] 23:01:04 DEBUG [...] 23:01:04 DEBUG [...] 23:01:04 DEBUG [...] 23:01:04 DEBUG [...] 23:01:04 DEBUG [...] 23:01:04 DEBUG 23:01:04 DEBUG [...] 23:01:04 DEBUG 23:01:04 DEBUG [...] 23:01:04 DEBUG [...] 23:01:04 DEBUG 23:01:04 DEBUG [...] 23:01:04 DEBUG [...] 23:01:04 DEBUG [...] 23:01:04 DEBUG 23:01:04 DEBUG [...] 23:01:04 DEBUG [...] 23:01:04 DEBUG [...] 23:01:04 DEBUG [...] 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG [...] 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG "instance": "10.142.0.43:9100" 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG "pod": "ds-idrepo-0" 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG [...] 23:01:04 DEBUG 23:01:04 DEBUG "pod": "ds-idrepo-0" 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG "pod": "ds-cts-2" 23:01:04 DEBUG 23:01:04 DEBUG "pod": "am-555f4db595-kzxsp" 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG 23:01:04 DEBUG }, 23:01:04 DEBUG "instance": "10.142.0.43:9100" 23:01:04 DEBUG 23:01:04 DEBUG }, 23:01:04 DEBUG "instance": "10.142.0.43:9100" 23:01:04 DEBUG "pod": "ds-cts-2" 23:01:04 DEBUG "pod": "ds-idrepo-0" 23:01:04 DEBUG "pod": "am-555f4db595-kzxsp" 23:01:04 DEBUG "pod": "ds-cts-2" 23:01:04 DEBUG "pod": "am-555f4db595-kzxsp" 23:01:04 DEBUG "instance": "10.142.0.43:9100" 23:01:04 DEBUG 23:01:04 DEBUG "pod": "am-555f4db595-kzxsp" 23:01:04 DEBUG }, 23:01:04 DEBUG "pod": "ds-cts-2" 23:01:04 DEBUG "pod": "ds-idrepo-0" 23:01:04 DEBUG }, 23:01:04 DEBUG 23:01:04 DEBUG }, 23:01:04 DEBUG "pod": "am-555f4db595-kzxsp" 23:01:04 DEBUG 23:01:04 DEBUG "pod": "am-555f4db595-kzxsp" 23:01:04 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 23:01:04 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 23:01:04 DEBUG "pod": "ds-cts-2" 23:01:04 DEBUG "value": [ 23:01:04 DEBUG }, 23:01:04 DEBUG "pod": "ds-idrepo-0" 23:01:04 DEBUG "value": [ 23:01:04 DEBUG }, 23:01:04 DEBUG }, 23:01:04 DEBUG }, 23:01:04 DEBUG }, 23:01:04 DEBUG }, 23:01:04 DEBUG }, 23:01:04 DEBUG }, 23:01:04 DEBUG 23:01:04 DEBUG }, 23:01:04 DEBUG "value": [ 23:01:04 DEBUG }, 23:01:04 DEBUG }, 23:01:04 DEBUG "value": [ 23:01:04 DEBUG "pod": "am-555f4db595-kzxsp" 23:01:04 DEBUG "value": [ 23:01:04 DEBUG }, 23:01:04 DEBUG 23:01:04 DEBUG }, 23:01:04 DEBUG }, 23:01:04 DEBUG }, 23:01:04 DEBUG }, 23:01:04 DEBUG 1713304864, 23:01:04 DEBUG "value": [ 23:01:04 DEBUG }, 23:01:04 DEBUG 1713304864, 23:01:04 DEBUG "value": [ 23:01:04 DEBUG "value": [ 23:01:04 DEBUG "value": [ 23:01:04 DEBUG "value": [ 23:01:04 DEBUG "value": [ 23:01:04 DEBUG "value": [ 23:01:04 DEBUG "value": [ 23:01:04 DEBUG 23:01:04 DEBUG 1713304864, 23:01:04 DEBUG "value": [ 23:01:04 DEBUG "value": [ 23:01:04 DEBUG "value": [ 23:01:04 DEBUG 1713304864, 23:01:04 DEBUG }, 23:01:04 DEBUG 1713304864, 23:01:04 DEBUG "value": [ 23:01:04 DEBUG "instance": "10.142.0.43:9100" 23:01:04 DEBUG "value": [ 23:01:04 DEBUG "value": [ 23:01:04 DEBUG 1713304864, 23:01:04 DEBUG "value": [ 23:01:04 DEBUG "49169723.3" 23:01:04 DEBUG 1713304864, 23:01:04 DEBUG "value": [ 23:01:04 DEBUG "0" 23:01:04 DEBUG 1713304864, 23:01:04 DEBUG 1713304864, 23:01:04 DEBUG 1713304864, 23:01:04 DEBUG 1713304864, 23:01:04 DEBUG 1713304864, 23:01:04 DEBUG 1713304864, 23:01:04 DEBUG 1713304864, 23:01:04 DEBUG "pod": "ds-idrepo-0" 23:01:04 DEBUG "2000.8666666666666" 23:01:04 DEBUG 1713304864, 23:01:04 DEBUG 1713304864, 23:01:04 DEBUG 1713304864, 23:01:04 DEBUG "10.233987872744972" 23:01:04 DEBUG "value": [ 23:01:04 DEBUG "2169.366666666667" 23:01:04 DEBUG 1713304864, 23:01:04 DEBUG }, 23:01:04 DEBUG 1713304864, 23:01:04 DEBUG "value": [ 23:01:04 DEBUG "1696505856" 23:01:04 DEBUG 1713304864, 23:01:04 DEBUG ] 23:01:04 DEBUG "49160344.766666666" 23:01:04 DEBUG 1713304864, 23:01:04 DEBUG ] 23:01:04 DEBUG "0.00040000000000001514" 23:01:04 DEBUG ] 23:01:04 DEBUG "410171622.81322503" 23:01:04 DEBUG "2169.733333333333" 23:01:04 DEBUG "20235737.7173179" 23:01:04 DEBUG "0.027" 23:01:04 DEBUG "19981192192" 23:01:04 DEBUG }, 23:01:04 DEBUG ] 23:01:04 DEBUG "0" 23:01:04 DEBUG "0" 23:01:04 DEBUG "0" 23:01:04 DEBUG ] 23:01:04 DEBUG 1713304864, 23:01:04 DEBUG ] 23:01:04 DEBUG "3663982.5" 23:01:04 DEBUG "value": [ 23:01:04 DEBUG "0" 23:01:04 DEBUG 1713304864, 23:01:04 DEBUG ] 23:01:04 DEBUG "6.770908688488059" 23:01:04 DEBUG } 23:01:04 DEBUG ] 23:01:04 DEBUG "0" 23:01:04 DEBUG } 23:01:04 DEBUG "0" 23:01:04 DEBUG } 23:01:04 DEBUG ] 23:01:04 DEBUG ] 23:01:04 DEBUG ] 23:01:04 DEBUG ] 23:01:04 DEBUG ] 23:01:04 DEBUG "value": [ 23:01:04 DEBUG } 23:01:04 DEBUG ] 23:01:04 DEBUG ] 23:01:04 DEBUG ] 23:01:04 DEBUG } 23:01:04 DEBUG "2167.6666666666665" 23:01:04 DEBUG } 23:01:04 DEBUG ] 23:01:04 DEBUG 1713304864, 23:01:04 DEBUG ] 23:01:04 DEBUG "34797574.5356043" 23:01:04 DEBUG } 23:01:04 DEBUG ] 23:01:04 DEBUG ] 23:01:04 DEBUG } 23:01:04 DEBUG ] 23:01:04 DEBUG ] 23:01:04 DEBUG ] 23:01:04 DEBUG ] 23:01:04 DEBUG } 23:01:04 DEBUG } 23:01:04 DEBUG } 23:01:04 DEBUG } 23:01:04 DEBUG } 23:01:04 DEBUG 1713304864, 23:01:04 DEBUG ] 23:01:04 DEBUG } 23:01:04 DEBUG } 23:01:04 DEBUG } 23:01:04 DEBUG ] 23:01:04 DEBUG ] 23:01:04 DEBUG ] 23:01:04 DEBUG } 23:01:04 DEBUG "0.0003333333333333262" 23:01:04 DEBUG } 23:01:04 DEBUG ] 23:01:04 DEBUG ] 23:01:04 DEBUG } 23:01:04 DEBUG } 23:01:04 DEBUG ] 23:01:04 DEBUG } 23:01:04 DEBUG } 23:01:04 DEBUG } 23:01:04 DEBUG } 23:01:04 DEBUG ] 23:01:04 DEBUG ] 23:01:04 DEBUG ] 23:01:04 DEBUG ] 23:01:04 DEBUG ] 23:01:04 DEBUG "0" 23:01:04 DEBUG } 23:01:04 DEBUG ] 23:01:04 DEBUG ] 23:01:04 DEBUG ] 23:01:04 DEBUG } 23:01:04 DEBUG } 23:01:04 DEBUG ] 23:01:04 DEBUG ] 23:01:04 DEBUG ] 23:01:04 DEBUG ] 23:01:04 DEBUG } 23:01:04 DEBUG } 23:01:04 DEBUG ] 23:01:04 DEBUG } 23:01:04 DEBUG } 23:01:04 DEBUG ] 23:01:04 DEBUG } 23:01:04 DEBUG } 23:01:04 DEBUG } 23:01:04 DEBUG } 23:01:04 DEBUG } 23:01:04 DEBUG } 23:01:04 DEBUG } 23:01:04 DEBUG ] 23:01:04 DEBUG } 23:01:04 DEBUG } 23:01:04 DEBUG } 23:01:04 DEBUG } 23:01:04 DEBUG } 23:01:04 DEBUG } 23:01:04 DEBUG } 23:01:04 DEBUG } 23:01:04 DEBUG } 23:01:04 DEBUG } 23:01:04 DEBUG ] 23:01:04 DEBUG } 23:01:04 DEBUG } 23:01:04 DEBUG } 23:01:04 DEBUG } 23:01:04 DEBUG ] 23:01:04 DEBUG } 23:01:04 DEBUG } 23:01:04 DEBUG } 23:01:04 DEBUG } 23:01:04 DEBUG } 23:01:04 DEBUG } 23:01:04 DEBUG } 23:01:04 DEBUG } 23:01:04 DEBUG } 23:01:04 DEBUG } 23:01:04 DEBUG } 23:01:04 DEBUG } 23:01:04 DEBUG ] 23:01:04 DEBUG } 23:01:04 DEBUG } 23:01:04 DEBUG } 23:01:04 DEBUG } 23:01:04 DEBUG } 23:01:04 DEBUG ] 23:01:04 DEBUG } 23:01:04 DEBUG } 23:01:04 DEBUG } 23:01:04 DEBUG } 23:01:04 DEBUG } 23:01:04 DEBUG } 23:01:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304918" 23:01:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713304918" 23:01:58 ERROR [http_cmd]: ERROR 23:01:58 ERROR --- status code --- 23:01:58 ERROR http status code is 400 (expected [200, 401, 403, 404]) 23:01:58 ERROR --- http response --- 23:01:58 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 23:01:58 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 23:01:58 INFO [http_cmd]: http status code OK 23:01:58 DEBUG --- status code --- 23:01:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:01:58 DEBUG --- http response --- 23:01:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 23:02:01 INFO 23:02:01 INFO [loop_until]: kubectl --namespace=xlou top pods 23:02:01 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 23:02:01 INFO [loop_until]: OK (rc = 0) 23:02:01 DEBUG --- stdout --- 23:02:01 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 9Mi am-555f4db595-4lbss 13292m 8682Mi am-555f4db595-fz8sn 13253m 8559Mi am-555f4db595-kzxsp 13150m 8985Mi ds-cts-0 19471m 12657Mi ds-cts-1 19294m 12894Mi ds-cts-2 17248m 13408Mi ds-idrepo-0 5781m 5030Mi ds-idrepo-1 5452m 2908Mi ds-idrepo-2 5361m 3220Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 3m 1208Mi idm-b85f9d978-rl76f 5m 1226Mi lodemon-bc756c647-cr7x6 2m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-7c9b667974-9mr2s 6654m 1540Mi 23:02:01 DEBUG --- stderr --- 23:02:01 DEBUG 23:02:04 INFO 23:02:04 INFO [loop_until]: kubectl --namespace=xlou top node 23:02:04 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 23:02:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304924" 23:02:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304924" 23:02:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304924" 23:02:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304924" 23:02:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304924" 23:02:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713304924" 23:02:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304924" 23:02:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304924" 23:02:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304924" 23:02:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713304924" 23:02:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304924" 23:02:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304924" 23:02:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304924" 23:02:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304924" 23:02:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304924" 23:02:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304924" 23:02:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713304924" 23:02:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304924" 23:02:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304924" 23:02:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713304924" 23:02:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304924" 23:02:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304924" 23:02:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304924" 23:02:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304924" 23:02:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304924" 23:02:04 INFO [loop_until]: OK (rc = 0) 23:02:04 INFO [http_cmd]: http status code OK 23:02:04 INFO [http_cmd]: http status code OK 23:02:04 INFO [http_cmd]: http status code OK 23:02:04 INFO [http_cmd]: http status code OK 23:02:04 INFO [http_cmd]: http status code OK 23:02:04 INFO [http_cmd]: http status code OK 23:02:04 INFO [http_cmd]: http status code OK 23:02:04 INFO [http_cmd]: http status code OK 23:02:04 INFO [http_cmd]: http status code OK 23:02:04 INFO [http_cmd]: http status code OK 23:02:04 INFO [http_cmd]: http status code OK 23:02:04 INFO [http_cmd]: http status code OK 23:02:04 INFO [http_cmd]: http status code OK 23:02:04 INFO [http_cmd]: http status code OK 23:02:04 INFO [http_cmd]: http status code OK 23:02:04 INFO [http_cmd]: http status code OK 23:02:04 DEBUG --- stdout --- 23:02:04 DEBUG --- status code --- 23:02:04 INFO [http_cmd]: http status code OK 23:02:04 DEBUG --- status code --- 23:02:04 INFO [http_cmd]: http status code OK 23:02:04 DEBUG --- status code --- 23:02:04 INFO [http_cmd]: http status code OK 23:02:04 DEBUG --- status code --- 23:02:04 INFO [http_cmd]: http status code OK 23:02:04 INFO [http_cmd]: http status code OK 23:02:04 DEBUG --- status code --- 23:02:04 INFO [http_cmd]: http status code OK 23:02:04 DEBUG --- status code --- 23:02:04 DEBUG --- status code --- 23:02:04 DEBUG --- status code --- 23:02:04 DEBUG --- status code --- 23:02:04 DEBUG --- status code --- 23:02:04 INFO [http_cmd]: http status code OK 23:02:04 DEBUG --- status code --- 23:02:04 DEBUG --- status code --- 23:02:04 DEBUG --- status code --- 23:02:04 DEBUG --- status code --- 23:02:04 DEBUG --- status code --- 23:02:04 DEBUG --- status code --- 23:02:04 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 65m 0% 5094Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 18865m 63% 15599Mi 13% gke-xlou-cdm-default-pool-1b0769c8-nhlf 13135m 43% 10822Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 83m 0% 2648Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 5658m 18% 6562Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 20438m 68% 17169Mi 15% gke-xlou-cdm-default-pool-1b0769c8-xnmp 59m 0% 3153Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 13120m 43% 10492Mi 9% gke-xlou-cdm-ds-0f601751-hmm7 5942m 19% 5657Mi 5% gke-xlou-cdm-ds-0f601751-j488 18629m 62% 15131Mi 13% gke-xlou-cdm-ds-0f601751-mc5d 11637m 38% 3405Mi 3% gke-xlou-cdm-ds-0f601751-nj6q 13752m 46% 10868Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 5983m 20% 7358Mi 6% gke-xlou-cdm-frontend-a8771548-i82n 6966m 43% 6947Mi 11% 23:02:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:02:04 DEBUG --- status code --- 23:02:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:02:04 DEBUG --- status code --- 23:02:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:02:04 DEBUG --- status code --- 23:02:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:02:04 INFO [http_cmd]: http status code OK 23:02:04 DEBUG --- status code --- 23:02:04 DEBUG --- status code --- 23:02:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:02:04 DEBUG --- status code --- 23:02:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:02:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:02:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:02:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:02:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:02:04 DEBUG --- status code --- 23:02:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:02:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:02:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:02:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:02:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:02:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:02:04 DEBUG --- stderr --- 23:02:04 DEBUG --- http response --- 23:02:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:02:04 DEBUG --- http response --- 23:02:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:02:04 DEBUG --- http response --- 23:02:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:02:04 DEBUG --- http response --- 23:02:04 DEBUG --- status code --- 23:02:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:02:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:02:04 DEBUG --- http response --- 23:02:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:02:04 DEBUG --- http response --- 23:02:04 DEBUG --- http response --- 23:02:04 DEBUG --- http response --- 23:02:04 DEBUG --- http response --- 23:02:04 DEBUG --- http response --- 23:02:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:02:04 DEBUG --- http response --- 23:02:04 DEBUG --- http response --- 23:02:04 DEBUG --- http response --- 23:02:04 DEBUG --- http response --- 23:02:04 DEBUG --- http response --- 23:02:04 DEBUG --- http response --- 23:02:04 DEBUG 23:02:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:02:04 DEBUG --- http response --- 23:02:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:02:04 DEBUG --- http response --- 23:02:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:02:04 DEBUG --- http response --- 23:02:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:02:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 23:02:04 INFO [http_cmd]: http status code OK 23:02:04 DEBUG --- http response --- 23:02:04 DEBUG --- http response --- 23:02:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:02:04 DEBUG --- http response --- 23:02:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:02:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:02:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:02:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 23:02:04 DEBUG --- http response --- 23:02:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:02:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:02:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:02:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:02:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 23:02:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:02:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:02:04 DEBUG ----- output ----- 23:02:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 23:02:04 DEBUG ----- output ----- 23:02:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:02:04 DEBUG ----- output ----- 23:02:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:02:04 DEBUG --- http response --- 23:02:04 DEBUG ----- output ----- 23:02:04 DEBUG --- status code --- 23:02:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:02:04 DEBUG ----- output ----- 23:02:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:02:04 DEBUG ----- output ----- 23:02:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 23:02:04 DEBUG ----- output ----- 23:02:04 DEBUG ----- output ----- 23:02:04 DEBUG ----- output ----- 23:02:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:02:04 DEBUG ----- output ----- 23:02:04 DEBUG ----- output ----- 23:02:04 DEBUG ----- output ----- 23:02:04 DEBUG ----- output ----- 23:02:04 DEBUG ----- output ----- 23:02:04 DEBUG ----- output ----- 23:02:04 DEBUG ----- output ----- 23:02:04 DEBUG { 23:02:04 DEBUG ----- output ----- 23:02:04 DEBUG { 23:02:04 DEBUG ----- output ----- 23:02:04 DEBUG { 23:02:04 DEBUG ----- output ----- 23:02:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 23:02:04 DEBUG { 23:02:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:02:04 DEBUG ----- output ----- 23:02:04 DEBUG { 23:02:04 DEBUG ----- output ----- 23:02:04 DEBUG { 23:02:04 DEBUG ----- output ----- 23:02:04 DEBUG { 23:02:04 DEBUG { 23:02:04 DEBUG { 23:02:04 DEBUG ----- output ----- 23:02:04 DEBUG { 23:02:04 DEBUG { 23:02:04 DEBUG { 23:02:04 DEBUG { 23:02:04 DEBUG { 23:02:04 DEBUG { 23:02:04 DEBUG { 23:02:04 DEBUG "status": "success", 23:02:04 DEBUG { 23:02:04 DEBUG "status": "success", 23:02:04 DEBUG { 23:02:04 DEBUG "status": "success", 23:02:04 DEBUG { 23:02:04 DEBUG ----- output ----- 23:02:04 DEBUG "status": "success", 23:02:04 DEBUG --- http response --- 23:02:04 DEBUG { 23:02:04 DEBUG "status": "success", 23:02:04 DEBUG { 23:02:04 DEBUG "status": "success", 23:02:04 DEBUG { 23:02:04 DEBUG "status": "success", 23:02:04 DEBUG "status": "success", 23:02:04 DEBUG "status": "success", 23:02:04 DEBUG { 23:02:04 DEBUG "status": "success", 23:02:04 DEBUG "status": "success", 23:02:04 DEBUG "status": "success", 23:02:04 DEBUG "status": "success", 23:02:04 DEBUG "status": "success", 23:02:04 DEBUG "status": "success", 23:02:04 DEBUG "status": "success", 23:02:04 DEBUG "data": { 23:02:04 DEBUG "status": "success", 23:02:04 DEBUG "data": { 23:02:04 DEBUG "status": "success", 23:02:04 DEBUG "data": { 23:02:04 DEBUG "status": "success", 23:02:04 DEBUG { 23:02:04 DEBUG "status": "success", 23:02:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:02:04 DEBUG "status": "success", 23:02:04 DEBUG "data": { 23:02:04 DEBUG "status": "success", 23:02:04 DEBUG "data": { 23:02:04 DEBUG "status": "success", 23:02:04 DEBUG "data": { 23:02:04 DEBUG "data": { 23:02:04 DEBUG "data": { 23:02:04 DEBUG "status": "success", 23:02:04 DEBUG "data": { 23:02:04 DEBUG "data": { 23:02:04 DEBUG "data": { 23:02:04 DEBUG "data": { 23:02:04 DEBUG "data": { 23:02:04 DEBUG "data": { 23:02:04 DEBUG "data": { 23:02:04 DEBUG "resultType": "vector", 23:02:04 DEBUG "data": { 23:02:04 DEBUG "resultType": "vector", 23:02:04 DEBUG "data": { 23:02:04 DEBUG "resultType": "vector", 23:02:04 DEBUG "data": { 23:02:04 DEBUG "data": { 23:02:04 DEBUG "data": { 23:02:04 DEBUG ----- output ----- 23:02:04 DEBUG "data": { 23:02:04 DEBUG "resultType": "vector", 23:02:04 DEBUG "data": { 23:02:04 DEBUG "resultType": "vector", 23:02:04 DEBUG "data": { 23:02:04 DEBUG "resultType": "vector", 23:02:04 DEBUG "resultType": "vector", 23:02:04 DEBUG "resultType": "vector", 23:02:04 DEBUG "data": { 23:02:04 DEBUG "resultType": "vector", 23:02:04 DEBUG "resultType": "vector", 23:02:04 DEBUG "resultType": "vector", 23:02:04 DEBUG "resultType": "vector", 23:02:04 DEBUG "resultType": "vector", 23:02:04 DEBUG "resultType": "vector", 23:02:04 DEBUG "resultType": "vector", 23:02:04 DEBUG "result": [ 23:02:04 DEBUG "resultType": "vector", 23:02:04 DEBUG "result": [ 23:02:04 DEBUG "resultType": "vector", 23:02:04 DEBUG "result": [ 23:02:04 DEBUG "resultType": "vector", 23:02:04 DEBUG "resultType": "vector", 23:02:04 DEBUG "resultType": "vector", 23:02:04 DEBUG { 23:02:04 DEBUG "resultType": "vector", 23:02:04 DEBUG "result": [ 23:02:04 DEBUG "resultType": "vector", 23:02:04 DEBUG "result": [ 23:02:04 DEBUG "resultType": "vector", 23:02:04 DEBUG "result": [ 23:02:04 DEBUG "result": [ 23:02:04 DEBUG "result": [ 23:02:04 DEBUG "resultType": "vector", 23:02:04 DEBUG "result": [ 23:02:04 DEBUG "result": [ 23:02:04 DEBUG "result": [ 23:02:04 DEBUG "result": [ 23:02:04 DEBUG "result": [ 23:02:04 DEBUG "result": [ 23:02:04 DEBUG "result": [ 23:02:04 DEBUG { 23:02:04 DEBUG "result": [ 23:02:04 DEBUG { 23:02:04 DEBUG "result": [ 23:02:04 DEBUG { 23:02:04 DEBUG "result": [ 23:02:04 DEBUG "result": [ 23:02:04 DEBUG "result": [ 23:02:04 DEBUG "status": "success", 23:02:04 DEBUG "result": [ 23:02:04 DEBUG { 23:02:04 DEBUG "result": [ 23:02:04 DEBUG { 23:02:04 DEBUG "result": [ 23:02:04 DEBUG { 23:02:04 DEBUG { 23:02:04 DEBUG { 23:02:04 DEBUG "result": [ 23:02:04 DEBUG { 23:02:04 DEBUG { 23:02:04 DEBUG { 23:02:04 DEBUG { 23:02:04 DEBUG { 23:02:04 DEBUG { 23:02:04 DEBUG { 23:02:04 DEBUG "metric": { 23:02:04 DEBUG { 23:02:04 DEBUG "metric": { 23:02:04 DEBUG { 23:02:04 DEBUG "metric": { 23:02:04 DEBUG { 23:02:04 DEBUG { 23:02:04 DEBUG { 23:02:04 DEBUG "data": { 23:02:04 DEBUG { 23:02:04 DEBUG "metric": { 23:02:04 DEBUG { 23:02:04 DEBUG "metric": { 23:02:04 DEBUG { 23:02:04 DEBUG "metric": { 23:02:04 DEBUG "metric": { 23:02:04 DEBUG "metric": { 23:02:04 DEBUG { 23:02:04 DEBUG "metric": { 23:02:04 DEBUG "metric": { 23:02:04 DEBUG "metric": { 23:02:04 DEBUG "metric": { 23:02:04 DEBUG "metric": { 23:02:04 DEBUG "metric": { 23:02:04 DEBUG "metric": { 23:02:04 DEBUG "pod": "am-555f4db595-fz8sn" 23:02:04 DEBUG "metric": { 23:02:04 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 23:02:04 DEBUG "metric": { 23:02:04 DEBUG "pod": "ds-idrepo-2" 23:02:04 DEBUG "metric": { 23:02:04 DEBUG "metric": { 23:02:04 DEBUG "metric": { 23:02:04 DEBUG "resultType": "vector", 23:02:04 DEBUG "metric": { 23:02:04 DEBUG "instance": "10.142.0.107:9100" 23:02:04 DEBUG "metric": { 23:02:04 DEBUG "instance": "10.142.0.107:9100" 23:02:04 DEBUG "metric": { 23:02:04 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 23:02:04 DEBUG "pod": "am-555f4db595-fz8sn" 23:02:04 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 23:02:04 DEBUG "metric": { 23:02:04 DEBUG "instance": "10.142.0.107:9100" 23:02:04 DEBUG "instance": "10.142.0.107:9100" 23:02:04 DEBUG "pod": "ds-idrepo-2" 23:02:04 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 23:02:04 DEBUG "pod": "ds-idrepo-2" 23:02:04 DEBUG "pod": "am-555f4db595-fz8sn" 23:02:04 DEBUG "pod": "am-555f4db595-fz8sn" 23:02:04 DEBUG }, 23:02:04 DEBUG "pod": "ds-idrepo-2" 23:02:04 DEBUG }, 23:02:04 DEBUG "pod": "am-555f4db595-fz8sn" 23:02:04 DEBUG }, 23:02:04 DEBUG "pod": "ds-idrepo-2" 23:02:04 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 23:02:04 DEBUG "pod": "ds-idrepo-2" 23:02:04 DEBUG "result": [ 23:02:04 DEBUG "pod": "am-555f4db595-fz8sn" 23:02:04 DEBUG }, 23:02:04 DEBUG "pod": "am-555f4db595-fz8sn" 23:02:04 DEBUG }, 23:02:04 DEBUG "pod": "ds-idrepo-2" 23:02:04 DEBUG }, 23:02:04 DEBUG }, 23:02:04 DEBUG }, 23:02:04 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 23:02:04 DEBUG }, 23:02:04 DEBUG }, 23:02:04 DEBUG }, 23:02:04 DEBUG }, 23:02:04 DEBUG }, 23:02:04 DEBUG }, 23:02:04 DEBUG }, 23:02:04 DEBUG "value": [ 23:02:04 DEBUG }, 23:02:04 DEBUG "value": [ 23:02:04 DEBUG }, 23:02:04 DEBUG "value": [ 23:02:04 DEBUG }, 23:02:04 DEBUG }, 23:02:04 DEBUG }, 23:02:04 DEBUG { 23:02:04 DEBUG }, 23:02:04 DEBUG "value": [ 23:02:04 DEBUG }, 23:02:04 DEBUG "value": [ 23:02:04 DEBUG }, 23:02:04 DEBUG "value": [ 23:02:04 DEBUG "value": [ 23:02:04 DEBUG "value": [ 23:02:04 DEBUG }, 23:02:04 DEBUG "value": [ 23:02:04 DEBUG "value": [ 23:02:04 DEBUG "value": [ 23:02:04 DEBUG "value": [ 23:02:04 DEBUG "value": [ 23:02:04 DEBUG "value": [ 23:02:04 DEBUG "value": [ 23:02:04 DEBUG 23:02:04 DEBUG "value": [ 23:02:04 DEBUG 23:02:04 DEBUG "value": [ 23:02:04 DEBUG 23:02:04 DEBUG "value": [ 23:02:04 DEBUG "value": [ 23:02:04 DEBUG "value": [ 23:02:04 DEBUG "metric": { 23:02:04 DEBUG "value": [ 23:02:04 DEBUG 23:02:04 DEBUG "value": [ 23:02:04 DEBUG 23:02:04 DEBUG "value": [ 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG "value": [ 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG "instance": "10.142.0.107:9100" 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG }, 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG [...] 23:02:04 DEBUG 23:02:04 DEBUG [...] 23:02:04 DEBUG 23:02:04 DEBUG [...] 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG "value": [ 23:02:04 DEBUG 23:02:04 DEBUG [...] 23:02:04 DEBUG 23:02:04 DEBUG [...] 23:02:04 DEBUG 23:02:04 DEBUG [...] 23:02:04 DEBUG [...] 23:02:04 DEBUG [...] 23:02:04 DEBUG 23:02:04 DEBUG [...] 23:02:04 DEBUG [...] 23:02:04 DEBUG [...] 23:02:04 DEBUG [...] 23:02:04 DEBUG [...] 23:02:04 DEBUG [...] 23:02:04 DEBUG [...] 23:02:04 DEBUG 23:02:04 DEBUG [...] 23:02:04 DEBUG 23:02:04 DEBUG [...] 23:02:04 DEBUG 23:02:04 DEBUG [...] 23:02:04 DEBUG [...] 23:02:04 DEBUG [...] 23:02:04 DEBUG 23:02:04 DEBUG [...] 23:02:04 DEBUG 23:02:04 DEBUG [...] 23:02:04 DEBUG 23:02:04 DEBUG [...] 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG [...] 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG "pod": "am-555f4db595-kzxsp" 23:02:04 DEBUG 23:02:04 DEBUG "pod": "end-user-ui-58878d7668-zg4h2" 23:02:04 DEBUG 23:02:04 DEBUG "pod": "ds-idrepo-0" 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG 23:02:04 DEBUG [...] 23:02:04 DEBUG 23:02:04 DEBUG "instance": "10.142.0.43:9100" 23:02:04 DEBUG 23:02:04 DEBUG "instance": "10.142.0.43:9100" 23:02:04 DEBUG 23:02:04 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 23:02:04 DEBUG "pod": "am-555f4db595-kzxsp" 23:02:04 DEBUG "pod": "end-user-ui-58878d7668-zg4h2" 23:02:04 DEBUG 23:02:04 DEBUG "instance": "10.142.0.43:9100" 23:02:04 DEBUG "instance": "10.142.0.43:9100" 23:02:04 DEBUG "pod": "ds-idrepo-0" 23:02:04 DEBUG "pod": "end-user-ui-58878d7668-zg4h2" 23:02:04 DEBUG "pod": "ds-cts-2" 23:02:04 DEBUG "pod": "am-555f4db595-kzxsp" 23:02:04 DEBUG "pod": "am-555f4db595-kzxsp" 23:02:04 DEBUG }, 23:02:04 DEBUG "pod": "ds-idrepo-0" 23:02:04 DEBUG }, 23:02:04 DEBUG "pod": "am-555f4db595-kzxsp" 23:02:04 DEBUG }, 23:02:04 DEBUG "pod": "ds-idrepo-0" 23:02:04 DEBUG "pod": "end-user-ui-58878d7668-zg4h2" 23:02:04 DEBUG "pod": "ds-idrepo-0" 23:02:04 DEBUG 23:02:04 DEBUG "pod": "am-555f4db595-kzxsp" 23:02:04 DEBUG }, 23:02:04 DEBUG "pod": "am-555f4db595-kzxsp" 23:02:04 DEBUG }, 23:02:04 DEBUG "pod": "ds-idrepo-0" 23:02:04 DEBUG }, 23:02:04 DEBUG }, 23:02:04 DEBUG }, 23:02:04 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 23:02:04 DEBUG }, 23:02:04 DEBUG }, 23:02:04 DEBUG }, 23:02:04 DEBUG }, 23:02:04 DEBUG }, 23:02:04 DEBUG }, 23:02:04 DEBUG }, 23:02:04 DEBUG "value": [ 23:02:04 DEBUG }, 23:02:04 DEBUG "value": [ 23:02:04 DEBUG }, 23:02:04 DEBUG "value": [ 23:02:04 DEBUG }, 23:02:04 DEBUG }, 23:02:04 DEBUG }, 23:02:04 DEBUG 23:02:04 DEBUG }, 23:02:04 DEBUG "value": [ 23:02:04 DEBUG }, 23:02:04 DEBUG "value": [ 23:02:04 DEBUG }, 23:02:04 DEBUG "value": [ 23:02:04 DEBUG "value": [ 23:02:04 DEBUG "value": [ 23:02:04 DEBUG }, 23:02:04 DEBUG "value": [ 23:02:04 DEBUG "value": [ 23:02:04 DEBUG "value": [ 23:02:04 DEBUG "value": [ 23:02:04 DEBUG "value": [ 23:02:04 DEBUG "value": [ 23:02:04 DEBUG "value": [ 23:02:04 DEBUG 1713304924, 23:02:04 DEBUG "value": [ 23:02:04 DEBUG 1713304924, 23:02:04 DEBUG "value": [ 23:02:04 DEBUG 1713304924, 23:02:04 DEBUG "value": [ 23:02:04 DEBUG "value": [ 23:02:04 DEBUG "value": [ 23:02:04 DEBUG 23:02:04 DEBUG "value": [ 23:02:04 DEBUG 1713304924, 23:02:04 DEBUG "value": [ 23:02:04 DEBUG 1713304924, 23:02:04 DEBUG "value": [ 23:02:04 DEBUG 1713304924, 23:02:04 DEBUG 1713304924, 23:02:04 DEBUG 1713304924, 23:02:04 DEBUG "value": [ 23:02:04 DEBUG 1713304924, 23:02:04 DEBUG 1713304924, 23:02:04 DEBUG 1713304924, 23:02:04 DEBUG 1713304924, 23:02:04 DEBUG 1713304924, 23:02:04 DEBUG 1713304924, 23:02:04 DEBUG 1713304924, 23:02:04 DEBUG "0.027" 23:02:04 DEBUG 1713304924, 23:02:04 DEBUG "0" 23:02:04 DEBUG 1713304924, 23:02:04 DEBUG "0" 23:02:04 DEBUG 1713304924, 23:02:04 DEBUG 1713304924, 23:02:04 DEBUG 1713304924, 23:02:04 DEBUG "instance": "10.142.0.43:9100" 23:02:04 DEBUG 1713304924, 23:02:04 DEBUG "0.0007000000000005002" 23:02:04 DEBUG 1713304924, 23:02:04 DEBUG "19982040064" 23:02:04 DEBUG 1713304924, 23:02:04 DEBUG "1699639296" 23:02:04 DEBUG "2040.0666666666666" 23:02:04 DEBUG "0.000018618236745672884" 23:02:04 DEBUG 1713304924, 23:02:04 DEBUG "49440008.33333333" 23:02:04 DEBUG "49430904.93333333" 23:02:04 DEBUG "0" 23:02:04 DEBUG "0" 23:02:04 DEBUG "0" 23:02:04 DEBUG "2165.7666666666664" 23:02:04 DEBUG "2165.5333333333333" 23:02:04 DEBUG ] 23:02:04 DEBUG "0" 23:02:04 DEBUG ] 23:02:04 DEBUG "2170.3" 23:02:04 DEBUG ] 23:02:04 DEBUG "0" 23:02:04 DEBUG "0" 23:02:04 DEBUG "0" 23:02:04 DEBUG }, 23:02:04 DEBUG "0" 23:02:04 DEBUG ] 23:02:04 DEBUG "3673587.5" 23:02:04 DEBUG ] 23:02:04 DEBUG "0" 23:02:04 DEBUG ] 23:02:04 DEBUG ] 23:02:04 DEBUG ] 23:02:04 DEBUG "6.800740073806887" 23:02:04 DEBUG ] 23:02:04 DEBUG ] 23:02:04 DEBUG ] 23:02:04 DEBUG ] 23:02:04 DEBUG ] 23:02:04 DEBUG ] 23:02:04 DEBUG ] 23:02:04 DEBUG } 23:02:04 DEBUG ] 23:02:04 DEBUG } 23:02:04 DEBUG ] 23:02:04 DEBUG } 23:02:04 DEBUG ] 23:02:04 DEBUG ] 23:02:04 DEBUG ] 23:02:04 DEBUG "value": [ 23:02:04 DEBUG ] 23:02:04 DEBUG } 23:02:04 DEBUG ] 23:02:04 DEBUG } 23:02:04 DEBUG ] 23:02:04 DEBUG } 23:02:04 DEBUG } 23:02:04 DEBUG } 23:02:04 DEBUG ] 23:02:04 DEBUG } 23:02:04 DEBUG } 23:02:04 DEBUG } 23:02:04 DEBUG } 23:02:04 DEBUG } 23:02:04 DEBUG } 23:02:04 DEBUG } 23:02:04 DEBUG ] 23:02:04 DEBUG } 23:02:04 DEBUG ] 23:02:04 DEBUG } 23:02:04 DEBUG ] 23:02:04 DEBUG } 23:02:04 DEBUG } 23:02:04 DEBUG ] 23:02:04 DEBUG } 23:02:04 DEBUG } 23:02:04 DEBUG ] 23:02:04 DEBUG } 23:02:04 DEBUG ] 23:02:04 DEBUG } 23:02:04 DEBUG ] 23:02:04 DEBUG ] 23:02:04 DEBUG ] 23:02:04 DEBUG } 23:02:04 DEBUG ] 23:02:04 DEBUG ] 23:02:04 DEBUG ] 23:02:04 DEBUG ] 23:02:04 DEBUG ] 23:02:04 DEBUG ] 23:02:04 DEBUG ] 23:02:04 DEBUG } 23:02:04 DEBUG ] 23:02:04 DEBUG } 23:02:04 DEBUG ] 23:02:04 DEBUG } 23:02:04 DEBUG ] 23:02:04 DEBUG 1713304924, 23:02:04 DEBUG } 23:02:04 DEBUG ] 23:02:04 DEBUG ] 23:02:04 DEBUG } 23:02:04 DEBUG ] 23:02:04 DEBUG } 23:02:04 DEBUG ] 23:02:04 DEBUG } 23:02:04 DEBUG } 23:02:04 DEBUG } 23:02:04 DEBUG ] 23:02:04 DEBUG } 23:02:04 DEBUG } 23:02:04 DEBUG } 23:02:04 DEBUG } 23:02:04 DEBUG } 23:02:04 DEBUG } 23:02:04 DEBUG } 23:02:04 DEBUG } 23:02:04 DEBUG } 23:02:04 DEBUG } 23:02:04 DEBUG } 23:02:04 DEBUG } 23:02:04 DEBUG } 23:02:04 DEBUG "0.0006666666666666598" 23:02:04 DEBUG } 23:02:04 DEBUG } 23:02:04 DEBUG } 23:02:04 DEBUG } 23:02:04 DEBUG } 23:02:04 DEBUG } 23:02:04 DEBUG } 23:02:04 DEBUG } 23:02:04 DEBUG } 23:02:04 DEBUG } 23:02:04 DEBUG } 23:02:04 DEBUG } 23:02:04 DEBUG } 23:02:04 DEBUG } 23:02:04 DEBUG } 23:02:04 DEBUG } 23:02:04 DEBUG } 23:02:04 DEBUG } 23:02:04 DEBUG } 23:02:04 DEBUG } 23:02:04 DEBUG } 23:02:04 DEBUG ] 23:02:04 DEBUG } 23:02:04 DEBUG } 23:02:04 DEBUG } 23:02:04 DEBUG } 23:02:04 DEBUG } 23:02:04 DEBUG } 23:02:04 DEBUG ] 23:02:04 DEBUG } 23:02:04 DEBUG } 23:02:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713304978" 23:02:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304978" 23:02:58 ERROR [http_cmd]: ERROR 23:02:58 INFO [http_cmd]: http status code OK 23:02:58 ERROR --- status code --- 23:02:58 DEBUG --- status code --- 23:02:58 ERROR http status code is 400 (expected [200, 401, 403, 404]) 23:02:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:02:58 ERROR --- http response --- 23:02:58 DEBUG --- http response --- 23:02:58 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 23:02:58 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 23:02:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 23:03:01 INFO 23:03:01 INFO [loop_until]: kubectl --namespace=xlou top pods 23:03:01 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 23:03:01 INFO [loop_until]: OK (rc = 0) 23:03:01 DEBUG --- stdout --- 23:03:01 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 9Mi am-555f4db595-4lbss 12920m 8700Mi am-555f4db595-fz8sn 13110m 8577Mi am-555f4db595-kzxsp 13135m 8988Mi ds-cts-0 19558m 12654Mi ds-cts-1 19044m 13347Mi ds-cts-2 18533m 13302Mi ds-idrepo-0 5642m 5030Mi ds-idrepo-1 5396m 2908Mi ds-idrepo-2 5338m 3220Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 3m 1207Mi idm-b85f9d978-rl76f 3m 1225Mi lodemon-bc756c647-cr7x6 2m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-7c9b667974-9mr2s 7126m 1710Mi 23:03:01 DEBUG --- stderr --- 23:03:01 DEBUG 23:03:04 INFO 23:03:04 INFO [loop_until]: kubectl --namespace=xlou top node 23:03:04 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 23:03:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713304984" 23:03:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304984" 23:03:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304984" 23:03:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304984" 23:03:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304984" 23:03:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304984" 23:03:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713304984" 23:03:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304984" 23:03:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304984" 23:03:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713304984" 23:03:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304984" 23:03:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304984" 23:03:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304984" 23:03:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304984" 23:03:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304984" 23:03:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304984" 23:03:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304984" 23:03:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304984" 23:03:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304984" 23:03:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304984" 23:03:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304984" 23:03:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304984" 23:03:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713304984" 23:03:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713304984" 23:03:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713304984" 23:03:04 INFO [loop_until]: OK (rc = 0) 23:03:04 INFO [http_cmd]: http status code OK 23:03:04 INFO [http_cmd]: http status code OK 23:03:04 INFO [http_cmd]: http status code OK 23:03:04 INFO [http_cmd]: http status code OK 23:03:04 INFO [http_cmd]: http status code OK 23:03:04 INFO [http_cmd]: http status code OK 23:03:04 INFO [http_cmd]: http status code OK 23:03:04 INFO [http_cmd]: http status code OK 23:03:04 INFO [http_cmd]: http status code OK 23:03:04 INFO [http_cmd]: http status code OK 23:03:04 INFO [http_cmd]: http status code OK 23:03:04 INFO [http_cmd]: http status code OK 23:03:04 INFO [http_cmd]: http status code OK 23:03:04 INFO [http_cmd]: http status code OK 23:03:04 INFO [http_cmd]: http status code OK 23:03:04 INFO [http_cmd]: http status code OK 23:03:04 INFO [http_cmd]: http status code OK 23:03:04 DEBUG --- stdout --- 23:03:04 DEBUG --- status code --- 23:03:04 DEBUG --- status code --- 23:03:04 DEBUG --- status code --- 23:03:04 DEBUG --- status code --- 23:03:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:03:04 DEBUG --- status code --- 23:03:04 DEBUG --- status code --- 23:03:04 DEBUG --- status code --- 23:03:04 DEBUG --- status code --- 23:03:04 DEBUG --- status code --- 23:03:04 DEBUG --- status code --- 23:03:04 DEBUG --- status code --- 23:03:04 DEBUG --- status code --- 23:03:04 DEBUG --- status code --- 23:03:04 DEBUG --- status code --- 23:03:04 DEBUG --- status code --- 23:03:04 DEBUG --- status code --- 23:03:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:03:04 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 68m 0% 5095Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 19696m 65% 15906Mi 14% gke-xlou-cdm-default-pool-1b0769c8-nhlf 12652m 42% 10884Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 82m 0% 2659Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 5579m 18% 6572Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 20345m 68% 17136Mi 15% gke-xlou-cdm-default-pool-1b0769c8-xnmp 58m 0% 3150Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 13054m 43% 10528Mi 9% gke-xlou-cdm-ds-0f601751-hmm7 6008m 20% 5665Mi 5% gke-xlou-cdm-ds-0f601751-j488 19127m 64% 14912Mi 13% gke-xlou-cdm-ds-0f601751-mc5d 12108m 40% 3546Mi 3% gke-xlou-cdm-ds-0f601751-nj6q 13926m 46% 10899Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 6062m 20% 7359Mi 6% gke-xlou-cdm-frontend-a8771548-i82n 7344m 46% 7089Mi 12% 23:03:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:03:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:03:04 INFO [http_cmd]: http status code OK 23:03:04 DEBUG --- status code --- 23:03:04 DEBUG --- http response --- 23:03:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:03:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:03:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:03:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:03:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:03:04 INFO [http_cmd]: http status code OK 23:03:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:03:04 INFO [http_cmd]: http status code OK 23:03:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:03:04 INFO [http_cmd]: http status code OK 23:03:04 INFO [http_cmd]: http status code OK 23:03:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:03:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:03:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:03:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:03:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:03:04 DEBUG --- http response --- 23:03:04 DEBUG --- stderr --- 23:03:04 DEBUG --- http response --- 23:03:04 DEBUG --- http response --- 23:03:04 DEBUG --- status code --- 23:03:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:03:04 DEBUG --- http response --- 23:03:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:03:04 DEBUG --- http response --- 23:03:04 DEBUG --- http response --- 23:03:04 DEBUG --- http response --- 23:03:04 DEBUG --- http response --- 23:03:04 DEBUG --- status code --- 23:03:04 DEBUG --- http response --- 23:03:04 DEBUG --- status code --- 23:03:04 DEBUG --- http response --- 23:03:04 DEBUG --- status code --- 23:03:04 DEBUG --- status code --- 23:03:04 DEBUG --- http response --- 23:03:04 INFO [http_cmd]: http status code OK 23:03:04 DEBUG --- http response --- 23:03:04 INFO [http_cmd]: http status code OK 23:03:04 DEBUG --- http response --- 23:03:04 DEBUG --- http response --- 23:03:04 DEBUG --- http response --- 23:03:04 DEBUG 23:03:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:03:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:03:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:03:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:03:04 DEBUG --- http response --- 23:03:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 23:03:04 DEBUG ----- output ----- 23:03:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:03:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:03:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:03:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:03:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 23:03:04 INFO [http_cmd]: http status code OK 23:03:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 23:03:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:03:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:03:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:03:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:03:04 DEBUG --- status code --- 23:03:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:03:04 DEBUG --- status code --- 23:03:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:03:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:03:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:03:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 23:03:04 DEBUG ----- output ----- 23:03:04 DEBUG ----- output ----- 23:03:04 DEBUG ----- output ----- 23:03:04 DEBUG --- http response --- 23:03:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:03:04 DEBUG ----- output ----- 23:03:04 DEBUG { 23:03:04 DEBUG ----- output ----- 23:03:04 DEBUG ----- output ----- 23:03:04 DEBUG ----- output ----- 23:03:04 DEBUG --- http response --- 23:03:04 DEBUG ----- output ----- 23:03:04 DEBUG --- status code --- 23:03:04 DEBUG ----- output ----- 23:03:04 DEBUG --- http response --- 23:03:04 DEBUG ----- output ----- 23:03:04 DEBUG --- http response --- 23:03:04 DEBUG --- http response --- 23:03:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:03:04 DEBUG ----- output ----- 23:03:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:03:04 DEBUG ----- output ----- 23:03:04 DEBUG ----- output ----- 23:03:04 DEBUG ----- output ----- 23:03:04 DEBUG ----- output ----- 23:03:04 DEBUG { 23:03:04 DEBUG { 23:03:04 DEBUG { 23:03:04 DEBUG ----- output ----- 23:03:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 23:03:04 DEBUG { 23:03:04 DEBUG "status": "success", 23:03:04 DEBUG { 23:03:04 DEBUG { 23:03:04 DEBUG { 23:03:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:03:04 DEBUG { 23:03:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:03:04 DEBUG { 23:03:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 23:03:04 DEBUG { 23:03:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:03:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:03:04 DEBUG --- http response --- 23:03:04 DEBUG { 23:03:04 DEBUG --- http response --- 23:03:04 DEBUG { 23:03:04 DEBUG { 23:03:04 DEBUG { 23:03:04 DEBUG { 23:03:04 DEBUG "status": "success", 23:03:04 DEBUG "status": "success", 23:03:04 DEBUG "status": "success", 23:03:04 DEBUG { 23:03:04 DEBUG ----- output ----- 23:03:04 DEBUG "status": "success", 23:03:04 DEBUG "data": { 23:03:04 DEBUG "status": "success", 23:03:04 DEBUG "status": "success", 23:03:04 DEBUG "status": "success", 23:03:04 DEBUG ----- output ----- 23:03:04 DEBUG "status": "success", 23:03:04 DEBUG --- http response --- 23:03:04 DEBUG "status": "success", 23:03:04 DEBUG ----- output ----- 23:03:04 DEBUG "status": "success", 23:03:04 DEBUG ----- output ----- 23:03:04 DEBUG ----- output ----- 23:03:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:03:04 DEBUG "status": "success", 23:03:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:03:04 DEBUG "status": "success", 23:03:04 DEBUG "status": "success", 23:03:04 DEBUG "status": "success", 23:03:04 DEBUG "status": "success", 23:03:04 DEBUG "data": { 23:03:04 DEBUG "data": { 23:03:04 DEBUG "data": { 23:03:04 DEBUG "status": "success", 23:03:04 DEBUG { 23:03:04 DEBUG "data": { 23:03:04 DEBUG "resultType": "vector", 23:03:04 DEBUG "data": { 23:03:04 DEBUG "data": { 23:03:04 DEBUG "data": { 23:03:04 DEBUG { 23:03:04 DEBUG "data": { 23:03:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:03:04 DEBUG "data": { 23:03:04 DEBUG { 23:03:04 DEBUG "data": { 23:03:04 DEBUG { 23:03:04 DEBUG "status": "success", 23:03:04 DEBUG ----- output ----- 23:03:04 DEBUG "data": { 23:03:04 DEBUG ----- output ----- 23:03:04 DEBUG "data": { 23:03:04 DEBUG "data": { 23:03:04 DEBUG "data": { 23:03:04 DEBUG "data": { 23:03:04 DEBUG "resultType": "vector", 23:03:04 DEBUG "resultType": "vector", 23:03:04 DEBUG "resultType": "vector", 23:03:04 DEBUG "data": { 23:03:04 DEBUG "status": "success", 23:03:04 DEBUG "resultType": "vector", 23:03:04 DEBUG "result": [ 23:03:04 DEBUG "resultType": "vector", 23:03:04 DEBUG "resultType": "vector", 23:03:04 DEBUG "resultType": "vector", 23:03:04 DEBUG "status": "success", 23:03:04 DEBUG "resultType": "vector", 23:03:04 DEBUG ----- output ----- 23:03:04 DEBUG "resultType": "vector", 23:03:04 DEBUG "status": "success", 23:03:04 DEBUG "resultType": "vector", 23:03:04 DEBUG { 23:03:04 DEBUG "data": { 23:03:04 DEBUG { 23:03:04 DEBUG "resultType": "vector", 23:03:04 DEBUG { 23:03:04 DEBUG "resultType": "vector", 23:03:04 DEBUG "resultType": "vector", 23:03:04 DEBUG "resultType": "vector", 23:03:04 DEBUG "resultType": "vector", 23:03:04 DEBUG "result": [ 23:03:04 DEBUG "result": [ 23:03:04 DEBUG "result": [ 23:03:04 DEBUG "resultType": "vector", 23:03:04 DEBUG "result": [ 23:03:04 DEBUG "result": [ 23:03:04 DEBUG { 23:03:04 DEBUG "result": [ 23:03:04 DEBUG "result": [ 23:03:04 DEBUG "result": [ 23:03:04 DEBUG "data": { 23:03:04 DEBUG "result": [ 23:03:04 DEBUG { 23:03:04 DEBUG "result": [ 23:03:04 DEBUG "data": { 23:03:04 DEBUG "result": [ 23:03:04 DEBUG "status": "success", 23:03:04 DEBUG "resultType": "vector", 23:03:04 DEBUG "status": "success", 23:03:04 DEBUG "result": [ 23:03:04 DEBUG "status": "success", 23:03:04 DEBUG "result": [ 23:03:04 DEBUG "result": [ 23:03:04 DEBUG "result": [ 23:03:04 DEBUG "result": [ 23:03:04 DEBUG { 23:03:04 DEBUG { 23:03:04 DEBUG { 23:03:04 DEBUG "data": { 23:03:04 DEBUG { 23:03:04 DEBUG { 23:03:04 DEBUG "metric": { 23:03:04 DEBUG { 23:03:04 DEBUG "metric": { 23:03:04 DEBUG { 23:03:04 DEBUG "resultType": "vector", 23:03:04 DEBUG { 23:03:04 DEBUG "status": "success", 23:03:04 DEBUG { 23:03:04 DEBUG "resultType": "vector", 23:03:04 DEBUG { 23:03:04 DEBUG "data": { 23:03:04 DEBUG "result": [ 23:03:04 DEBUG "data": { 23:03:04 DEBUG { 23:03:04 DEBUG "data": { 23:03:04 DEBUG { 23:03:04 DEBUG { 23:03:04 DEBUG { 23:03:04 DEBUG { 23:03:04 DEBUG "metric": { 23:03:04 DEBUG "metric": { 23:03:04 DEBUG "metric": { 23:03:04 DEBUG "resultType": "vector", 23:03:04 DEBUG "metric": { 23:03:04 DEBUG "metric": { 23:03:04 DEBUG "instance": "10.142.0.107:9100" 23:03:04 DEBUG { 23:03:04 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 23:03:04 DEBUG "metric": { 23:03:04 DEBUG "result": [ 23:03:04 DEBUG "metric": { 23:03:04 DEBUG "data": { 23:03:04 DEBUG "metric": { 23:03:04 DEBUG "result": [ 23:03:04 DEBUG "metric": { 23:03:04 DEBUG "resultType": "vector", 23:03:04 DEBUG { 23:03:04 DEBUG "resultType": "vector", 23:03:04 DEBUG "metric": { 23:03:04 DEBUG "resultType": "vector", 23:03:04 DEBUG "metric": { 23:03:04 DEBUG "metric": { 23:03:04 DEBUG "metric": { 23:03:04 DEBUG "metric": { 23:03:04 DEBUG "pod": "am-555f4db595-fz8sn" 23:03:04 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 23:03:04 DEBUG "pod": "ds-idrepo-2" 23:03:04 DEBUG "result": [ 23:03:04 DEBUG "pod": "ds-idrepo-2" 23:03:04 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 23:03:04 DEBUG }, 23:03:04 DEBUG "metric": { 23:03:04 DEBUG }, 23:03:04 DEBUG "instance": "10.142.0.107:9100" 23:03:04 DEBUG { 23:03:04 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 23:03:04 DEBUG "resultType": "vector", 23:03:04 DEBUG "pod": "ds-idrepo-2" 23:03:04 DEBUG { 23:03:04 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 23:03:04 DEBUG "result": [ 23:03:04 DEBUG "metric": { 23:03:04 DEBUG "result": [ 23:03:04 DEBUG "pod": "am-555f4db595-fz8sn" 23:03:04 DEBUG "result": [ 23:03:04 DEBUG "pod": "am-555f4db595-fz8sn" 23:03:04 DEBUG "pod": "am-555f4db595-fz8sn" 23:03:04 DEBUG "pod": "ds-idrepo-2" 23:03:04 DEBUG "pod": "ds-idrepo-2" 23:03:04 DEBUG }, 23:03:04 DEBUG }, 23:03:04 DEBUG }, 23:03:04 DEBUG { 23:03:04 DEBUG }, 23:03:04 DEBUG }, 23:03:04 DEBUG "value": [ 23:03:04 DEBUG "instance": "10.142.0.107:9100" 23:03:04 DEBUG "value": [ 23:03:04 DEBUG }, 23:03:04 DEBUG "metric": { 23:03:04 DEBUG }, 23:03:04 DEBUG "result": [ 23:03:04 DEBUG }, 23:03:04 DEBUG "metric": { 23:03:04 DEBUG }, 23:03:04 DEBUG { 23:03:04 DEBUG "pod": "am-555f4db595-fz8sn" 23:03:04 DEBUG { 23:03:04 DEBUG }, 23:03:04 DEBUG { 23:03:04 DEBUG }, 23:03:04 DEBUG }, 23:03:04 DEBUG }, 23:03:04 DEBUG }, 23:03:04 DEBUG "value": [ 23:03:04 DEBUG "value": [ 23:03:04 DEBUG "value": [ 23:03:04 DEBUG "metric": { 23:03:04 DEBUG "value": [ 23:03:04 DEBUG "value": [ 23:03:04 DEBUG 23:03:04 DEBUG }, 23:03:04 DEBUG 23:03:04 DEBUG "value": [ 23:03:04 DEBUG "instance": "10.142.0.107:9100" 23:03:04 DEBUG "value": [ 23:03:04 DEBUG { 23:03:04 DEBUG "value": [ 23:03:04 DEBUG "pod": "ds-idrepo-2" 23:03:04 DEBUG "value": [ 23:03:04 DEBUG "metric": { 23:03:04 DEBUG }, 23:03:04 DEBUG "metric": { 23:03:04 DEBUG "value": [ 23:03:04 DEBUG "metric": { 23:03:04 DEBUG "value": [ 23:03:04 DEBUG "value": [ 23:03:04 DEBUG "value": [ 23:03:04 DEBUG "value": [ 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG "pod": "ds-idrepo-2" 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG "value": [ 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG }, 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG }, 23:03:04 DEBUG "value": [ 23:03:04 DEBUG "pod": "am-555f4db595-fz8sn" 23:03:04 DEBUG 23:03:04 DEBUG "pod": "am-555f4db595-fz8sn" 23:03:04 DEBUG 23:03:04 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG }, 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG "value": [ 23:03:04 DEBUG "metric": { 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG "value": [ 23:03:04 DEBUG }, 23:03:04 DEBUG 23:03:04 DEBUG }, 23:03:04 DEBUG 23:03:04 DEBUG }, 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG [...] 23:03:04 DEBUG 23:03:04 DEBUG "value": [ 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG [...] 23:03:04 DEBUG 23:03:04 DEBUG [...] 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG "instance": "10.142.0.107:9100" 23:03:04 DEBUG [...] 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG "value": [ 23:03:04 DEBUG 23:03:04 DEBUG "value": [ 23:03:04 DEBUG 23:03:04 DEBUG "value": [ 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG [...] 23:03:04 DEBUG 23:03:04 DEBUG [...] 23:03:04 DEBUG [...] 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG [...] 23:03:04 DEBUG 23:03:04 DEBUG }, 23:03:04 DEBUG 23:03:04 DEBUG [...] 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG [...] 23:03:04 DEBUG 23:03:04 DEBUG [...] 23:03:04 DEBUG 23:03:04 DEBUG [...] 23:03:04 DEBUG [...] 23:03:04 DEBUG 23:03:04 DEBUG [...] 23:03:04 DEBUG [...] 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG [...] 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG "value": [ 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG [...] 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG [...] 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG [...] 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG "pod": "idm-b85f9d978-j5pbm" 23:03:04 DEBUG 23:03:04 DEBUG [...] 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG "pod": "am-555f4db595-kzxsp" 23:03:04 DEBUG 23:03:04 DEBUG [...] 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG "pod": "idm-b85f9d978-j5pbm" 23:03:04 DEBUG }, 23:03:04 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG }, 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG [...] 23:03:04 DEBUG 23:03:04 DEBUG [...] 23:03:04 DEBUG 23:03:04 DEBUG [...] 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG "pod": "am-555f4db595-kzxsp" 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG }, 23:03:04 DEBUG "pod": "ds-idrepo-0" 23:03:04 DEBUG 23:03:04 DEBUG "pod": "ds-idrepo-0" 23:03:04 DEBUG "instance": "10.142.0.43:9100" 23:03:04 DEBUG 23:03:04 DEBUG "value": [ 23:03:04 DEBUG }, 23:03:04 DEBUG "instance": "10.142.0.43:9100" 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG "value": [ 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG "pod": "idm-b85f9d978-j5pbm" 23:03:04 DEBUG 23:03:04 DEBUG "pod": "am-555f4db595-kzxsp" 23:03:04 DEBUG 23:03:04 DEBUG "pod": "am-555f4db595-kzxsp" 23:03:04 DEBUG 23:03:04 DEBUG }, 23:03:04 DEBUG "pod": "ds-idrepo-0" 23:03:04 DEBUG "pod": "idm-b85f9d978-j5pbm" 23:03:04 DEBUG "value": [ 23:03:04 DEBUG }, 23:03:04 DEBUG 23:03:04 DEBUG }, 23:03:04 DEBUG }, 23:03:04 DEBUG 23:03:04 DEBUG 1713304984, 23:03:04 DEBUG "value": [ 23:03:04 DEBUG }, 23:03:04 DEBUG 23:03:04 DEBUG [...] 23:03:04 DEBUG "pod": "ds-cts-2" 23:03:04 DEBUG 1713304984, 23:03:04 DEBUG "pod": "ds-idrepo-0" 23:03:04 DEBUG 23:03:04 DEBUG 23:03:04 DEBUG }, 23:03:04 DEBUG 23:03:04 DEBUG }, 23:03:04 DEBUG 23:03:04 DEBUG }, 23:03:04 DEBUG "pod": "ds-idrepo-0" 23:03:04 DEBUG "value": [ 23:03:04 DEBUG }, 23:03:04 DEBUG }, 23:03:04 DEBUG 1713304984, 23:03:04 DEBUG "value": [ 23:03:04 DEBUG 1713304984, 23:03:04 DEBUG 23:03:04 DEBUG "value": [ 23:03:04 DEBUG "instance": "10.142.0.43:9100" 23:03:04 DEBUG "41748524.69561222" 23:03:04 DEBUG 1713304984, 23:03:04 DEBUG "value": [ 23:03:04 DEBUG "instance": "10.142.0.43:9100" 23:03:04 DEBUG 23:03:04 DEBUG }, 23:03:04 DEBUG "0.002993698926849091" 23:03:04 DEBUG }, 23:03:04 DEBUG "pod": "am-555f4db595-kzxsp" 23:03:04 DEBUG 23:03:04 DEBUG "value": [ 23:03:04 DEBUG 23:03:04 DEBUG "value": [ 23:03:04 DEBUG 23:03:04 DEBUG "value": [ 23:03:04 DEBUG }, 23:03:04 DEBUG 1713304984, 23:03:04 DEBUG "value": [ 23:03:04 DEBUG "value": [ 23:03:04 DEBUG "0.027" 23:03:04 DEBUG "value": [ 23:03:04 DEBUG "0" 23:03:04 DEBUG "pod": "ds-idrepo-0" 23:03:04 DEBUG 1713304984, 23:03:04 DEBUG }, 23:03:04 DEBUG ] 23:03:04 DEBUG "1703641088" 23:03:04 DEBUG 1713304984, 23:03:04 DEBUG }, 23:03:04 DEBUG 23:03:04 DEBUG "value": [ 23:03:04 DEBUG ] 23:03:04 DEBUG "value": [ 23:03:04 DEBUG }, 23:03:04 DEBUG "pod": "am-555f4db595-kzxsp" 23:03:04 DEBUG 1713304984, 23:03:04 DEBUG "pod": "am-555f4db595-kzxsp" 23:03:04 DEBUG 1713304984, 23:03:04 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 23:03:04 DEBUG 1713304984, 23:03:04 DEBUG "value": [ 23:03:04 DEBUG "2164.7" 23:03:04 DEBUG 1713304984, 23:03:04 DEBUG 1713304984, 23:03:04 DEBUG ] 23:03:04 DEBUG 1713304984, 23:03:04 DEBUG ] 23:03:04 DEBUG }, 23:03:04 DEBUG "0.0005666666666665302" 23:03:04 DEBUG "value": [ 23:03:04 DEBUG } 23:03:04 DEBUG ] 23:03:04 DEBUG "48860122.46666667" 23:03:04 DEBUG "value": [ 23:03:04 DEBUG 23:03:04 DEBUG 1713304984, 23:03:04 DEBUG } 23:03:04 DEBUG 1713304984, 23:03:04 DEBUG "value": [ 23:03:04 DEBUG }, 23:03:04 DEBUG "403.26868169735155" 23:03:04 DEBUG }, 23:03:04 DEBUG "2164.5333333333333" 23:03:04 DEBUG }, 23:03:04 DEBUG "2164.5666666666666" 23:03:04 DEBUG 1713304984, 23:03:04 DEBUG ] 23:03:04 DEBUG "0" 23:03:04 DEBUG "0" 23:03:04 DEBUG } 23:03:04 DEBUG "0" 23:03:04 DEBUG } 23:03:04 DEBUG "value": [ 23:03:04 DEBUG ] 23:03:04 DEBUG 1713304984, 23:03:04 DEBUG ] 23:03:04 DEBUG } 23:03:04 DEBUG ] 23:03:04 DEBUG 1713304984, 23:03:04 DEBUG "instance": "10.142.0.43:9100" 23:03:04 DEBUG "0" 23:03:04 DEBUG ] 23:03:04 DEBUG "0" 23:03:04 DEBUG 1713304984, 23:03:04 DEBUG "value": [ 23:03:04 DEBUG ] 23:03:04 DEBUG "value": [ 23:03:04 DEBUG ] 23:03:04 DEBUG "value": [ 23:03:04 DEBUG ] 23:03:04 DEBUG "0" 23:03:04 DEBUG } 23:03:04 DEBUG ] 23:03:04 DEBUG ] 23:03:04 DEBUG ] 23:03:04 DEBUG ] 23:03:04 DEBUG ] 23:03:04 DEBUG 1713304984, 23:03:04 DEBUG } 23:03:04 DEBUG "19982806016" 23:03:04 DEBUG } 23:03:04 DEBUG ] 23:03:04 DEBUG } 23:03:04 DEBUG "48869065.83333333" 23:03:04 DEBUG }, 23:03:04 DEBUG ] 23:03:04 DEBUG } 23:03:04 DEBUG ] 23:03:04 DEBUG "3683246" 23:03:04 DEBUG 1713304984, 23:03:04 DEBUG } 23:03:04 DEBUG 1713304984, 23:03:04 DEBUG } 23:03:04 DEBUG 1713304984, 23:03:04 DEBUG } 23:03:04 DEBUG ] 23:03:04 DEBUG ] 23:03:04 DEBUG } 23:03:04 DEBUG } 23:03:04 DEBUG } 23:03:04 DEBUG } 23:03:04 DEBUG } 23:03:04 DEBUG "0" 23:03:04 DEBUG ] 23:03:04 DEBUG ] 23:03:04 DEBUG } 23:03:04 DEBUG } 23:03:04 DEBUG ] 23:03:04 DEBUG ] 23:03:04 DEBUG "value": [ 23:03:04 DEBUG } 23:03:04 DEBUG } 23:03:04 DEBUG } 23:03:04 DEBUG ] 23:03:04 DEBUG "2004.1666666666667" 23:03:04 DEBUG ] 23:03:04 DEBUG "0" 23:03:04 DEBUG ] 23:03:04 DEBUG "6.887967683769929" 23:03:04 DEBUG ] 23:03:04 DEBUG } 23:03:04 DEBUG } 23:03:04 DEBUG ] 23:03:04 DEBUG ] 23:03:04 DEBUG } 23:03:04 DEBUG ] 23:03:04 DEBUG } 23:03:04 DEBUG ] 23:03:04 DEBUG } 23:03:04 DEBUG } 23:03:04 DEBUG } 23:03:04 DEBUG } 23:03:04 DEBUG } 23:03:04 DEBUG 1713304984, 23:03:04 DEBUG ] 23:03:04 DEBUG ] 23:03:04 DEBUG } 23:03:04 DEBUG ] 23:03:04 DEBUG } 23:03:04 DEBUG ] 23:03:04 DEBUG } 23:03:04 DEBUG ] 23:03:04 DEBUG } 23:03:04 DEBUG ] 23:03:04 DEBUG } 23:03:04 DEBUG } 23:03:04 DEBUG } 23:03:04 DEBUG } 23:03:04 DEBUG } 23:03:04 DEBUG } 23:03:04 DEBUG ] 23:03:04 DEBUG } 23:03:04 DEBUG ] 23:03:04 DEBUG "0.00033333333333333365" 23:03:04 DEBUG } 23:03:04 DEBUG } 23:03:04 DEBUG ] 23:03:04 DEBUG } 23:03:04 DEBUG } 23:03:04 DEBUG } 23:03:04 DEBUG } 23:03:04 DEBUG } 23:03:04 DEBUG } 23:03:04 DEBUG } 23:03:04 DEBUG } 23:03:04 DEBUG } 23:03:04 DEBUG ] 23:03:04 DEBUG } 23:03:04 DEBUG } 23:03:04 DEBUG ] 23:03:04 DEBUG } 23:03:04 DEBUG } 23:03:04 DEBUG } 23:03:04 DEBUG ] 23:03:04 DEBUG ] 23:03:04 DEBUG ] 23:03:04 DEBUG } 23:03:04 DEBUG } 23:03:04 DEBUG } 23:03:04 DEBUG } 23:03:04 DEBUG } 23:03:04 DEBUG } 23:03:04 DEBUG } 23:03:04 DEBUG } 23:03:04 DEBUG } 23:03:04 DEBUG } 23:03:04 DEBUG } 23:03:04 DEBUG ] 23:03:04 DEBUG } 23:03:04 DEBUG } 23:03:04 DEBUG } 23:03:04 DEBUG } 23:03:04 DEBUG } 23:03:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713305038" 23:03:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305038" 23:03:58 INFO [http_cmd]: http status code OK 23:03:58 DEBUG --- status code --- 23:03:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:03:58 DEBUG --- http response --- 23:03:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 23:03:58 ERROR [http_cmd]: ERROR 23:03:58 ERROR --- status code --- 23:03:58 ERROR http status code is 400 (expected [200, 401, 403, 404]) 23:03:58 ERROR --- http response --- 23:03:58 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 23:03:58 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 23:04:01 INFO 23:04:01 INFO [loop_until]: kubectl --namespace=xlou top pods 23:04:01 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 23:04:01 INFO [loop_until]: OK (rc = 0) 23:04:01 DEBUG --- stdout --- 23:04:01 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 9Mi am-555f4db595-4lbss 13333m 8690Mi am-555f4db595-fz8sn 13201m 8583Mi am-555f4db595-kzxsp 13960m 8982Mi ds-cts-0 18974m 12858Mi ds-cts-1 18797m 13263Mi ds-cts-2 20224m 13386Mi ds-idrepo-0 6038m 5030Mi ds-idrepo-1 5424m 2900Mi ds-idrepo-2 5174m 3220Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 4m 1208Mi idm-b85f9d978-rl76f 3m 1226Mi lodemon-bc756c647-cr7x6 2m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-7c9b667974-9mr2s 6605m 1726Mi 23:04:01 DEBUG --- stderr --- 23:04:01 DEBUG 23:04:04 INFO 23:04:04 INFO [loop_until]: kubectl --namespace=xlou top node 23:04:04 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 23:04:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305044" 23:04:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305044" 23:04:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713305044" 23:04:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305044" 23:04:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305044" 23:04:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713305044" 23:04:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713305044" 23:04:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713305044" 23:04:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305044" 23:04:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305044" 23:04:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713305044" 23:04:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305044" 23:04:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305044" 23:04:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305044" 23:04:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305044" 23:04:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305044" 23:04:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305044" 23:04:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713305044" 23:04:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713305044" 23:04:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305044" 23:04:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305044" 23:04:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713305044" 23:04:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713305044" 23:04:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305044" 23:04:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305044" 23:04:04 INFO [http_cmd]: http status code OK 23:04:04 INFO [http_cmd]: http status code OK 23:04:04 INFO [http_cmd]: http status code OK 23:04:04 INFO [loop_until]: OK (rc = 0) 23:04:04 INFO [http_cmd]: http status code OK 23:04:04 INFO [http_cmd]: http status code OK 23:04:04 INFO [http_cmd]: http status code OK 23:04:04 INFO [http_cmd]: http status code OK 23:04:04 INFO [http_cmd]: http status code OK 23:04:04 INFO [http_cmd]: http status code OK 23:04:04 INFO [http_cmd]: http status code OK 23:04:04 INFO [http_cmd]: http status code OK 23:04:04 INFO [http_cmd]: http status code OK 23:04:04 INFO [http_cmd]: http status code OK 23:04:04 INFO [http_cmd]: http status code OK 23:04:04 INFO [http_cmd]: http status code OK 23:04:04 DEBUG --- status code --- 23:04:04 DEBUG --- status code --- 23:04:04 DEBUG --- status code --- 23:04:04 DEBUG --- stdout --- 23:04:04 DEBUG --- status code --- 23:04:04 DEBUG --- status code --- 23:04:04 DEBUG --- status code --- 23:04:04 INFO [http_cmd]: http status code OK 23:04:04 INFO [http_cmd]: http status code OK 23:04:04 INFO [http_cmd]: http status code OK 23:04:04 INFO [http_cmd]: http status code OK 23:04:04 DEBUG --- status code --- 23:04:04 INFO [http_cmd]: http status code OK 23:04:04 INFO [http_cmd]: http status code OK 23:04:04 DEBUG --- status code --- 23:04:04 DEBUG --- status code --- 23:04:04 DEBUG --- status code --- 23:04:04 DEBUG --- status code --- 23:04:04 DEBUG --- status code --- 23:04:04 DEBUG --- status code --- 23:04:04 INFO [http_cmd]: http status code OK 23:04:04 DEBUG --- status code --- 23:04:04 DEBUG --- status code --- 23:04:04 INFO [http_cmd]: http status code OK 23:04:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:04:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:04:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:04:04 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 62m 0% 5096Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 18652m 62% 15855Mi 14% gke-xlou-cdm-default-pool-1b0769c8-nhlf 13474m 45% 10817Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 82m 0% 2661Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 5633m 18% 6579Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 20368m 68% 17279Mi 15% gke-xlou-cdm-default-pool-1b0769c8-xnmp 62m 0% 3127Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 13077m 43% 10540Mi 9% gke-xlou-cdm-ds-0f601751-hmm7 6237m 20% 5653Mi 5% gke-xlou-cdm-ds-0f601751-j488 18356m 61% 15049Mi 13% gke-xlou-cdm-ds-0f601751-mc5d 11681m 39% 3510Mi 3% gke-xlou-cdm-ds-0f601751-nj6q 13388m 44% 10945Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 5995m 20% 7350Mi 6% gke-xlou-cdm-frontend-a8771548-i82n 6934m 43% 7092Mi 12% 23:04:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:04:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:04:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:04:04 DEBUG --- status code --- 23:04:04 DEBUG --- status code --- 23:04:04 DEBUG --- status code --- 23:04:04 DEBUG --- status code --- 23:04:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:04:04 DEBUG --- status code --- 23:04:04 DEBUG --- status code --- 23:04:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:04:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:04:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:04:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:04:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:04:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:04:04 DEBUG --- status code --- 23:04:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:04:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:04:04 DEBUG --- status code --- 23:04:04 DEBUG --- http response --- 23:04:04 INFO [http_cmd]: http status code OK 23:04:04 DEBUG --- http response --- 23:04:04 DEBUG --- http response --- 23:04:04 DEBUG --- stderr --- 23:04:04 DEBUG --- http response --- 23:04:04 DEBUG --- http response --- 23:04:04 DEBUG --- http response --- 23:04:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:04:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:04:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:04:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:04:04 DEBUG --- http response --- 23:04:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:04:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:04:04 DEBUG --- http response --- 23:04:04 DEBUG --- http response --- 23:04:04 DEBUG --- http response --- 23:04:04 DEBUG --- http response --- 23:04:04 DEBUG --- http response --- 23:04:04 DEBUG --- http response --- 23:04:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:04:04 DEBUG --- http response --- 23:04:04 DEBUG --- http response --- 23:04:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:04:04 INFO [http_cmd]: http status code OK 23:04:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:04:04 DEBUG --- status code --- 23:04:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 23:04:04 DEBUG 23:04:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:04:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:04:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:04:04 DEBUG --- http response --- 23:04:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 23:04:04 DEBUG --- http response --- 23:04:04 DEBUG --- http response --- 23:04:04 DEBUG --- http response --- 23:04:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 23:04:04 DEBUG --- http response --- 23:04:04 DEBUG --- http response --- 23:04:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:04:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:04:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:04:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 23:04:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:04:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 23:04:04 DEBUG --- http response --- 23:04:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:04:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:04:04 DEBUG --- http response --- 23:04:04 DEBUG --- status code --- 23:04:04 DEBUG ----- output ----- 23:04:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:04:04 DEBUG ----- output ----- 23:04:04 DEBUG ----- output ----- 23:04:04 DEBUG ----- output ----- 23:04:04 DEBUG ----- output ----- 23:04:04 DEBUG ----- output ----- 23:04:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:04:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:04:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:04:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:04:04 DEBUG ----- output ----- 23:04:04 DEBUG ----- output ----- 23:04:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:04:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:04:04 DEBUG ----- output ----- 23:04:04 DEBUG ----- output ----- 23:04:04 DEBUG ----- output ----- 23:04:04 DEBUG ----- output ----- 23:04:04 DEBUG ----- output ----- 23:04:04 DEBUG ----- output ----- 23:04:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:04:04 DEBUG ----- output ----- 23:04:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:04:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:04:04 DEBUG { 23:04:04 DEBUG --- http response --- 23:04:04 DEBUG { 23:04:04 DEBUG { 23:04:04 DEBUG { 23:04:04 DEBUG { 23:04:04 DEBUG { 23:04:04 DEBUG ----- output ----- 23:04:04 DEBUG ----- output ----- 23:04:04 DEBUG ----- output ----- 23:04:04 DEBUG ----- output ----- 23:04:04 DEBUG { 23:04:04 DEBUG { 23:04:04 DEBUG ----- output ----- 23:04:04 DEBUG ----- output ----- 23:04:04 DEBUG { 23:04:04 DEBUG { 23:04:04 DEBUG { 23:04:04 DEBUG { 23:04:04 DEBUG { 23:04:04 DEBUG { 23:04:04 DEBUG ----- output ----- 23:04:04 DEBUG { 23:04:04 DEBUG ----- output ----- 23:04:04 DEBUG --- http response --- 23:04:04 DEBUG "status": "success", 23:04:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:04:04 DEBUG "status": "success", 23:04:04 DEBUG "status": "success", 23:04:04 DEBUG "status": "success", 23:04:04 DEBUG "status": "success", 23:04:04 DEBUG "status": "success", 23:04:04 DEBUG { 23:04:04 DEBUG { 23:04:04 DEBUG { 23:04:04 DEBUG { 23:04:04 DEBUG "status": "success", 23:04:04 DEBUG "status": "success", 23:04:04 DEBUG { 23:04:04 DEBUG { 23:04:04 DEBUG "status": "success", 23:04:04 DEBUG "status": "success", 23:04:04 DEBUG "status": "success", 23:04:04 DEBUG "status": "success", 23:04:04 DEBUG "status": "success", 23:04:04 DEBUG "status": "success", 23:04:04 DEBUG { 23:04:04 DEBUG "status": "success", 23:04:04 DEBUG { 23:04:04 DEBUG "data": { 23:04:04 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 23:04:04 DEBUG ----- output ----- 23:04:04 DEBUG "data": { 23:04:04 DEBUG "data": { 23:04:04 DEBUG "resultType": "vector", 23:04:04 DEBUG "data": { 23:04:04 DEBUG "data": { 23:04:04 DEBUG "status": "success", 23:04:04 DEBUG "data": { 23:04:04 DEBUG "status": "success", 23:04:04 DEBUG "status": "success", 23:04:04 DEBUG "data": { 23:04:04 DEBUG "data": { 23:04:04 DEBUG "status": "success", 23:04:04 DEBUG "status": "success", 23:04:04 DEBUG "data": { 23:04:04 DEBUG "data": { 23:04:04 DEBUG "data": { 23:04:04 DEBUG "data": { 23:04:04 DEBUG "data": { 23:04:04 DEBUG "data": { 23:04:04 DEBUG "status": "success", 23:04:04 DEBUG "data": { 23:04:04 DEBUG "status": "success", 23:04:04 DEBUG "resultType": "vector", 23:04:04 DEBUG ----- output ----- 23:04:04 DEBUG { 23:04:04 DEBUG "resultType": "vector", 23:04:04 DEBUG "result": [ 23:04:04 DEBUG "resultType": "vector", 23:04:04 DEBUG "resultType": "vector", 23:04:04 DEBUG "status": "success", 23:04:04 DEBUG "resultType": "vector", 23:04:04 DEBUG "data": { 23:04:04 DEBUG "data": { 23:04:04 DEBUG "resultType": "vector", 23:04:04 DEBUG "resultType": "vector", 23:04:04 DEBUG "data": { 23:04:04 DEBUG "data": { 23:04:04 DEBUG "resultType": "vector", 23:04:04 DEBUG "resultType": "vector", 23:04:04 DEBUG "resultType": "vector", 23:04:04 DEBUG "resultType": "vector", 23:04:04 DEBUG "resultType": "vector", 23:04:04 DEBUG "resultType": "vector", 23:04:04 DEBUG "data": { 23:04:04 DEBUG "resultType": "vector", 23:04:04 DEBUG "data": { 23:04:04 DEBUG "data": { 23:04:04 DEBUG "result": [ 23:04:04 DEBUG { 23:04:04 DEBUG "status": "success", 23:04:04 DEBUG "result": [ 23:04:04 DEBUG { 23:04:04 DEBUG "result": [ 23:04:04 DEBUG "result": [ 23:04:04 DEBUG "data": { 23:04:04 DEBUG "result": [ 23:04:04 DEBUG "resultType": "vector", 23:04:04 DEBUG "resultType": "vector", 23:04:04 DEBUG "result": [ 23:04:04 DEBUG "result": [ 23:04:04 DEBUG "resultType": "vector", 23:04:04 DEBUG "resultType": "vector", 23:04:04 DEBUG "result": [ 23:04:04 DEBUG "result": [ 23:04:04 DEBUG "result": [ 23:04:04 DEBUG "result": [ 23:04:04 DEBUG "result": [ 23:04:04 DEBUG "result": [ 23:04:04 DEBUG "resultType": "vector", 23:04:04 DEBUG "result": [ 23:04:04 DEBUG "resultType": "vector", 23:04:04 DEBUG "resultType": "vector", 23:04:04 DEBUG { 23:04:04 DEBUG "status": "success", 23:04:04 DEBUG "data": { 23:04:04 DEBUG { 23:04:04 DEBUG "metric": { 23:04:04 DEBUG { 23:04:04 DEBUG { 23:04:04 DEBUG "resultType": "vector", 23:04:04 DEBUG { 23:04:04 DEBUG "result": [ 23:04:04 DEBUG "result": [ 23:04:04 DEBUG { 23:04:04 DEBUG { 23:04:04 DEBUG "result": [ 23:04:04 DEBUG "result": [ 23:04:04 DEBUG { 23:04:04 DEBUG { 23:04:04 DEBUG { 23:04:04 DEBUG { 23:04:04 DEBUG { 23:04:04 DEBUG { 23:04:04 DEBUG "result": [ 23:04:04 DEBUG { 23:04:04 DEBUG "result": [ 23:04:04 DEBUG "result": [ 23:04:04 DEBUG "metric": { 23:04:04 DEBUG "data": { 23:04:04 DEBUG "resultType": "vector", 23:04:04 DEBUG "metric": { 23:04:04 DEBUG "instance": "10.142.0.107:9100" 23:04:04 DEBUG "metric": { 23:04:04 DEBUG "metric": { 23:04:04 DEBUG "result": [ 23:04:04 DEBUG "metric": { 23:04:04 DEBUG { 23:04:04 DEBUG "metric": { 23:04:04 DEBUG { 23:04:04 DEBUG "metric": { 23:04:04 DEBUG { 23:04:04 DEBUG { 23:04:04 DEBUG "metric": { 23:04:04 DEBUG "metric": { 23:04:04 DEBUG "metric": { 23:04:04 DEBUG "metric": { 23:04:04 DEBUG "metric": { 23:04:04 DEBUG "metric": { 23:04:04 DEBUG { 23:04:04 DEBUG "metric": { 23:04:04 DEBUG { 23:04:04 DEBUG { 23:04:04 DEBUG "pod": "am-555f4db595-fz8sn" 23:04:04 DEBUG "resultType": "vector", 23:04:04 DEBUG "result": [ 23:04:04 DEBUG "pod": "ds-cts-1" 23:04:04 DEBUG }, 23:04:04 DEBUG "pod": "ds-idrepo-2" 23:04:04 DEBUG "pod": "ds-cts-1" 23:04:04 DEBUG { 23:04:04 DEBUG "instance": "10.142.0.107:9100" 23:04:04 DEBUG "metric": { 23:04:04 DEBUG "pod": "ds-idrepo-2" 23:04:04 DEBUG "metric": { 23:04:04 DEBUG "instance": "10.142.0.107:9100" 23:04:04 DEBUG "metric": { 23:04:04 DEBUG "metric": { 23:04:04 DEBUG "pod": "am-555f4db595-fz8sn" 23:04:04 DEBUG "pod": "ds-cts-1" 23:04:04 DEBUG "pod": "ds-idrepo-2" 23:04:04 DEBUG "pod": "am-555f4db595-fz8sn" 23:04:04 DEBUG "pod": "ds-idrepo-2" 23:04:04 DEBUG "pod": "am-555f4db595-fz8sn" 23:04:04 DEBUG "metric": { 23:04:04 DEBUG "pod": "ds-cts-1" 23:04:04 DEBUG "metric": { 23:04:04 DEBUG "metric": { 23:04:04 DEBUG }, 23:04:04 DEBUG "result": [ 23:04:04 DEBUG { 23:04:04 DEBUG }, 23:04:04 DEBUG "value": [ 23:04:04 DEBUG }, 23:04:04 DEBUG }, 23:04:04 DEBUG "metric": { 23:04:04 DEBUG }, 23:04:04 DEBUG "pod": "am-555f4db595-fz8sn" 23:04:04 DEBUG }, 23:04:04 DEBUG "pod": "ds-idrepo-2" 23:04:04 DEBUG }, 23:04:04 DEBUG "pod": "am-555f4db595-fz8sn" 23:04:04 DEBUG "pod": "ds-idrepo-2" 23:04:04 DEBUG }, 23:04:04 DEBUG }, 23:04:04 DEBUG }, 23:04:04 DEBUG }, 23:04:04 DEBUG }, 23:04:04 DEBUG }, 23:04:04 DEBUG "instance": "10.142.0.107:9100" 23:04:04 DEBUG }, 23:04:04 DEBUG "pod": "am-555f4db595-fz8sn" 23:04:04 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 23:04:04 DEBUG "value": [ 23:04:04 DEBUG { 23:04:04 DEBUG "metric": { 23:04:04 DEBUG "value": [ 23:04:04 DEBUG 23:04:04 DEBUG "value": [ 23:04:04 DEBUG "value": [ 23:04:04 DEBUG "instance": "10.142.0.107:9100" 23:04:04 DEBUG "value": [ 23:04:04 DEBUG }, 23:04:04 DEBUG "value": [ 23:04:04 DEBUG }, 23:04:04 DEBUG "value": [ 23:04:04 DEBUG }, 23:04:04 DEBUG }, 23:04:04 DEBUG "value": [ 23:04:04 DEBUG "value": [ 23:04:04 DEBUG "value": [ 23:04:04 DEBUG "value": [ 23:04:04 DEBUG "value": [ 23:04:04 DEBUG "value": [ 23:04:04 DEBUG }, 23:04:04 DEBUG "value": [ 23:04:04 DEBUG }, 23:04:04 DEBUG }, 23:04:04 DEBUG 23:04:04 DEBUG "metric": { 23:04:04 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG }, 23:04:04 DEBUG 23:04:04 DEBUG "value": [ 23:04:04 DEBUG 23:04:04 DEBUG "value": [ 23:04:04 DEBUG 23:04:04 DEBUG "value": [ 23:04:04 DEBUG "value": [ 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG "value": [ 23:04:04 DEBUG 23:04:04 DEBUG "value": [ 23:04:04 DEBUG "value": [ 23:04:04 DEBUG 23:04:04 DEBUG "pod": "ds-idrepo-2" 23:04:04 DEBUG }, 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG "value": [ 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG }, 23:04:04 DEBUG "value": [ 23:04:04 DEBUG 23:04:04 DEBUG [...] 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG [...] 23:04:04 DEBUG "value": [ 23:04:04 DEBUG 23:04:04 DEBUG [...] 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG [...] 23:04:04 DEBUG 23:04:04 DEBUG [...] 23:04:04 DEBUG 23:04:04 DEBUG [...] 23:04:04 DEBUG 23:04:04 DEBUG [...] 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG [...] 23:04:04 DEBUG [...] 23:04:04 DEBUG [...] 23:04:04 DEBUG [...] 23:04:04 DEBUG [...] 23:04:04 DEBUG [...] 23:04:04 DEBUG 23:04:04 DEBUG [...] 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG [...] 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG [...] 23:04:04 DEBUG 23:04:04 DEBUG [...] 23:04:04 DEBUG 23:04:04 DEBUG [...] 23:04:04 DEBUG [...] 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG [...] 23:04:04 DEBUG 23:04:04 DEBUG [...] 23:04:04 DEBUG [...] 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG "instance": "10.142.0.43:9100" 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG [...] 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG }, 23:04:04 DEBUG [...] 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG "pod": "am-555f4db595-kzxsp" 23:04:04 DEBUG [...] 23:04:04 DEBUG 23:04:04 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 23:04:04 DEBUG 23:04:04 DEBUG "value": [ 23:04:04 DEBUG 23:04:04 DEBUG "instance": "10.142.0.43:9100" 23:04:04 DEBUG 23:04:04 DEBUG "pod": "am-555f4db595-kzxsp" 23:04:04 DEBUG 23:04:04 DEBUG "instance": "10.142.0.43:9100" 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG "pod": "am-555f4db595-kzxsp" 23:04:04 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 23:04:04 DEBUG "pod": "ds-idrepo-0" 23:04:04 DEBUG "pod": "am-555f4db595-kzxsp" 23:04:04 DEBUG "pod": "ds-cts-2" 23:04:04 DEBUG "pod": "am-555f4db595-kzxsp" 23:04:04 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 23:04:04 DEBUG 23:04:04 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG }, 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG }, 23:04:04 DEBUG "pod": "ds-idrepo-0" 23:04:04 DEBUG 1713305044, 23:04:04 DEBUG 23:04:04 DEBUG }, 23:04:04 DEBUG "pod": "ds-idrepo-0" 23:04:04 DEBUG }, 23:04:04 DEBUG "pod": "ds-idrepo-0" 23:04:04 DEBUG }, 23:04:04 DEBUG "pod": "am-555f4db595-kzxsp" 23:04:04 DEBUG "pod": "ds-idrepo-0" 23:04:04 DEBUG }, 23:04:04 DEBUG }, 23:04:04 DEBUG }, 23:04:04 DEBUG }, 23:04:04 DEBUG }, 23:04:04 DEBUG }, 23:04:04 DEBUG }, 23:04:04 DEBUG "instance": "10.142.0.43:9100" 23:04:04 DEBUG }, 23:04:04 DEBUG "pod": "am-555f4db595-kzxsp" 23:04:04 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 23:04:04 DEBUG "value": [ 23:04:04 DEBUG 23:04:04 DEBUG 23:04:04 DEBUG "value": [ 23:04:04 DEBUG }, 23:04:04 DEBUG "0.00040000000000001514" 23:04:04 DEBUG "instance": "10.142.0.43:9100" 23:04:04 DEBUG "value": [ 23:04:04 DEBUG }, 23:04:04 DEBUG "value": [ 23:04:04 DEBUG }, 23:04:04 DEBUG "value": [ 23:04:04 DEBUG }, 23:04:04 DEBUG }, 23:04:04 DEBUG "value": [ 23:04:04 DEBUG "value": [ 23:04:04 DEBUG "value": [ 23:04:04 DEBUG 1713305044, 23:04:04 DEBUG "value": [ 23:04:04 DEBUG "value": [ 23:04:04 DEBUG "value": [ 23:04:04 DEBUG }, 23:04:04 DEBUG "value": [ 23:04:04 DEBUG }, 23:04:04 DEBUG }, 23:04:04 DEBUG 1713305044, 23:04:04 DEBUG 23:04:04 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 23:04:04 DEBUG 1713305044, 23:04:04 DEBUG "value": [ 23:04:04 DEBUG ] 23:04:04 DEBUG }, 23:04:04 DEBUG 1713305044, 23:04:04 DEBUG "value": [ 23:04:04 DEBUG 1713305044, 23:04:04 DEBUG "value": [ 23:04:04 DEBUG 1713305044, 23:04:04 DEBUG "value": [ 23:04:04 DEBUG "value": [ 23:04:04 DEBUG 1713305044, 23:04:04 DEBUG 1713305044, 23:04:04 DEBUG "value": [ 23:04:04 DEBUG "0" 23:04:04 DEBUG 1713305044, 23:04:04 DEBUG 1713305044, 23:04:04 DEBUG 1713305044, 23:04:04 DEBUG "value": [ 23:04:04 DEBUG 1713305044, 23:04:04 DEBUG "value": [ 23:04:04 DEBUG "value": [ 23:04:04 DEBUG "2159.1666666666665" 23:04:04 DEBUG "pod": "ds-idrepo-0" 23:04:04 DEBUG }, 23:04:04 DEBUG "0.000011581936065199279" 23:04:04 DEBUG 1713305044, 23:04:04 DEBUG } 23:04:04 DEBUG "value": [ 23:04:04 DEBUG "49036720.23333333" 23:04:04 DEBUG 1713305044, 23:04:04 DEBUG "0" 23:04:04 DEBUG 1713305044, 23:04:04 DEBUG "49027612.86666667" 23:04:04 DEBUG 1713305044, 23:04:04 DEBUG 1713305044, 23:04:04 DEBUG "0.027" 23:04:04 DEBUG "0" 23:04:04 DEBUG 1713305044, 23:04:04 DEBUG ] 23:04:04 DEBUG "2158.9" 23:04:04 DEBUG "2160.733333333333" 23:04:04 DEBUG "0" 23:04:04 DEBUG 1713305044, 23:04:04 DEBUG "0" 23:04:04 DEBUG 1713305044, 23:04:04 DEBUG 1713305044, 23:04:04 DEBUG ] 23:04:04 DEBUG }, 23:04:04 DEBUG "value": [ 23:04:04 DEBUG ] 23:04:04 DEBUG "0" 23:04:04 DEBUG ] 23:04:04 DEBUG 1713305044, 23:04:04 DEBUG ] 23:04:04 DEBUG "0" 23:04:04 DEBUG ] 23:04:04 DEBUG "0" 23:04:04 DEBUG ] 23:04:04 DEBUG "3692764" 23:04:04 DEBUG "0" 23:04:04 DEBUG ] 23:04:04 DEBUG ] 23:04:04 DEBUG "0" 23:04:04 DEBUG } 23:04:04 DEBUG ] 23:04:04 DEBUG ] 23:04:04 DEBUG ] 23:04:04 DEBUG "0" 23:04:04 DEBUG ] 23:04:04 DEBUG "2098.4666666666667" 23:04:04 DEBUG "1863819264" 23:04:04 DEBUG } 23:04:04 DEBUG "value": [ 23:04:04 DEBUG 1713305044, 23:04:04 DEBUG "7.0863961018847395" 23:04:04 DEBUG ] 23:04:04 DEBUG } 23:04:04 DEBUG "19983594496" 23:04:04 DEBUG } 23:04:04 DEBUG ] 23:04:04 DEBUG } 23:04:04 DEBUG ] 23:04:04 DEBUG } 23:04:04 DEBUG ] 23:04:04 DEBUG ] 23:04:04 DEBUG } 23:04:04 DEBUG } 23:04:04 DEBUG ] 23:04:04 DEBUG ] 23:04:04 DEBUG } 23:04:04 DEBUG } 23:04:04 DEBUG } 23:04:04 DEBUG ] 23:04:04 DEBUG } 23:04:04 DEBUG ] 23:04:04 DEBUG ] 23:04:04 DEBUG ] 23:04:04 DEBUG 1713305044, 23:04:04 DEBUG } 23:04:04 DEBUG ] 23:04:04 DEBUG } 23:04:04 DEBUG } 23:04:04 DEBUG ] 23:04:04 DEBUG ] 23:04:04 DEBUG } 23:04:04 DEBUG ] 23:04:04 DEBUG } 23:04:04 DEBUG ] 23:04:04 DEBUG } 23:04:04 DEBUG } 23:04:04 DEBUG ] 23:04:04 DEBUG ] 23:04:04 DEBUG } 23:04:04 DEBUG } 23:04:04 DEBUG ] 23:04:04 DEBUG ] 23:04:04 DEBUG } 23:04:04 DEBUG } 23:04:04 DEBUG ] 23:04:04 DEBUG } 23:04:04 DEBUG } 23:04:04 DEBUG } 23:04:04 DEBUG "0" 23:04:04 DEBUG ] 23:04:04 DEBUG } 23:04:04 DEBUG ] 23:04:04 DEBUG } 23:04:04 DEBUG } 23:04:04 DEBUG ] 23:04:04 DEBUG } 23:04:04 DEBUG ] 23:04:04 DEBUG } 23:04:04 DEBUG ] 23:04:04 DEBUG ] 23:04:04 DEBUG } 23:04:04 DEBUG } 23:04:04 DEBUG ] 23:04:04 DEBUG } 23:04:04 DEBUG } 23:04:04 DEBUG ] 23:04:04 DEBUG } 23:04:04 DEBUG ] 23:04:04 DEBUG } 23:04:04 DEBUG ] 23:04:04 DEBUG ] 23:04:04 DEBUG } 23:04:04 DEBUG ] 23:04:04 DEBUG } 23:04:04 DEBUG ] 23:04:04 DEBUG } 23:04:04 DEBUG ] 23:04:04 DEBUG } 23:04:04 DEBUG } 23:04:04 DEBUG } 23:04:04 DEBUG } 23:04:04 DEBUG } 23:04:04 DEBUG } 23:04:04 DEBUG } 23:04:04 DEBUG } 23:04:04 DEBUG } 23:04:04 DEBUG } 23:04:04 DEBUG } 23:04:04 DEBUG } 23:04:04 DEBUG } 23:04:04 DEBUG } 23:04:04 DEBUG } 23:04:04 DEBUG } 23:04:04 DEBUG } 23:04:04 DEBUG } 23:04:04 DEBUG } 23:04:04 DEBUG } 23:04:04 DEBUG } 23:04:04 DEBUG } 23:04:04 DEBUG } 23:04:04 DEBUG } 23:04:04 DEBUG } 23:04:04 DEBUG } 23:04:04 DEBUG } 23:04:04 DEBUG } 23:04:04 DEBUG } 23:04:04 DEBUG } 23:04:04 DEBUG ] 23:04:04 DEBUG } 23:04:04 DEBUG } 23:04:04 DEBUG } 23:04:04 DEBUG } 23:04:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305098" 23:04:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713305098" 23:04:58 ERROR [http_cmd]: ERROR 23:04:58 INFO [http_cmd]: http status code OK 23:04:58 ERROR --- status code --- 23:04:58 DEBUG --- status code --- 23:04:58 ERROR http status code is 400 (expected [200, 401, 403, 404]) 23:04:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:04:58 ERROR --- http response --- 23:04:58 DEBUG --- http response --- 23:04:58 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 23:04:58 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 23:04:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 23:05:01 INFO 23:05:01 INFO [loop_until]: kubectl --namespace=xlou top pods 23:05:01 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 23:05:01 INFO [loop_until]: OK (rc = 0) 23:05:01 DEBUG --- stdout --- 23:05:01 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 9Mi am-555f4db595-4lbss 9580m 8684Mi am-555f4db595-fz8sn 9610m 8567Mi am-555f4db595-kzxsp 9712m 8977Mi ds-cts-0 18971m 13231Mi ds-cts-1 15639m 13422Mi ds-cts-2 16376m 13212Mi ds-idrepo-0 4069m 5030Mi ds-idrepo-1 4049m 2899Mi ds-idrepo-2 4015m 3220Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 4m 1207Mi idm-b85f9d978-rl76f 3m 1225Mi lodemon-bc756c647-cr7x6 2m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-7c9b667974-9mr2s 8080m 3687Mi 23:05:01 DEBUG --- stderr --- 23:05:01 DEBUG 23:05:04 INFO 23:05:04 INFO [loop_until]: kubectl --namespace=xlou top node 23:05:04 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 23:05:04 INFO [loop_until]: OK (rc = 0) 23:05:04 DEBUG --- stdout --- 23:05:04 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 61m 0% 5084Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 15767m 52% 15869Mi 14% gke-xlou-cdm-default-pool-1b0769c8-nhlf 9854m 33% 10801Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 80m 0% 2648Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 4152m 13% 6571Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 20239m 67% 17721Mi 15% gke-xlou-cdm-default-pool-1b0769c8-xnmp 58m 0% 3137Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 9446m 31% 10487Mi 9% gke-xlou-cdm-ds-0f601751-hmm7 4537m 15% 5649Mi 5% gke-xlou-cdm-ds-0f601751-j488 15699m 52% 15340Mi 13% gke-xlou-cdm-ds-0f601751-mc5d 9523m 31% 4196Mi 3% gke-xlou-cdm-ds-0f601751-nj6q 10397m 34% 10859Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 4518m 15% 7353Mi 6% gke-xlou-cdm-frontend-a8771548-i82n 8159m 51% 8890Mi 15% 23:05:04 DEBUG --- stderr --- 23:05:04 DEBUG 23:05:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305104" 23:05:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713305104" 23:05:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305104" 23:05:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305104" 23:05:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305104" 23:05:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713305104" 23:05:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305104" 23:05:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305104" 23:05:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305104" 23:05:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713305104" 23:05:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305104" 23:05:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713305104" 23:05:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305104" 23:05:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305104" 23:05:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713305104" 23:05:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305104" 23:05:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305104" 23:05:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713305104" 23:05:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305104" 23:05:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713305104" 23:05:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305104" 23:05:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305104" 23:05:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713305104" 23:05:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713305104" 23:05:04 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305104" 23:05:04 INFO [http_cmd]: http status code OK 23:05:04 INFO [http_cmd]: http status code OK 23:05:04 INFO [http_cmd]: http status code OK 23:05:04 INFO [http_cmd]: http status code OK 23:05:04 INFO [http_cmd]: http status code OK 23:05:04 INFO [http_cmd]: http status code OK 23:05:04 INFO [http_cmd]: http status code OK 23:05:04 INFO [http_cmd]: http status code OK 23:05:04 INFO [http_cmd]: http status code OK 23:05:04 INFO [http_cmd]: http status code OK 23:05:04 INFO [http_cmd]: http status code OK 23:05:04 INFO [http_cmd]: http status code OK 23:05:04 INFO [http_cmd]: http status code OK 23:05:04 INFO [http_cmd]: http status code OK 23:05:04 INFO [http_cmd]: http status code OK 23:05:04 INFO [http_cmd]: http status code OK 23:05:04 INFO [http_cmd]: http status code OK 23:05:04 INFO [http_cmd]: http status code OK 23:05:04 DEBUG --- status code --- 23:05:04 INFO [http_cmd]: http status code OK 23:05:04 DEBUG --- status code --- 23:05:04 DEBUG --- status code --- 23:05:04 DEBUG --- status code --- 23:05:04 DEBUG --- status code --- 23:05:04 DEBUG --- status code --- 23:05:04 DEBUG --- status code --- 23:05:04 DEBUG --- status code --- 23:05:04 DEBUG --- status code --- 23:05:04 DEBUG --- status code --- 23:05:04 DEBUG --- status code --- 23:05:04 DEBUG --- status code --- 23:05:04 DEBUG --- status code --- 23:05:04 DEBUG --- status code --- 23:05:04 DEBUG --- status code --- 23:05:04 DEBUG --- status code --- 23:05:04 DEBUG --- status code --- 23:05:04 DEBUG --- status code --- 23:05:04 INFO [http_cmd]: http status code OK 23:05:04 INFO [http_cmd]: http status code OK 23:05:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:05:04 INFO [http_cmd]: http status code OK 23:05:04 DEBUG --- status code --- 23:05:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:05:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:05:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:05:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:05:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:05:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:05:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:05:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:05:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:05:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:05:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:05:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:05:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:05:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:05:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:05:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:05:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:05:04 DEBUG --- status code --- 23:05:04 DEBUG --- status code --- 23:05:04 DEBUG --- http response --- 23:05:04 DEBUG --- status code --- 23:05:04 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:05:04 DEBUG --- http response --- 23:05:04 DEBUG --- http response --- 23:05:04 DEBUG --- http response --- 23:05:05 DEBUG --- http response --- 23:05:05 INFO [http_cmd]: http status code OK 23:05:05 DEBUG --- http response --- 23:05:05 DEBUG --- http response --- 23:05:05 DEBUG --- http response --- 23:05:05 DEBUG --- http response --- 23:05:05 DEBUG --- http response --- 23:05:05 DEBUG --- http response --- 23:05:05 DEBUG --- http response --- 23:05:05 DEBUG --- http response --- 23:05:05 DEBUG --- http response --- 23:05:05 DEBUG --- http response --- 23:05:05 DEBUG --- http response --- 23:05:05 DEBUG --- http response --- 23:05:05 INFO [http_cmd]: http status code OK 23:05:05 DEBUG --- http response --- 23:05:05 INFO [http_cmd]: http status code OK 23:05:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:05:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:05:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:05:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:05:05 DEBUG --- http response --- 23:05:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 23:05:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:05:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:05:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:05:05 DEBUG --- status code --- 23:05:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:05:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:05:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:05:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:05:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:05:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:05:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 23:05:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:05:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:05:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 23:05:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:05:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 23:05:05 DEBUG --- status code --- 23:05:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:05:05 DEBUG --- status code --- 23:05:05 DEBUG --- http response --- 23:05:05 DEBUG --- http response --- 23:05:05 DEBUG --- http response --- 23:05:05 DEBUG ----- output ----- 23:05:05 DEBUG ----- output ----- 23:05:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:05:05 DEBUG ----- output ----- 23:05:05 DEBUG ----- output ----- 23:05:05 DEBUG ----- output ----- 23:05:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:05:05 DEBUG ----- output ----- 23:05:05 DEBUG ----- output ----- 23:05:05 DEBUG ----- output ----- 23:05:05 DEBUG ----- output ----- 23:05:05 DEBUG ----- output ----- 23:05:05 DEBUG ----- output ----- 23:05:05 DEBUG ----- output ----- 23:05:05 DEBUG ----- output ----- 23:05:05 DEBUG ----- output ----- 23:05:05 DEBUG ----- output ----- 23:05:05 DEBUG ----- output ----- 23:05:05 DEBUG ----- output ----- 23:05:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:05:05 DEBUG ----- output ----- 23:05:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:05:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:05:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:05:05 DEBUG { 23:05:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:05:05 DEBUG { 23:05:05 DEBUG ----- output ----- 23:05:05 DEBUG { 23:05:05 DEBUG { 23:05:05 DEBUG { 23:05:05 DEBUG --- http response --- 23:05:05 DEBUG { 23:05:05 DEBUG { 23:05:05 DEBUG { 23:05:05 DEBUG { 23:05:05 DEBUG { 23:05:05 DEBUG { 23:05:05 DEBUG { 23:05:05 DEBUG { 23:05:05 DEBUG { 23:05:05 DEBUG { 23:05:05 DEBUG { 23:05:05 DEBUG { 23:05:05 DEBUG --- http response --- 23:05:05 DEBUG { 23:05:05 DEBUG --- http response --- 23:05:05 DEBUG ----- output ----- 23:05:05 DEBUG ----- output ----- 23:05:05 DEBUG "status": "success", 23:05:05 DEBUG ----- output ----- 23:05:05 DEBUG "status": "success", 23:05:05 DEBUG { 23:05:05 DEBUG "status": "success", 23:05:05 DEBUG "status": "success", 23:05:05 DEBUG "status": "success", 23:05:05 DEBUG "status": "success", 23:05:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:05:05 DEBUG "status": "success", 23:05:05 DEBUG "status": "success", 23:05:05 DEBUG "status": "success", 23:05:05 DEBUG "status": "success", 23:05:05 DEBUG "status": "success", 23:05:05 DEBUG "status": "success", 23:05:05 DEBUG "status": "success", 23:05:05 DEBUG "status": "success", 23:05:05 DEBUG "status": "success", 23:05:05 DEBUG "status": "success", 23:05:05 DEBUG "status": "success", 23:05:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:05:05 DEBUG "status": "success", 23:05:05 DEBUG { 23:05:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:05:05 DEBUG { 23:05:05 DEBUG "data": { 23:05:05 DEBUG { 23:05:05 DEBUG "data": { 23:05:05 DEBUG "status": "success", 23:05:05 DEBUG "data": { 23:05:05 DEBUG "data": { 23:05:05 DEBUG "data": { 23:05:05 DEBUG "data": { 23:05:05 DEBUG ----- output ----- 23:05:05 DEBUG "data": { 23:05:05 DEBUG "data": { 23:05:05 DEBUG "data": { 23:05:05 DEBUG "data": { 23:05:05 DEBUG "data": { 23:05:05 DEBUG "data": { 23:05:05 DEBUG "data": { 23:05:05 DEBUG "data": { 23:05:05 DEBUG "data": { 23:05:05 DEBUG "data": { 23:05:05 DEBUG "data": { 23:05:05 DEBUG ----- output ----- 23:05:05 DEBUG "data": { 23:05:05 DEBUG "status": "success", 23:05:05 DEBUG ----- output ----- 23:05:05 DEBUG "status": "success", 23:05:05 DEBUG "resultType": "vector", 23:05:05 DEBUG "status": "success", 23:05:05 DEBUG "resultType": "vector", 23:05:05 DEBUG "data": { 23:05:05 DEBUG "resultType": "vector", 23:05:05 DEBUG "resultType": "vector", 23:05:05 DEBUG "resultType": "vector", 23:05:05 DEBUG "resultType": "vector", 23:05:05 DEBUG { 23:05:05 DEBUG "resultType": "vector", 23:05:05 DEBUG "resultType": "vector", 23:05:05 DEBUG "resultType": "vector", 23:05:05 DEBUG "resultType": "vector", 23:05:05 DEBUG "resultType": "vector", 23:05:05 DEBUG "resultType": "vector", 23:05:05 DEBUG "resultType": "vector", 23:05:05 DEBUG "resultType": "vector", 23:05:05 DEBUG "resultType": "vector", 23:05:05 DEBUG "resultType": "vector", 23:05:05 DEBUG "resultType": "vector", 23:05:05 DEBUG { 23:05:05 DEBUG "resultType": "vector", 23:05:05 DEBUG "data": { 23:05:05 DEBUG { 23:05:05 DEBUG "data": { 23:05:05 DEBUG "result": [ 23:05:05 DEBUG "data": { 23:05:05 DEBUG "result": [ 23:05:05 DEBUG "resultType": "vector", 23:05:05 DEBUG "result": [ 23:05:05 DEBUG "result": [ 23:05:05 DEBUG "result": [ 23:05:05 DEBUG "result": [ 23:05:05 DEBUG "status": "success", 23:05:05 DEBUG "result": [ 23:05:05 DEBUG "result": [ 23:05:05 DEBUG "result": [ 23:05:05 DEBUG "result": [ 23:05:05 DEBUG "result": [ 23:05:05 DEBUG "result": [ 23:05:05 DEBUG "result": [ 23:05:05 DEBUG "result": [ 23:05:05 DEBUG "result": [ 23:05:05 DEBUG "result": [ 23:05:05 DEBUG "result": [ 23:05:05 DEBUG "status": "success", 23:05:05 DEBUG "result": [ 23:05:05 DEBUG "resultType": "vector", 23:05:05 DEBUG "status": "success", 23:05:05 DEBUG "resultType": "vector", 23:05:05 DEBUG { 23:05:05 DEBUG "resultType": "vector", 23:05:05 DEBUG { 23:05:05 DEBUG "result": [ 23:05:05 DEBUG { 23:05:05 DEBUG { 23:05:05 DEBUG { 23:05:05 DEBUG { 23:05:05 DEBUG "data": { 23:05:05 DEBUG { 23:05:05 DEBUG "metric": { 23:05:05 DEBUG { 23:05:05 DEBUG { 23:05:05 DEBUG { 23:05:05 DEBUG { 23:05:05 DEBUG { 23:05:05 DEBUG { 23:05:05 DEBUG { 23:05:05 DEBUG { 23:05:05 DEBUG { 23:05:05 DEBUG "data": { 23:05:05 DEBUG { 23:05:05 DEBUG "result": [ 23:05:05 DEBUG "data": { 23:05:05 DEBUG "resultType": "vector", 23:05:05 DEBUG "metric": { 23:05:05 DEBUG "result": [ 23:05:05 DEBUG "metric": { 23:05:05 DEBUG { 23:05:05 DEBUG "metric": { 23:05:05 DEBUG "metric": { 23:05:05 DEBUG "metric": { 23:05:05 DEBUG "metric": { 23:05:05 DEBUG "resultType": "vector", 23:05:05 DEBUG { 23:05:05 DEBUG "pod": "am-555f4db595-fz8sn" 23:05:05 DEBUG "metric": { 23:05:05 DEBUG "metric": { 23:05:05 DEBUG "metric": { 23:05:05 DEBUG "metric": { 23:05:05 DEBUG "metric": { 23:05:05 DEBUG "metric": { 23:05:05 DEBUG "metric": { 23:05:05 DEBUG "metric": { 23:05:05 DEBUG "metric": { 23:05:05 DEBUG "resultType": "vector", 23:05:05 DEBUG "metric": { 23:05:05 DEBUG { 23:05:05 DEBUG "result": [ 23:05:05 DEBUG "result": [ 23:05:05 DEBUG "instance": "10.142.0.107:9100" 23:05:05 DEBUG { 23:05:05 DEBUG "pod": "ds-idrepo-2" 23:05:05 DEBUG "metric": { 23:05:05 DEBUG "instance": "10.142.0.107:9100" 23:05:05 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 23:05:05 DEBUG "pod": "am-555f4db595-fz8sn" 23:05:05 DEBUG "pod": "am-555f4db595-fz8sn" 23:05:05 DEBUG "result": [ 23:05:05 DEBUG "metric": { 23:05:05 DEBUG }, 23:05:05 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 23:05:05 DEBUG "pod": "am-555f4db595-fz8sn" 23:05:05 DEBUG "pod": "ds-idrepo-2" 23:05:05 DEBUG "pod": "ds-idrepo-2" 23:05:05 DEBUG "instance": "10.142.0.107:9100" 23:05:05 DEBUG "pod": "ds-idrepo-2" 23:05:05 DEBUG "pod": "ds-idrepo-2" 23:05:05 DEBUG "pod": "am-555f4db595-fz8sn" 23:05:05 DEBUG "pod": "ds-idrepo-2" 23:05:05 DEBUG "result": [ 23:05:05 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 23:05:05 DEBUG "metric": { 23:05:05 DEBUG { 23:05:05 DEBUG { 23:05:05 DEBUG }, 23:05:05 DEBUG "metric": { 23:05:05 DEBUG }, 23:05:05 DEBUG "instance": "10.142.0.107:9100" 23:05:05 DEBUG }, 23:05:05 DEBUG }, 23:05:05 DEBUG }, 23:05:05 DEBUG }, 23:05:05 DEBUG { 23:05:05 DEBUG "pod": "am-555f4db595-fz8sn" 23:05:05 DEBUG "value": [ 23:05:05 DEBUG }, 23:05:05 DEBUG }, 23:05:05 DEBUG }, 23:05:05 DEBUG }, 23:05:05 DEBUG }, 23:05:05 DEBUG }, 23:05:05 DEBUG }, 23:05:05 DEBUG }, 23:05:05 DEBUG }, 23:05:05 DEBUG { 23:05:05 DEBUG }, 23:05:05 DEBUG "pod": "am-555f4db595-fz8sn" 23:05:05 DEBUG "metric": { 23:05:05 DEBUG "metric": { 23:05:05 DEBUG "value": [ 23:05:05 DEBUG "pod": "ds-idrepo-2" 23:05:05 DEBUG }, 23:05:05 DEBUG }, 23:05:05 DEBUG "value": [ 23:05:05 DEBUG "value": [ 23:05:05 DEBUG "value": [ 23:05:05 DEBUG "value": [ 23:05:05 DEBUG "metric": { 23:05:05 DEBUG }, 23:05:05 DEBUG 23:05:05 DEBUG "value": [ 23:05:05 DEBUG "value": [ 23:05:05 DEBUG "value": [ 23:05:05 DEBUG "value": [ 23:05:05 DEBUG "value": [ 23:05:05 DEBUG "value": [ 23:05:05 DEBUG "value": [ 23:05:05 DEBUG "value": [ 23:05:05 DEBUG "value": [ 23:05:05 DEBUG "metric": { 23:05:05 DEBUG "value": [ 23:05:05 DEBUG }, 23:05:05 DEBUG "instance": "10.142.0.107:9100" 23:05:05 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 23:05:05 DEBUG 23:05:05 DEBUG "value": [ 23:05:05 DEBUG "value": [ 23:05:05 DEBUG "value": [ 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 23:05:05 DEBUG "value": [ 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 23:05:05 DEBUG 23:05:05 DEBUG "value": [ 23:05:05 DEBUG }, 23:05:05 DEBUG }, 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG }, 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG }, 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG "value": [ 23:05:05 DEBUG "value": [ 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG "value": [ 23:05:05 DEBUG 23:05:05 DEBUG [...] 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG "value": [ 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG [...] 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG [...] 23:05:05 DEBUG [...] 23:05:05 DEBUG [...] 23:05:05 DEBUG [...] 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG [...] 23:05:05 DEBUG [...] 23:05:05 DEBUG [...] 23:05:05 DEBUG [...] 23:05:05 DEBUG [...] 23:05:05 DEBUG [...] 23:05:05 DEBUG [...] 23:05:05 DEBUG [...] 23:05:05 DEBUG [...] 23:05:05 DEBUG 23:05:05 DEBUG [...] 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG [...] 23:05:05 DEBUG [...] 23:05:05 DEBUG [...] 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG [...] 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG [...] 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG [...] 23:05:05 DEBUG [...] 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG [...] 23:05:05 DEBUG 23:05:05 DEBUG "pod": "am-555f4db595-kzxsp" 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG [...] 23:05:05 DEBUG 23:05:05 DEBUG "pod": "end-user-ui-58878d7668-zg4h2" 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG "instance": "10.142.0.43:9100" 23:05:05 DEBUG }, 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG "instance": "10.142.0.43:9100" 23:05:05 DEBUG "pod": "end-user-ui-58878d7668-zg4h2" 23:05:05 DEBUG }, 23:05:05 DEBUG "pod": "am-555f4db595-kzxsp" 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG }, 23:05:05 DEBUG "value": [ 23:05:05 DEBUG 1713305104, 23:05:05 DEBUG "1463.7666666666667" 23:05:05 DEBUG ] 23:05:05 DEBUG } 23:05:05 DEBUG "pod": "ds-idrepo-0" 23:05:05 DEBUG "pod": "ds-idrepo-0" 23:05:05 DEBUG }, 23:05:05 DEBUG "pod": "ds-cts-2" 23:05:05 DEBUG }, 23:05:05 DEBUG 23:05:05 DEBUG }, 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG "value": [ 23:05:05 DEBUG "pod": "ds-idrepo-0" 23:05:05 DEBUG }, 23:05:05 DEBUG "value": [ 23:05:05 DEBUG 1713305104, 23:05:05 DEBUG "0" 23:05:05 DEBUG ] 23:05:05 DEBUG 23:05:05 DEBUG "pod": "am-555f4db595-kzxsp" 23:05:05 DEBUG }, 23:05:05 DEBUG "value": [ 23:05:05 DEBUG 1713305104, 23:05:05 DEBUG "1463.3333333333333" 23:05:05 DEBUG "pod": "end-user-ui-58878d7668-zg4h2" 23:05:05 DEBUG "pod": "am-555f4db595-kzxsp" 23:05:05 DEBUG }, 23:05:05 DEBUG "value": [ 23:05:05 DEBUG 1713305104, 23:05:05 DEBUG "0" 23:05:05 DEBUG 23:05:05 DEBUG "value": [ 23:05:05 DEBUG 23:05:05 DEBUG 23:05:05 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 23:05:05 DEBUG "pod": "ds-idrepo-0" 23:05:05 DEBUG }, 23:05:05 DEBUG "value": [ 23:05:05 DEBUG "value": [ 23:05:05 DEBUG 23:05:05 DEBUG "instance": "10.142.0.43:9100" 23:05:05 DEBUG ] 23:05:05 DEBUG } 23:05:05 DEBUG ] 23:05:05 DEBUG "pod": "am-555f4db595-kzxsp" 23:05:05 DEBUG "pod": "ds-idrepo-0" 23:05:05 DEBUG 23:05:05 DEBUG }, 23:05:05 DEBUG "instance": "10.142.0.43:9100" 23:05:05 DEBUG }, 23:05:05 DEBUG }, 23:05:05 DEBUG ] 23:05:05 DEBUG "pod": "am-555f4db595-kzxsp" 23:05:05 DEBUG }, 23:05:05 DEBUG "pod": "am-555f4db595-kzxsp" 23:05:05 DEBUG "instance": "10.142.0.43:9100" 23:05:05 DEBUG "value": [ 23:05:05 DEBUG "value": [ 23:05:05 DEBUG 1713305104, 23:05:05 DEBUG "0" 23:05:05 DEBUG }, 23:05:05 DEBUG 1713305104, 23:05:05 DEBUG 1713305104, 23:05:05 DEBUG "pod": "end-user-ui-58878d7668-zg4h2" 23:05:05 DEBUG }, 23:05:05 DEBUG "pod": "ds-idrepo-0" 23:05:05 DEBUG ] 23:05:05 DEBUG 1713305104, 23:05:05 DEBUG } 23:05:05 DEBUG }, 23:05:05 DEBUG }, 23:05:05 DEBUG 23:05:05 DEBUG "value": [ 23:05:05 DEBUG }, 23:05:05 DEBUG "value": [ 23:05:05 DEBUG 1713305104, 23:05:05 DEBUG } 23:05:05 DEBUG "value": [ 23:05:05 DEBUG 1713305104, 23:05:05 DEBUG }, 23:05:05 DEBUG }, 23:05:05 DEBUG 1713305104, 23:05:05 DEBUG ] 23:05:05 DEBUG }, 23:05:05 DEBUG } 23:05:05 DEBUG "value": [ 23:05:05 DEBUG "35671958.76666666" 23:05:05 DEBUG "0" 23:05:05 DEBUG ] 23:05:05 DEBUG }, 23:05:05 DEBUG }, 23:05:05 DEBUG } 23:05:05 DEBUG "0.00040000000000001514" 23:05:05 DEBUG } 23:05:05 DEBUG "value": [ 23:05:05 DEBUG 1713305104, 23:05:05 DEBUG "0" 23:05:05 DEBUG 1713305104, 23:05:05 DEBUG "1463.7" 23:05:05 DEBUG ] 23:05:05 DEBUG "0" 23:05:05 DEBUG ] 23:05:05 DEBUG 1713305104, 23:05:05 DEBUG "1997.2333333333333" 23:05:05 DEBUG "value": [ 23:05:05 DEBUG "value": [ 23:05:05 DEBUG "0" 23:05:05 DEBUG } 23:05:05 DEBUG "value": [ 23:05:05 DEBUG ] 23:05:05 DEBUG } 23:05:05 DEBUG } 23:05:05 DEBUG } 23:05:05 DEBUG ] 23:05:05 DEBUG "value": [ 23:05:05 DEBUG 1713305104, 23:05:05 DEBUG "0" 23:05:05 DEBUG ] 23:05:05 DEBUG 23:05:05 DEBUG "value": [ 23:05:05 DEBUG 1713305104, 23:05:05 DEBUG "0" 23:05:05 DEBUG ] 23:05:05 DEBUG } 23:05:05 DEBUG ] 23:05:05 DEBUG } 23:05:05 DEBUG } 23:05:05 DEBUG "0" 23:05:05 DEBUG "value": [ 23:05:05 DEBUG 1713305104, 23:05:05 DEBUG } 23:05:05 DEBUG ] 23:05:05 DEBUG ] 23:05:05 DEBUG } 23:05:05 DEBUG ] 23:05:05 DEBUG 1713305104, 23:05:05 DEBUG } 23:05:05 DEBUG ] 23:05:05 DEBUG } 23:05:05 DEBUG ] 23:05:05 DEBUG } 23:05:05 DEBUG } 23:05:05 DEBUG } 23:05:05 DEBUG ] 23:05:05 DEBUG ] 23:05:05 DEBUG } 23:05:05 DEBUG ] 23:05:05 DEBUG } 23:05:05 DEBUG } 23:05:05 DEBUG ] 23:05:05 DEBUG } 23:05:05 DEBUG } 23:05:05 DEBUG } 23:05:05 DEBUG "0.027" 23:05:05 DEBUG "value": [ 23:05:05 DEBUG 1713305104, 23:05:05 DEBUG 1713305104, 23:05:05 DEBUG "2675347456" 23:05:05 DEBUG ] 23:05:05 DEBUG ] 23:05:05 DEBUG } 23:05:05 DEBUG } 23:05:05 DEBUG "value": [ 23:05:05 DEBUG 1713305104, 23:05:05 DEBUG ] 23:05:05 DEBUG } 23:05:05 DEBUG 1713305104, 23:05:05 DEBUG ] 23:05:05 DEBUG ] 23:05:05 DEBUG } 23:05:05 DEBUG } 23:05:05 DEBUG "value": [ 23:05:05 DEBUG ] 23:05:05 DEBUG "0" 23:05:05 DEBUG ] 23:05:05 DEBUG } 23:05:05 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 23:05:05 DEBUG "0" 23:05:05 DEBUG } 23:05:05 DEBUG } 23:05:05 DEBUG ] 23:05:05 DEBUG } 23:05:05 DEBUG } 23:05:05 DEBUG } 23:05:05 DEBUG ] 23:05:05 DEBUG } 23:05:05 DEBUG } 23:05:05 DEBUG } 23:05:05 DEBUG 1713305104, 23:05:05 DEBUG } 23:05:05 DEBUG ] 23:05:05 DEBUG } 23:05:05 DEBUG } 23:05:05 DEBUG ] 23:05:05 DEBUG }, 23:05:05 DEBUG ] 23:05:05 DEBUG "3681813" 23:05:05 DEBUG ] 23:05:05 DEBUG 1713305104, 23:05:05 DEBUG } 23:05:05 DEBUG "19984192512" 23:05:05 DEBUG ] 23:05:05 DEBUG "35665013.36666667" 23:05:05 DEBUG ] 23:05:05 DEBUG } 23:05:05 DEBUG } 23:05:05 DEBUG } 23:05:05 DEBUG "value": [ 23:05:05 DEBUG 1713305104, 23:05:05 DEBUG } 23:05:05 DEBUG ] 23:05:05 DEBUG } 23:05:05 DEBUG ] 23:05:05 DEBUG } 23:05:05 DEBUG } 23:05:05 DEBUG ] 23:05:05 DEBUG } 23:05:05 DEBUG } 23:05:05 DEBUG } 23:05:05 DEBUG "7.712086220179939" 23:05:05 DEBUG } 23:05:05 DEBUG "0.000012671172450858958" 23:05:05 DEBUG } 23:05:05 DEBUG } 23:05:05 DEBUG ] 23:05:05 DEBUG ] 23:05:05 DEBUG } 23:05:05 DEBUG ] 23:05:05 DEBUG ] 23:05:05 DEBUG } 23:05:05 DEBUG ] 23:05:05 DEBUG ] 23:05:05 DEBUG } 23:05:05 DEBUG } 23:05:05 DEBUG } 23:05:05 DEBUG } 23:05:05 DEBUG } 23:05:05 DEBUG } 23:05:05 DEBUG } 23:05:05 DEBUG } 23:05:05 DEBUG } 23:05:05 DEBUG } 23:05:05 DEBUG ] 23:05:05 DEBUG ] 23:05:05 DEBUG } 23:05:05 DEBUG } 23:05:05 DEBUG } 23:05:05 DEBUG } 23:05:05 DEBUG } 23:05:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305158" 23:05:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713305158" 23:05:58 ERROR [http_cmd]: ERROR 23:05:58 INFO [http_cmd]: http status code OK 23:05:58 ERROR --- status code --- 23:05:58 DEBUG --- status code --- 23:05:58 ERROR http status code is 400 (expected [200, 401, 403, 404]) 23:05:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:05:58 ERROR --- http response --- 23:05:58 DEBUG --- http response --- 23:05:58 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 23:05:58 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 23:05:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 23:06:01 INFO 23:06:01 INFO [loop_until]: kubectl --namespace=xlou top pods 23:06:01 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 23:06:01 INFO [loop_until]: OK (rc = 0) 23:06:01 DEBUG --- stdout --- 23:06:01 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 9Mi am-555f4db595-4lbss 9203m 8683Mi am-555f4db595-fz8sn 9169m 8564Mi am-555f4db595-kzxsp 9324m 8974Mi ds-cts-0 19598m 13531Mi ds-cts-1 14269m 13259Mi ds-cts-2 14850m 12962Mi ds-idrepo-0 3892m 5029Mi ds-idrepo-1 3804m 2900Mi ds-idrepo-2 3699m 3220Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 4m 1207Mi idm-b85f9d978-rl76f 3m 1226Mi lodemon-bc756c647-cr7x6 2m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-7c9b667974-9mr2s 7978m 5626Mi 23:06:01 DEBUG --- stderr --- 23:06:01 DEBUG 23:06:04 INFO 23:06:04 INFO [loop_until]: kubectl --namespace=xlou top node 23:06:04 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 23:06:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305165" 23:06:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305165" 23:06:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305165" 23:06:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305165" 23:06:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713305165" 23:06:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713305165" 23:06:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305165" 23:06:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713305165" 23:06:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305165" 23:06:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305165" 23:06:05 INFO [http_cmd]: http status code OK 23:06:05 INFO [http_cmd]: http status code OK 23:06:05 DEBUG --- status code --- 23:06:05 DEBUG --- status code --- 23:06:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:06:05 INFO [http_cmd]: http status code OK 23:06:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:06:05 DEBUG --- http response --- 23:06:05 DEBUG --- status code --- 23:06:05 DEBUG --- http response --- 23:06:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:06:05 INFO [http_cmd]: http status code OK 23:06:05 INFO [http_cmd]: http status code OK 23:06:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:06:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:06:05 DEBUG --- http response --- 23:06:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305165" 23:06:05 DEBUG --- status code --- 23:06:05 DEBUG --- status code --- 23:06:05 DEBUG ----- output ----- 23:06:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305165" 23:06:05 DEBUG ----- output ----- 23:06:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:06:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305165" 23:06:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:06:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713305165" 23:06:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305165" 23:06:05 INFO [http_cmd]: http status code OK 23:06:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:06:05 INFO [http_cmd]: http status code OK 23:06:05 INFO [http_cmd]: http status code OK 23:06:05 INFO [http_cmd]: http status code OK 23:06:05 INFO [http_cmd]: http status code OK 23:06:05 DEBUG { 23:06:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713305165" 23:06:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305165" 23:06:05 DEBUG { 23:06:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305165" 23:06:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305165" 23:06:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305165" 23:06:05 DEBUG --- http response --- 23:06:05 DEBUG --- http response --- 23:06:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713305165" 23:06:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713305165" 23:06:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305165" 23:06:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713305165" 23:06:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713305165" 23:06:05 DEBUG --- status code --- 23:06:05 INFO [loop_until]: OK (rc = 0) 23:06:05 INFO [http_cmd]: http status code OK 23:06:05 DEBUG ----- output ----- 23:06:05 DEBUG --- status code --- 23:06:05 DEBUG --- status code --- 23:06:05 DEBUG --- status code --- 23:06:05 DEBUG --- status code --- 23:06:05 DEBUG "status": "success", 23:06:05 DEBUG "status": "success", 23:06:05 INFO [http_cmd]: http status code OK 23:06:05 INFO [http_cmd]: http status code OK 23:06:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:06:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 23:06:05 INFO [http_cmd]: http status code OK 23:06:05 INFO [http_cmd]: http status code OK 23:06:05 INFO [http_cmd]: http status code OK 23:06:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:06:05 DEBUG --- stdout --- 23:06:05 INFO [http_cmd]: http status code OK 23:06:05 INFO [http_cmd]: http status code OK 23:06:05 DEBUG --- status code --- 23:06:05 INFO [http_cmd]: http status code OK 23:06:05 INFO [http_cmd]: http status code OK 23:06:05 DEBUG { 23:06:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:06:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:06:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:06:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:06:05 DEBUG "data": { 23:06:05 DEBUG "data": { 23:06:05 DEBUG --- status code --- 23:06:05 DEBUG --- status code --- 23:06:05 DEBUG ----- output ----- 23:06:05 DEBUG ----- output ----- 23:06:05 INFO [http_cmd]: http status code OK 23:06:05 DEBUG --- status code --- 23:06:05 DEBUG --- status code --- 23:06:05 DEBUG --- status code --- 23:06:05 DEBUG --- http response --- 23:06:05 INFO [http_cmd]: http status code OK 23:06:05 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 5086Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 14582m 48% 15820Mi 14% gke-xlou-cdm-default-pool-1b0769c8-nhlf 9192m 30% 10797Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 80m 0% 2636Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 3914m 13% 6573Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 20121m 67% 18041Mi 16% gke-xlou-cdm-default-pool-1b0769c8-xnmp 57m 0% 3132Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 8944m 29% 10481Mi 9% gke-xlou-cdm-ds-0f601751-hmm7 4240m 14% 5654Mi 5% gke-xlou-cdm-ds-0f601751-j488 14349m 48% 14965Mi 13% gke-xlou-cdm-ds-0f601751-mc5d 8915m 29% 4179Mi 3% gke-xlou-cdm-ds-0f601751-nj6q 9345m 31% 10858Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 4349m 14% 7358Mi 6% gke-xlou-cdm-frontend-a8771548-i82n 8213m 51% 10472Mi 17% 23:06:05 DEBUG --- status code --- 23:06:05 DEBUG --- status code --- 23:06:05 INFO [http_cmd]: http status code OK 23:06:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:06:05 DEBUG --- status code --- 23:06:05 DEBUG --- status code --- 23:06:05 DEBUG "status": "success", 23:06:05 DEBUG --- http response --- 23:06:05 DEBUG --- http response --- 23:06:05 DEBUG --- http response --- 23:06:05 DEBUG --- http response --- 23:06:05 DEBUG "resultType": "vector", 23:06:05 DEBUG "resultType": "vector", 23:06:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:06:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:06:05 DEBUG { 23:06:05 DEBUG { 23:06:05 DEBUG --- status code --- 23:06:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:06:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:06:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:06:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 23:06:05 DEBUG --- status code --- 23:06:05 DEBUG --- stderr --- 23:06:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:06:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:06:05 DEBUG --- status code --- 23:06:05 DEBUG --- http response --- 23:06:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:06:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:06:05 DEBUG "data": { 23:06:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:06:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 23:06:05 DEBUG "result": [ 23:06:05 INFO [http_cmd]: http status code OK 23:06:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:06:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:06:05 DEBUG "result": [ 23:06:05 DEBUG --- http response --- 23:06:05 DEBUG --- http response --- 23:06:05 DEBUG "status": "success", 23:06:05 INFO [http_cmd]: http status code OK 23:06:05 DEBUG "status": "success", 23:06:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:06:05 DEBUG --- http response --- 23:06:05 DEBUG --- http response --- 23:06:05 DEBUG --- http response --- 23:06:05 DEBUG ----- output ----- 23:06:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:06:05 DEBUG 23:06:05 DEBUG --- http response --- 23:06:05 DEBUG --- http response --- 23:06:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:06:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:06:05 DEBUG --- http response --- 23:06:05 DEBUG --- http response --- 23:06:05 DEBUG "resultType": "vector", 23:06:05 DEBUG ----- output ----- 23:06:05 DEBUG ----- output ----- 23:06:05 DEBUG { 23:06:05 DEBUG --- status code --- 23:06:05 DEBUG ----- output ----- 23:06:05 DEBUG ----- output ----- 23:06:05 DEBUG { 23:06:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:06:05 DEBUG "data": { 23:06:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 23:06:05 DEBUG --- status code --- 23:06:05 DEBUG "data": { 23:06:05 DEBUG --- http response --- 23:06:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:06:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 23:06:05 DEBUG { 23:06:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:06:05 DEBUG --- http response --- 23:06:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:06:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 23:06:05 DEBUG --- http response --- 23:06:05 DEBUG ----- output ----- 23:06:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:06:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:06:05 DEBUG "result": [ 23:06:05 DEBUG { 23:06:05 DEBUG { 23:06:05 DEBUG "metric": { 23:06:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:06:05 DEBUG { 23:06:05 DEBUG { 23:06:05 DEBUG "metric": { 23:06:05 DEBUG ----- output ----- 23:06:05 DEBUG "resultType": "vector", 23:06:05 DEBUG ----- output ----- 23:06:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:06:05 DEBUG "resultType": "vector", 23:06:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:06:05 DEBUG ----- output ----- 23:06:05 DEBUG ----- output ----- 23:06:05 DEBUG "status": "success", 23:06:05 DEBUG ----- output ----- 23:06:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:06:05 DEBUG ----- output ----- 23:06:05 DEBUG ----- output ----- 23:06:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:06:05 DEBUG { 23:06:05 DEBUG ----- output ----- 23:06:05 DEBUG ----- output ----- 23:06:05 DEBUG { 23:06:05 DEBUG "status": "success", 23:06:05 DEBUG "status": "success", 23:06:05 DEBUG "pod": "am-555f4db595-fz8sn" 23:06:05 DEBUG --- http response --- 23:06:05 DEBUG "status": "success", 23:06:05 DEBUG "status": "success", 23:06:05 DEBUG "pod": "am-555f4db595-fz8sn" 23:06:05 DEBUG { 23:06:05 DEBUG "result": [ 23:06:05 DEBUG { 23:06:05 DEBUG --- http response --- 23:06:05 DEBUG "result": [ 23:06:05 DEBUG ----- output ----- 23:06:05 DEBUG { 23:06:05 DEBUG { 23:06:05 DEBUG "data": { 23:06:05 DEBUG { 23:06:05 DEBUG ----- output ----- 23:06:05 DEBUG { 23:06:05 DEBUG { 23:06:05 DEBUG ----- output ----- 23:06:05 DEBUG "status": "success", 23:06:05 DEBUG { 23:06:05 DEBUG { 23:06:05 DEBUG "metric": { 23:06:05 DEBUG "data": { 23:06:05 DEBUG "data": { 23:06:05 DEBUG }, 23:06:05 DEBUG "data": { 23:06:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:06:05 DEBUG "data": { 23:06:05 DEBUG }, 23:06:05 DEBUG "status": "success", 23:06:05 DEBUG { 23:06:05 DEBUG "status": "success", 23:06:05 DEBUG { 23:06:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:06:05 DEBUG { 23:06:05 DEBUG "status": "success", 23:06:05 DEBUG "status": "success", 23:06:05 DEBUG "resultType": "vector", 23:06:05 DEBUG "status": "success", 23:06:05 DEBUG { 23:06:05 DEBUG "status": "success", 23:06:05 DEBUG "status": "success", 23:06:05 DEBUG { 23:06:05 DEBUG "data": { 23:06:05 DEBUG "status": "success", 23:06:05 DEBUG "status": "success", 23:06:05 DEBUG "pod": "ds-idrepo-2" 23:06:05 DEBUG "resultType": "vector", 23:06:05 DEBUG "resultType": "vector", 23:06:05 DEBUG "value": [ 23:06:05 DEBUG "resultType": "vector", 23:06:05 DEBUG ----- output ----- 23:06:05 DEBUG "resultType": "vector", 23:06:05 DEBUG "value": [ 23:06:05 DEBUG "data": { 23:06:05 DEBUG "metric": { 23:06:05 DEBUG "data": { 23:06:05 DEBUG "metric": { 23:06:05 DEBUG ----- output ----- 23:06:05 DEBUG "status": "success", 23:06:05 DEBUG "data": { 23:06:05 DEBUG "data": { 23:06:05 DEBUG "result": [ 23:06:05 DEBUG "data": { 23:06:05 DEBUG "status": "success", 23:06:05 DEBUG "data": { 23:06:05 DEBUG "data": { 23:06:05 DEBUG "status": "success", 23:06:05 DEBUG "resultType": "vector", 23:06:05 DEBUG "data": { 23:06:05 DEBUG "data": { 23:06:05 DEBUG }, 23:06:05 DEBUG "result": [ 23:06:05 DEBUG "result": [ 23:06:05 DEBUG 23:06:05 DEBUG "result": [ 23:06:05 DEBUG { 23:06:05 DEBUG "result": [ 23:06:05 DEBUG 23:06:05 DEBUG "resultType": "vector", 23:06:05 DEBUG "instance": "10.142.0.107:9100" 23:06:05 DEBUG "resultType": "vector", 23:06:05 DEBUG "pod": "ds-idrepo-2" 23:06:05 DEBUG { 23:06:05 DEBUG "data": { 23:06:05 DEBUG "resultType": "vector", 23:06:05 DEBUG "resultType": "vector", 23:06:05 DEBUG { 23:06:05 DEBUG "resultType": "vector", 23:06:05 DEBUG "data": { 23:06:05 DEBUG "resultType": "vector", 23:06:05 DEBUG "resultType": "vector", 23:06:05 DEBUG "data": { 23:06:05 DEBUG "result": [ 23:06:05 DEBUG "resultType": "vector", 23:06:05 DEBUG "resultType": "vector", 23:06:05 DEBUG "value": [ 23:06:05 DEBUG { 23:06:05 DEBUG { 23:06:05 DEBUG 23:06:05 DEBUG { 23:06:05 DEBUG "status": "success", 23:06:05 DEBUG { 23:06:05 DEBUG 23:06:05 DEBUG "result": [ 23:06:05 DEBUG }, 23:06:05 DEBUG "result": [ 23:06:05 DEBUG }, 23:06:05 DEBUG "status": "success", 23:06:05 DEBUG "resultType": "vector", 23:06:05 DEBUG "result": [ 23:06:05 DEBUG "result": [ 23:06:05 DEBUG "metric": { 23:06:05 DEBUG "result": [ 23:06:05 DEBUG "resultType": "vector", 23:06:05 DEBUG "result": [ 23:06:05 DEBUG "result": [ 23:06:05 DEBUG "resultType": "vector", 23:06:05 DEBUG { 23:06:05 DEBUG "result": [ 23:06:05 DEBUG "result": [ 23:06:05 DEBUG 23:06:05 DEBUG "metric": { 23:06:05 DEBUG "metric": { 23:06:05 DEBUG 23:06:05 DEBUG "metric": { 23:06:05 DEBUG "data": { 23:06:05 DEBUG "metric": { 23:06:05 DEBUG 23:06:05 DEBUG { 23:06:05 DEBUG "value": [ 23:06:05 DEBUG { 23:06:05 DEBUG "value": [ 23:06:05 DEBUG "data": { 23:06:05 DEBUG "result": [ 23:06:05 DEBUG { 23:06:05 DEBUG { 23:06:05 DEBUG "pod": "ds-idrepo-2" 23:06:05 DEBUG { 23:06:05 DEBUG "result": [ 23:06:05 DEBUG { 23:06:05 DEBUG { 23:06:05 DEBUG "result": [ 23:06:05 DEBUG "metric": { 23:06:05 DEBUG { 23:06:05 DEBUG { 23:06:05 DEBUG 23:06:05 DEBUG "pod": "am-555f4db595-fz8sn" 23:06:05 DEBUG "pod": "ds-idrepo-2" 23:06:05 DEBUG [...] 23:06:05 DEBUG "instance": "10.142.0.107:9100" 23:06:05 DEBUG "resultType": "vector", 23:06:05 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 23:06:05 DEBUG [...] 23:06:05 DEBUG "metric": { 23:06:05 DEBUG 23:06:05 DEBUG "metric": { 23:06:05 DEBUG 23:06:05 DEBUG "resultType": "vector", 23:06:05 DEBUG { 23:06:05 DEBUG "metric": { 23:06:05 DEBUG "metric": { 23:06:05 DEBUG }, 23:06:05 DEBUG "metric": { 23:06:05 DEBUG { 23:06:05 DEBUG "metric": { 23:06:05 DEBUG "metric": { 23:06:05 DEBUG { 23:06:05 DEBUG "pod": "ds-idrepo-2" 23:06:05 DEBUG "metric": { 23:06:05 DEBUG "metric": { 23:06:05 DEBUG 23:06:05 DEBUG }, 23:06:05 DEBUG }, 23:06:05 DEBUG 23:06:05 DEBUG }, 23:06:05 DEBUG "result": [ 23:06:05 DEBUG }, 23:06:05 DEBUG 23:06:05 DEBUG "pod": "ds-idrepo-2" 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG "pod": "ds-idrepo-2" 23:06:05 DEBUG "result": [ 23:06:05 DEBUG "metric": { 23:06:05 DEBUG "instance": "10.142.0.107:9100" 23:06:05 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 23:06:05 DEBUG "value": [ 23:06:05 DEBUG "pod": "am-555f4db595-fz8sn" 23:06:05 DEBUG "metric": { 23:06:05 DEBUG "pod": "am-555f4db595-fz8sn" 23:06:05 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 23:06:05 DEBUG "metric": { 23:06:05 DEBUG }, 23:06:05 DEBUG "pod": "am-555f4db595-fz8sn" 23:06:05 DEBUG "instance": "10.142.0.107:9100" 23:06:05 DEBUG [...] 23:06:05 DEBUG "value": [ 23:06:05 DEBUG "value": [ 23:06:05 DEBUG 23:06:05 DEBUG "value": [ 23:06:05 DEBUG { 23:06:05 DEBUG "value": [ 23:06:05 DEBUG 23:06:05 DEBUG }, 23:06:05 DEBUG 23:06:05 DEBUG [...] 23:06:05 DEBUG }, 23:06:05 DEBUG { 23:06:05 DEBUG "pod": "am-555f4db595-fz8sn" 23:06:05 DEBUG }, 23:06:05 DEBUG }, 23:06:05 DEBUG 23:06:05 DEBUG }, 23:06:05 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 23:06:05 DEBUG }, 23:06:05 DEBUG }, 23:06:05 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 23:06:05 DEBUG "value": [ 23:06:05 DEBUG }, 23:06:05 DEBUG }, 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG "metric": { 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG "value": [ 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG "value": [ 23:06:05 DEBUG "metric": { 23:06:05 DEBUG }, 23:06:05 DEBUG "value": [ 23:06:05 DEBUG "value": [ 23:06:05 DEBUG 23:06:05 DEBUG "value": [ 23:06:05 DEBUG }, 23:06:05 DEBUG "value": [ 23:06:05 DEBUG "value": [ 23:06:05 DEBUG }, 23:06:05 DEBUG 23:06:05 DEBUG "value": [ 23:06:05 DEBUG "value": [ 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG "pod": "am-555f4db595-kzxsp" 23:06:05 DEBUG 23:06:05 DEBUG "instance": "10.142.0.107:9100" 23:06:05 DEBUG 23:06:05 DEBUG "pod": "am-555f4db595-kzxsp" 23:06:05 DEBUG 23:06:05 DEBUG [...] 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 23:06:05 DEBUG "value": [ 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG "value": [ 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG "value": [ 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG }, 23:06:05 DEBUG 23:06:05 DEBUG }, 23:06:05 DEBUG 23:06:05 DEBUG }, 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG }, 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG [...] 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG "pod": "ds-idrepo-0" 23:06:05 DEBUG [...] 23:06:05 DEBUG [...] 23:06:05 DEBUG "value": [ 23:06:05 DEBUG [...] 23:06:05 DEBUG "value": [ 23:06:05 DEBUG [...] 23:06:05 DEBUG "value": [ 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG "instance": "10.142.0.43:9100" 23:06:05 DEBUG 23:06:05 DEBUG "value": [ 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG [...] 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG }, 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG 1713305165, 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG 1713305165, 23:06:05 DEBUG [...] 23:06:05 DEBUG 23:06:05 DEBUG }, 23:06:05 DEBUG [...] 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG [...] 23:06:05 DEBUG [...] 23:06:05 DEBUG 23:06:05 DEBUG [...] 23:06:05 DEBUG 23:06:05 DEBUG [...] 23:06:05 DEBUG [...] 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG [...] 23:06:05 DEBUG [...] 23:06:05 DEBUG "value": [ 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG "1992.2" 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG "1367.5" 23:06:05 DEBUG 23:06:05 DEBUG "pod": "ds-idrepo-0" 23:06:05 DEBUG "value": [ 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG [...] 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG [...] 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG 1713305165, 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG ] 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG ] 23:06:05 DEBUG 23:06:05 DEBUG }, 23:06:05 DEBUG 1713305165, 23:06:05 DEBUG 23:06:05 DEBUG [...] 23:06:05 DEBUG [...] 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG "pod": "ds-cts-2" 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG "0" 23:06:05 DEBUG "pod": "am-555f4db595-kzxsp" 23:06:05 DEBUG "pod": "ds-idrepo-0" 23:06:05 DEBUG } 23:06:05 DEBUG ] 23:06:05 DEBUG [...] 23:06:05 DEBUG "pod": "end-user-ui-58878d7668-zg4h2" 23:06:05 DEBUG } 23:06:05 DEBUG 23:06:05 DEBUG "value": [ 23:06:05 DEBUG "0.0005333333333330377" 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG }, 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG "pod": "ds-idrepo-0" 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG ] 23:06:05 DEBUG }, 23:06:05 DEBUG }, 23:06:05 DEBUG } 23:06:05 DEBUG "instance": "10.142.0.43:9100" 23:06:05 DEBUG 23:06:05 DEBUG }, 23:06:05 DEBUG ] 23:06:05 DEBUG "pod": "ds-idrepo-0" 23:06:05 DEBUG 1713305165, 23:06:05 DEBUG ] 23:06:05 DEBUG "pod": "ds-idrepo-0" 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG "instance": "10.142.0.43:9100" 23:06:05 DEBUG "pod": "end-user-ui-58878d7668-zg4h2" 23:06:05 DEBUG "value": [ 23:06:05 DEBUG "pod": "am-555f4db595-kzxsp" 23:06:05 DEBUG 23:06:05 DEBUG "pod": "am-555f4db595-kzxsp" 23:06:05 DEBUG "pod": "end-user-ui-58878d7668-zg4h2" 23:06:05 DEBUG 23:06:05 DEBUG }, 23:06:05 DEBUG "pod": "am-555f4db595-kzxsp" 23:06:05 DEBUG "instance": "10.142.0.43:9100" 23:06:05 DEBUG } 23:06:05 DEBUG "value": [ 23:06:05 DEBUG "value": [ 23:06:05 DEBUG } 23:06:05 DEBUG }, 23:06:05 DEBUG 23:06:05 DEBUG "value": [ 23:06:05 DEBUG } 23:06:05 DEBUG }, 23:06:05 DEBUG "0" 23:06:05 DEBUG } 23:06:05 DEBUG }, 23:06:05 DEBUG 23:06:05 DEBUG 23:06:05 DEBUG }, 23:06:05 DEBUG }, 23:06:05 DEBUG 1713305165, 23:06:05 DEBUG }, 23:06:05 DEBUG "pod": "end-user-ui-58878d7668-zg4h2" 23:06:05 DEBUG }, 23:06:05 DEBUG }, 23:06:05 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 23:06:05 DEBUG "value": [ 23:06:05 DEBUG }, 23:06:05 DEBUG }, 23:06:05 DEBUG ] 23:06:05 DEBUG 1713305165, 23:06:05 DEBUG 1713305165, 23:06:05 DEBUG "value": [ 23:06:05 DEBUG 23:06:05 DEBUG 1713305165, 23:06:05 DEBUG } 23:06:05 DEBUG "value": [ 23:06:05 DEBUG ] 23:06:05 DEBUG ] 23:06:05 DEBUG "value": [ 23:06:05 DEBUG "pod": "am-555f4db595-kzxsp" 23:06:05 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 23:06:05 DEBUG "value": [ 23:06:05 DEBUG "value": [ 23:06:05 DEBUG "0" 23:06:05 DEBUG "value": [ 23:06:05 DEBUG }, 23:06:05 DEBUG "value": [ 23:06:05 DEBUG "value": [ 23:06:05 DEBUG }, 23:06:05 DEBUG 1713305165, 23:06:05 DEBUG "value": [ 23:06:05 DEBUG "value": [ 23:06:05 DEBUG } 23:06:05 DEBUG "1363.6333333333334" 23:06:05 DEBUG "0" 23:06:05 DEBUG 1713305165, 23:06:05 DEBUG "instance": "10.142.0.43:9100" 23:06:05 DEBUG "0" 23:06:05 DEBUG 1713305165, 23:06:05 DEBUG } 23:06:05 DEBUG } 23:06:05 DEBUG 1713305165, 23:06:05 DEBUG }, 23:06:05 DEBUG }, 23:06:05 DEBUG 1713305165, 23:06:05 DEBUG 1713305165, 23:06:05 DEBUG ] 23:06:05 DEBUG 1713305165, 23:06:05 DEBUG "value": [ 23:06:05 DEBUG 1713305165, 23:06:05 DEBUG 1713305165, 23:06:05 DEBUG "value": [ 23:06:05 DEBUG "0" 23:06:05 DEBUG 1713305165, 23:06:05 DEBUG 1713305165, 23:06:05 DEBUG } 23:06:05 DEBUG ] 23:06:05 DEBUG ] 23:06:05 DEBUG "32228012.36666667" 23:06:05 DEBUG }, 23:06:05 DEBUG ] 23:06:05 DEBUG "0" 23:06:05 DEBUG ] 23:06:05 DEBUG } 23:06:05 DEBUG "0" 23:06:05 DEBUG "value": [ 23:06:05 DEBUG "value": [ 23:06:05 DEBUG "0" 23:06:05 DEBUG "0" 23:06:05 DEBUG } 23:06:05 DEBUG "1369.5333333333333" 23:06:05 DEBUG 1713305165, 23:06:05 DEBUG "3645866" 23:06:05 DEBUG "0" 23:06:05 DEBUG 1713305165, 23:06:05 DEBUG ] 23:06:05 DEBUG "0" 23:06:05 DEBUG "32222792.166666664" 23:06:05 DEBUG } 23:06:05 DEBUG } 23:06:05 DEBUG ] 23:06:05 DEBUG "value": [ 23:06:05 DEBUG } 23:06:05 DEBUG ] 23:06:05 DEBUG } 23:06:05 DEBUG ] 23:06:05 DEBUG 1713305165, 23:06:05 DEBUG 1713305165, 23:06:05 DEBUG ] 23:06:05 DEBUG ] 23:06:05 DEBUG ] 23:06:05 DEBUG ] 23:06:05 DEBUG "0.00003702237893070254" 23:06:05 DEBUG ] 23:06:05 DEBUG ] 23:06:05 DEBUG "4804202496" 23:06:05 DEBUG } 23:06:05 DEBUG ] 23:06:05 DEBUG ] 23:06:05 DEBUG ] 23:06:05 DEBUG ] 23:06:05 DEBUG } 23:06:05 DEBUG 1713305165, 23:06:05 DEBUG ] 23:06:05 DEBUG } 23:06:05 DEBUG } 23:06:05 DEBUG } 23:06:05 DEBUG "0.027" 23:06:05 DEBUG "8.020481885072831" 23:06:05 DEBUG } 23:06:05 DEBUG } 23:06:05 DEBUG } 23:06:05 DEBUG } 23:06:05 DEBUG ] 23:06:05 DEBUG } 23:06:05 DEBUG } 23:06:05 DEBUG ] 23:06:05 DEBUG ] 23:06:05 DEBUG } 23:06:05 DEBUG } 23:06:05 DEBUG } 23:06:05 DEBUG } 23:06:05 DEBUG ] 23:06:05 DEBUG "19984796672" 23:06:05 DEBUG } 23:06:05 DEBUG ] 23:06:05 DEBUG ] 23:06:05 DEBUG ] 23:06:05 DEBUG ] 23:06:05 DEBUG ] 23:06:05 DEBUG ] 23:06:05 DEBUG } 23:06:05 DEBUG ] 23:06:05 DEBUG } 23:06:05 DEBUG ] 23:06:05 DEBUG ] 23:06:05 DEBUG } 23:06:05 DEBUG } 23:06:05 DEBUG ] 23:06:05 DEBUG ] 23:06:05 DEBUG } 23:06:05 DEBUG } 23:06:05 DEBUG } 23:06:05 DEBUG ] 23:06:05 DEBUG } 23:06:05 DEBUG } 23:06:05 DEBUG } 23:06:05 DEBUG } 23:06:05 DEBUG } 23:06:05 DEBUG } 23:06:05 DEBUG } 23:06:05 DEBUG } 23:06:05 DEBUG ] 23:06:05 DEBUG } 23:06:05 DEBUG } 23:06:05 DEBUG ] 23:06:05 DEBUG } 23:06:05 DEBUG } 23:06:05 DEBUG } 23:06:05 DEBUG } 23:06:05 DEBUG } 23:06:05 DEBUG } 23:06:05 DEBUG } 23:06:05 DEBUG ] 23:06:05 DEBUG ] 23:06:05 DEBUG } 23:06:05 DEBUG } 23:06:05 DEBUG } 23:06:05 DEBUG } 23:06:05 DEBUG } 23:06:05 DEBUG } 23:06:05 DEBUG } 23:06:05 DEBUG } 23:06:05 DEBUG } 23:06:05 DEBUG ] 23:06:05 DEBUG } 23:06:05 DEBUG } 23:06:05 DEBUG } 23:06:05 DEBUG } 23:06:05 DEBUG } 23:06:05 DEBUG } 23:06:05 DEBUG } 23:06:05 DEBUG } 23:06:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713305218" 23:06:58 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305218" 23:06:58 ERROR [http_cmd]: ERROR 23:06:58 INFO [http_cmd]: http status code OK 23:06:58 ERROR --- status code --- 23:06:58 DEBUG --- status code --- 23:06:58 ERROR http status code is 400 (expected [200, 401, 403, 404]) 23:06:58 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:06:58 ERROR --- http response --- 23:06:58 DEBUG --- http response --- 23:06:58 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 23:06:58 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 23:06:58 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 23:07:01 INFO 23:07:01 INFO [loop_until]: kubectl --namespace=xlou top pods 23:07:01 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 23:07:02 INFO [loop_until]: OK (rc = 0) 23:07:02 DEBUG --- stdout --- 23:07:02 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 9Mi am-555f4db595-4lbss 9430m 8682Mi am-555f4db595-fz8sn 9213m 8565Mi am-555f4db595-kzxsp 9359m 8975Mi ds-cts-0 19511m 12955Mi ds-cts-1 14686m 13511Mi ds-cts-2 16000m 13369Mi ds-idrepo-0 4078m 5033Mi ds-idrepo-1 3931m 2900Mi ds-idrepo-2 4052m 3222Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 4m 1207Mi idm-b85f9d978-rl76f 3m 1225Mi lodemon-bc756c647-cr7x6 1m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-7c9b667974-9mr2s 8079m 6752Mi 23:07:02 DEBUG --- stderr --- 23:07:02 DEBUG 23:07:05 INFO 23:07:05 INFO [loop_until]: kubectl --namespace=xlou top node 23:07:05 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 23:07:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305225" 23:07:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305225" 23:07:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305225" 23:07:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713305225" 23:07:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305225" 23:07:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713305225" 23:07:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305225" 23:07:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305225" 23:07:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305225" 23:07:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305225" 23:07:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713305225" 23:07:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305225" 23:07:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305225" 23:07:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713305225" 23:07:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305225" 23:07:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305225" 23:07:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305225" 23:07:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305225" 23:07:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713305225" 23:07:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713305225" 23:07:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713305225" 23:07:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305225" 23:07:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305225" 23:07:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713305225" 23:07:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713305225" 23:07:05 INFO [http_cmd]: http status code OK 23:07:05 INFO [http_cmd]: http status code OK 23:07:05 INFO [http_cmd]: http status code OK 23:07:05 INFO [http_cmd]: http status code OK 23:07:05 INFO [http_cmd]: http status code OK 23:07:05 INFO [http_cmd]: http status code OK 23:07:05 INFO [http_cmd]: http status code OK 23:07:05 INFO [http_cmd]: http status code OK 23:07:05 INFO [http_cmd]: http status code OK 23:07:05 INFO [http_cmd]: http status code OK 23:07:05 INFO [http_cmd]: http status code OK 23:07:05 INFO [http_cmd]: http status code OK 23:07:05 INFO [http_cmd]: http status code OK 23:07:05 INFO [http_cmd]: http status code OK 23:07:05 INFO [http_cmd]: http status code OK 23:07:05 DEBUG --- status code --- 23:07:05 INFO [loop_until]: OK (rc = 0) 23:07:05 DEBUG --- status code --- 23:07:05 INFO [http_cmd]: http status code OK 23:07:05 INFO [http_cmd]: http status code OK 23:07:05 DEBUG --- status code --- 23:07:05 DEBUG --- status code --- 23:07:05 INFO [http_cmd]: http status code OK 23:07:05 DEBUG --- status code --- 23:07:05 DEBUG --- status code --- 23:07:05 DEBUG --- status code --- 23:07:05 DEBUG --- status code --- 23:07:05 DEBUG --- status code --- 23:07:05 DEBUG --- status code --- 23:07:05 DEBUG --- status code --- 23:07:05 DEBUG --- status code --- 23:07:05 INFO [http_cmd]: http status code OK 23:07:05 DEBUG --- status code --- 23:07:05 DEBUG --- status code --- 23:07:05 DEBUG --- status code --- 23:07:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:07:05 DEBUG --- stdout --- 23:07:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:07:05 DEBUG --- status code --- 23:07:05 DEBUG --- status code --- 23:07:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:07:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:07:05 DEBUG --- status code --- 23:07:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:07:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:07:05 INFO [http_cmd]: http status code OK 23:07:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:07:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:07:05 INFO [http_cmd]: http status code OK 23:07:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:07:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:07:05 INFO [http_cmd]: http status code OK 23:07:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:07:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:07:05 INFO [http_cmd]: http status code OK 23:07:05 DEBUG --- status code --- 23:07:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:07:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:07:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:07:05 DEBUG --- http response --- 23:07:05 DEBUG --- http response --- 23:07:05 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 63m 0% 5084Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 14826m 49% 15900Mi 14% gke-xlou-cdm-default-pool-1b0769c8-nhlf 9285m 31% 10804Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 83m 0% 2634Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 3966m 13% 6586Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 20312m 68% 17475Mi 15% gke-xlou-cdm-default-pool-1b0769c8-xnmp 58m 0% 3136Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 9548m 31% 10483Mi 9% gke-xlou-cdm-ds-0f601751-hmm7 4280m 14% 5652Mi 5% gke-xlou-cdm-ds-0f601751-j488 15006m 50% 15293Mi 13% gke-xlou-cdm-ds-0f601751-mc5d 9226m 30% 4206Mi 3% gke-xlou-cdm-ds-0f601751-nj6q 9880m 33% 10858Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 4476m 14% 7364Mi 6% gke-xlou-cdm-frontend-a8771548-i82n 8282m 52% 11654Mi 19% 23:07:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:07:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:07:05 DEBUG --- http response --- 23:07:05 DEBUG --- http response --- 23:07:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:07:05 DEBUG --- http response --- 23:07:05 DEBUG --- http response --- 23:07:05 DEBUG --- status code --- 23:07:05 DEBUG --- http response --- 23:07:05 INFO [http_cmd]: http status code OK 23:07:05 DEBUG --- http response --- 23:07:05 DEBUG --- status code --- 23:07:05 DEBUG --- http response --- 23:07:05 DEBUG --- http response --- 23:07:05 DEBUG --- status code --- 23:07:05 DEBUG --- http response --- 23:07:05 DEBUG --- http response --- 23:07:05 DEBUG --- status code --- 23:07:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:07:05 DEBUG --- http response --- 23:07:05 DEBUG --- http response --- 23:07:05 DEBUG --- http response --- 23:07:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 23:07:05 DEBUG --- stderr --- 23:07:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:07:05 INFO [http_cmd]: http status code OK 23:07:05 DEBUG --- http response --- 23:07:05 DEBUG --- http response --- 23:07:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:07:05 DEBUG --- http response --- 23:07:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 23:07:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:07:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:07:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:07:05 DEBUG --- status code --- 23:07:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:07:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:07:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:07:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:07:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 23:07:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:07:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 23:07:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 23:07:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:07:05 DEBUG --- http response --- 23:07:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:07:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:07:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:07:05 DEBUG ----- output ----- 23:07:05 DEBUG 23:07:05 DEBUG ----- output ----- 23:07:05 DEBUG --- status code --- 23:07:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:07:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:07:05 DEBUG ----- output ----- 23:07:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:07:05 DEBUG ----- output ----- 23:07:05 DEBUG ----- output ----- 23:07:05 DEBUG --- http response --- 23:07:05 DEBUG ----- output ----- 23:07:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:07:05 DEBUG ----- output ----- 23:07:05 DEBUG ----- output ----- 23:07:05 DEBUG --- http response --- 23:07:05 DEBUG ----- output ----- 23:07:05 DEBUG ----- output ----- 23:07:05 DEBUG --- http response --- 23:07:05 DEBUG ----- output ----- 23:07:05 DEBUG ----- output ----- 23:07:05 DEBUG --- http response --- 23:07:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:07:05 DEBUG ----- output ----- 23:07:05 DEBUG ----- output ----- 23:07:05 DEBUG ----- output ----- 23:07:05 DEBUG { 23:07:05 DEBUG { 23:07:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:07:05 DEBUG ----- output ----- 23:07:05 DEBUG ----- output ----- 23:07:05 DEBUG { 23:07:05 DEBUG ----- output ----- 23:07:05 DEBUG { 23:07:05 DEBUG { 23:07:05 DEBUG { 23:07:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:07:05 DEBUG --- http response --- 23:07:05 DEBUG { 23:07:05 DEBUG { 23:07:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:07:05 DEBUG { 23:07:05 DEBUG { 23:07:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:07:05 DEBUG { 23:07:05 DEBUG { 23:07:05 DEBUG ----- output ----- 23:07:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 23:07:05 DEBUG { 23:07:05 DEBUG { 23:07:05 DEBUG { 23:07:05 DEBUG "status": "success", 23:07:05 DEBUG "status": "success", 23:07:05 DEBUG --- http response --- 23:07:05 DEBUG { 23:07:05 DEBUG { 23:07:05 DEBUG "status": "success", 23:07:05 DEBUG { 23:07:05 DEBUG "status": "success", 23:07:05 DEBUG "status": "success", 23:07:05 DEBUG "status": "success", 23:07:05 DEBUG ----- output ----- 23:07:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:07:05 DEBUG "status": "success", 23:07:05 DEBUG "status": "success", 23:07:05 DEBUG ----- output ----- 23:07:05 DEBUG "status": "success", 23:07:05 DEBUG "status": "success", 23:07:05 DEBUG "data": { 23:07:05 DEBUG "status": "success", 23:07:05 DEBUG "status": "success", 23:07:05 DEBUG { 23:07:05 DEBUG ----- output ----- 23:07:05 DEBUG { 23:07:05 DEBUG "status": "success", 23:07:05 DEBUG "data": { 23:07:05 DEBUG "data": { 23:07:05 DEBUG "data": { 23:07:05 DEBUG "status": "success", 23:07:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:07:05 DEBUG ----- output ----- 23:07:05 DEBUG "data": { 23:07:05 DEBUG "status": "success", 23:07:05 DEBUG "data": { 23:07:05 DEBUG "data": { 23:07:05 DEBUG "data": { 23:07:05 DEBUG { 23:07:05 DEBUG ----- output ----- 23:07:05 DEBUG "data": { 23:07:05 DEBUG "data": { 23:07:05 DEBUG { 23:07:05 DEBUG "data": { 23:07:05 DEBUG ----- output ----- 23:07:05 DEBUG "resultType": "vector", 23:07:05 DEBUG "data": { 23:07:05 DEBUG "data": { 23:07:05 DEBUG "status": "success", 23:07:05 DEBUG "status": "success", 23:07:05 DEBUG "status": "success", 23:07:05 DEBUG "resultType": "vector", 23:07:05 DEBUG "status": "success", 23:07:05 DEBUG "resultType": "vector", 23:07:05 DEBUG "resultType": "vector", 23:07:05 DEBUG "data": { 23:07:05 DEBUG { 23:07:05 DEBUG "status": "success", 23:07:05 DEBUG "data": { 23:07:05 DEBUG "resultType": "vector", 23:07:05 DEBUG "result": [ 23:07:05 DEBUG { 23:07:05 DEBUG "metric": { 23:07:05 DEBUG "status": "success", 23:07:05 DEBUG { 23:07:05 DEBUG "status": "success", 23:07:05 DEBUG "resultType": "vector", 23:07:05 DEBUG "status": "success", 23:07:05 DEBUG "resultType": "vector", 23:07:05 DEBUG { 23:07:05 DEBUG "result": [ 23:07:05 DEBUG "resultType": "vector", 23:07:05 DEBUG "resultType": "vector", 23:07:05 DEBUG "data": { 23:07:05 DEBUG "data": { 23:07:05 DEBUG "data": { 23:07:05 DEBUG "result": [ 23:07:05 DEBUG "data": { 23:07:05 DEBUG "result": [ 23:07:05 DEBUG "result": [ 23:07:05 DEBUG "resultType": "vector", 23:07:05 DEBUG "status": "success", 23:07:05 DEBUG "resultType": "vector", 23:07:05 DEBUG "data": { 23:07:05 DEBUG "pod": "ds-cts-1" 23:07:05 DEBUG "resultType": "vector", 23:07:05 DEBUG "resultType": "vector", 23:07:05 DEBUG "resultType": "vector", 23:07:05 DEBUG "result": [ 23:07:05 DEBUG "resultType": "vector", 23:07:05 DEBUG "data": { 23:07:05 DEBUG "result": [ 23:07:05 DEBUG "data": { 23:07:05 DEBUG "result": [ 23:07:05 DEBUG "status": "success", 23:07:05 DEBUG "data": { 23:07:05 DEBUG { 23:07:05 DEBUG "result": [ 23:07:05 DEBUG "resultType": "vector", 23:07:05 DEBUG "resultType": "vector", 23:07:05 DEBUG "resultType": "vector", 23:07:05 DEBUG { 23:07:05 DEBUG "resultType": "vector", 23:07:05 DEBUG { 23:07:05 DEBUG { 23:07:05 DEBUG "result": [ 23:07:05 DEBUG "data": { 23:07:05 DEBUG "result": [ 23:07:05 DEBUG "resultType": "vector", 23:07:05 DEBUG }, 23:07:05 DEBUG "result": [ 23:07:05 DEBUG "result": [ 23:07:05 DEBUG { 23:07:05 DEBUG "data": { 23:07:05 DEBUG "result": [ 23:07:05 DEBUG "resultType": "vector", 23:07:05 DEBUG { 23:07:05 DEBUG "resultType": "vector", 23:07:05 DEBUG { 23:07:05 DEBUG "metric": { 23:07:05 DEBUG "result": [ 23:07:05 DEBUG "metric": { 23:07:05 DEBUG { 23:07:05 DEBUG "result": [ 23:07:05 DEBUG "result": [ 23:07:05 DEBUG "result": [ 23:07:05 DEBUG "metric": { 23:07:05 DEBUG "result": [ 23:07:05 DEBUG "metric": { 23:07:05 DEBUG "metric": { 23:07:05 DEBUG { 23:07:05 DEBUG "resultType": "vector", 23:07:05 DEBUG { 23:07:05 DEBUG "result": [ 23:07:05 DEBUG "value": [ 23:07:05 DEBUG { 23:07:05 DEBUG { 23:07:05 DEBUG "metric": { 23:07:05 DEBUG "resultType": "vector", 23:07:05 DEBUG { 23:07:05 DEBUG "result": [ 23:07:05 DEBUG "metric": { 23:07:05 DEBUG "result": [ 23:07:05 DEBUG "pod": "am-555f4db595-fz8sn" 23:07:05 DEBUG { 23:07:05 DEBUG "pod": "ds-cts-1" 23:07:05 DEBUG "metric": { 23:07:05 DEBUG { 23:07:05 DEBUG { 23:07:05 DEBUG "metric": { 23:07:05 DEBUG { 23:07:05 DEBUG { 23:07:05 DEBUG "pod": "ds-idrepo-2" 23:07:05 DEBUG "instance": "10.142.0.107:9100" 23:07:05 DEBUG "metric": { 23:07:05 DEBUG "result": [ 23:07:05 DEBUG "metric": { 23:07:05 DEBUG "resultType": "vector", 23:07:05 DEBUG { 23:07:05 DEBUG 23:07:05 DEBUG "metric": { 23:07:05 DEBUG "metric": { 23:07:05 DEBUG "pod": "am-555f4db595-fz8sn" 23:07:05 DEBUG "result": [ 23:07:05 DEBUG "metric": { 23:07:05 DEBUG { 23:07:05 DEBUG "instance": "10.142.0.107:9100" 23:07:05 DEBUG { 23:07:05 DEBUG }, 23:07:05 DEBUG "value": [ 23:07:05 DEBUG }, 23:07:05 DEBUG "pod": "ds-idrepo-2" 23:07:05 DEBUG "metric": { 23:07:05 DEBUG "pod": "ds-cts-1" 23:07:05 DEBUG "pod": "am-555f4db595-fz8sn" 23:07:05 DEBUG "metric": { 23:07:05 DEBUG "metric": { 23:07:05 DEBUG }, 23:07:05 DEBUG }, 23:07:05 DEBUG "pod": "am-555f4db595-fz8sn" 23:07:05 DEBUG { 23:07:05 DEBUG "pod": "ds-idrepo-2" 23:07:05 DEBUG "result": [ 23:07:05 DEBUG "metric": { 23:07:05 DEBUG 23:07:05 DEBUG "pod": "ds-idrepo-2" 23:07:05 DEBUG "pod": "am-555f4db595-fz8sn" 23:07:05 DEBUG }, 23:07:05 DEBUG { 23:07:05 DEBUG "pod": "ds-idrepo-2" 23:07:05 DEBUG "metric": { 23:07:05 DEBUG }, 23:07:05 DEBUG "metric": { 23:07:05 DEBUG 23:07:05 DEBUG "value": [ 23:07:05 DEBUG }, 23:07:05 DEBUG "instance": "10.142.0.107:9100" 23:07:05 DEBUG }, 23:07:05 DEBUG }, 23:07:05 DEBUG "pod": "ds-idrepo-2" 23:07:05 DEBUG "instance": "10.142.0.107:9100" 23:07:05 DEBUG "value": [ 23:07:05 DEBUG "value": [ 23:07:05 DEBUG }, 23:07:05 DEBUG "metric": { 23:07:05 DEBUG }, 23:07:05 DEBUG { 23:07:05 DEBUG "metric": { 23:07:05 DEBUG "instance": "10.142.0.107:9100" 23:07:05 DEBUG 23:07:05 DEBUG }, 23:07:05 DEBUG }, 23:07:05 DEBUG "value": [ 23:07:05 DEBUG "metric": { 23:07:05 DEBUG }, 23:07:05 DEBUG "pod": "am-555f4db595-fz8sn" 23:07:05 DEBUG "value": [ 23:07:05 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG "value": [ 23:07:05 DEBUG }, 23:07:05 DEBUG "value": [ 23:07:05 DEBUG "value": [ 23:07:05 DEBUG }, 23:07:05 DEBUG }, 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG "value": [ 23:07:05 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 23:07:05 DEBUG "value": [ 23:07:05 DEBUG "metric": { 23:07:05 DEBUG "pod": "ds-idrepo-2" 23:07:05 DEBUG }, 23:07:05 DEBUG [...] 23:07:05 DEBUG "value": [ 23:07:05 DEBUG "value": [ 23:07:05 DEBUG 23:07:05 DEBUG "pod": "ds-cts-1" 23:07:05 DEBUG "value": [ 23:07:05 DEBUG }, 23:07:05 DEBUG 23:07:05 DEBUG }, 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG "value": [ 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG "value": [ 23:07:05 DEBUG "value": [ 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG }, 23:07:05 DEBUG 23:07:05 DEBUG "pod": "am-555f4db595-fz8sn" 23:07:05 DEBUG }, 23:07:05 DEBUG "value": [ 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG }, 23:07:05 DEBUG 23:07:05 DEBUG "value": [ 23:07:05 DEBUG 23:07:05 DEBUG "value": [ 23:07:05 DEBUG [...] 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG "value": [ 23:07:05 DEBUG 23:07:05 DEBUG }, 23:07:05 DEBUG "value": [ 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG "value": [ 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG [...] 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG [...] 23:07:05 DEBUG [...] 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG "value": [ 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG [...] 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG [...] 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG [...] 23:07:05 DEBUG 23:07:05 DEBUG [...] 23:07:05 DEBUG [...] 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG [...] 23:07:05 DEBUG 23:07:05 DEBUG [...] 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 23:07:05 DEBUG [...] 23:07:05 DEBUG [...] 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG [...] 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG [...] 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG [...] 23:07:05 DEBUG [...] 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG [...] 23:07:05 DEBUG }, 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG [...] 23:07:05 DEBUG 23:07:05 DEBUG [...] 23:07:05 DEBUG "pod": "am-555f4db595-kzxsp" 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG [...] 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG [...] 23:07:05 DEBUG 23:07:05 DEBUG "value": [ 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG [...] 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG }, 23:07:05 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG "pod": "ds-cts-2" 23:07:05 DEBUG "instance": "10.142.0.43:9100" 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG [...] 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG 1713305225, 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG "pod": "am-555f4db595-kzxsp" 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG "instance": "10.142.0.43:9100" 23:07:05 DEBUG 23:07:05 DEBUG "value": [ 23:07:05 DEBUG }, 23:07:05 DEBUG "pod": "ds-idrepo-0" 23:07:05 DEBUG 23:07:05 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 23:07:05 DEBUG "pod": "am-555f4db595-kzxsp" 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG }, 23:07:05 DEBUG }, 23:07:05 DEBUG "pod": "am-555f4db595-kzxsp" 23:07:05 DEBUG 23:07:05 DEBUG "pod": "ds-idrepo-0" 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG "0" 23:07:05 DEBUG "pod": "ds-idrepo-0" 23:07:05 DEBUG "pod": "am-555f4db595-kzxsp" 23:07:05 DEBUG }, 23:07:05 DEBUG 23:07:05 DEBUG "pod": "ds-idrepo-0" 23:07:05 DEBUG 23:07:05 DEBUG }, 23:07:05 DEBUG 23:07:05 DEBUG 1713305225, 23:07:05 DEBUG "value": [ 23:07:05 DEBUG }, 23:07:05 DEBUG "instance": "10.142.0.43:9100" 23:07:05 DEBUG }, 23:07:05 DEBUG }, 23:07:05 DEBUG "pod": "ds-idrepo-0" 23:07:05 DEBUG "instance": "10.142.0.43:9100" 23:07:05 DEBUG "value": [ 23:07:05 DEBUG "value": [ 23:07:05 DEBUG }, 23:07:05 DEBUG 23:07:05 DEBUG }, 23:07:05 DEBUG 23:07:05 DEBUG 23:07:05 DEBUG "instance": "10.142.0.43:9100" 23:07:05 DEBUG ] 23:07:05 DEBUG }, 23:07:05 DEBUG }, 23:07:05 DEBUG "value": [ 23:07:05 DEBUG 23:07:05 DEBUG }, 23:07:05 DEBUG "pod": "am-555f4db595-kzxsp" 23:07:05 DEBUG "value": [ 23:07:05 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 23:07:05 DEBUG "1406.9666666666667" 23:07:05 DEBUG 1713305225, 23:07:05 DEBUG "value": [ 23:07:05 DEBUG }, 23:07:05 DEBUG "value": [ 23:07:05 DEBUG "value": [ 23:07:05 DEBUG }, 23:07:05 DEBUG }, 23:07:05 DEBUG 1713305225, 23:07:05 DEBUG 1713305225, 23:07:05 DEBUG "value": [ 23:07:05 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 23:07:05 DEBUG "value": [ 23:07:05 DEBUG 23:07:05 DEBUG "pod": "ds-idrepo-0" 23:07:05 DEBUG }, 23:07:05 DEBUG } 23:07:05 DEBUG "value": [ 23:07:05 DEBUG "value": [ 23:07:05 DEBUG 1713305225, 23:07:05 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 23:07:05 DEBUG }, 23:07:05 DEBUG }, 23:07:05 DEBUG 1713305225, 23:07:05 DEBUG }, 23:07:05 DEBUG ] 23:07:05 DEBUG "0" 23:07:05 DEBUG 1713305225, 23:07:05 DEBUG "value": [ 23:07:05 DEBUG 1713305225, 23:07:05 DEBUG 1713305225, 23:07:05 DEBUG "value": [ 23:07:05 DEBUG "value": [ 23:07:05 DEBUG "0" 23:07:05 DEBUG "0.00046666666666700016" 23:07:05 DEBUG 1713305225, 23:07:05 DEBUG }, 23:07:05 DEBUG 1713305225, 23:07:05 DEBUG "pod": "am-555f4db595-kzxsp" 23:07:05 DEBUG }, 23:07:05 DEBUG "value": [ 23:07:05 DEBUG ] 23:07:05 DEBUG 1713305225, 23:07:05 DEBUG 1713305225, 23:07:05 DEBUG "1407.5" 23:07:05 DEBUG "value": [ 23:07:05 DEBUG "value": [ 23:07:05 DEBUG "value": [ 23:07:05 DEBUG "31626535.333333336" 23:07:05 DEBUG "value": [ 23:07:05 DEBUG } 23:07:05 DEBUG ] 23:07:05 DEBUG "0" 23:07:05 DEBUG 1713305225, 23:07:05 DEBUG "0.00002352360077794367" 23:07:05 DEBUG "1407.7333333333333" 23:07:05 DEBUG 1713305225, 23:07:05 DEBUG 1713305225, 23:07:05 DEBUG ] 23:07:05 DEBUG ] 23:07:05 DEBUG "3608823.5" 23:07:05 DEBUG "value": [ 23:07:05 DEBUG "0" 23:07:05 DEBUG }, 23:07:05 DEBUG "value": [ 23:07:05 DEBUG 1713305225, 23:07:05 DEBUG } 23:07:05 DEBUG "0" 23:07:05 DEBUG "2001.8999999999999" 23:07:05 DEBUG ] 23:07:05 DEBUG 1713305225, 23:07:05 DEBUG 1713305225, 23:07:05 DEBUG 1713305225, 23:07:05 DEBUG ] 23:07:05 DEBUG 1713305225, 23:07:05 DEBUG ] 23:07:05 DEBUG } 23:07:05 DEBUG ] 23:07:05 DEBUG "19985527808" 23:07:05 DEBUG ] 23:07:05 DEBUG ] 23:07:05 DEBUG "0" 23:07:05 DEBUG "0.00033333333333333365" 23:07:05 DEBUG } 23:07:05 DEBUG } 23:07:05 DEBUG ] 23:07:05 DEBUG 1713305225, 23:07:05 DEBUG ] 23:07:05 DEBUG "value": [ 23:07:05 DEBUG 1713305225, 23:07:05 DEBUG "31620086.633333333" 23:07:05 DEBUG } 23:07:05 DEBUG ] 23:07:05 DEBUG ] 23:07:05 DEBUG } 23:07:05 DEBUG "0" 23:07:05 DEBUG "0" 23:07:05 DEBUG "0.027" 23:07:05 DEBUG } 23:07:05 DEBUG "7003914240" 23:07:05 DEBUG } 23:07:05 DEBUG ] 23:07:05 DEBUG } 23:07:05 DEBUG ] 23:07:05 DEBUG } 23:07:05 DEBUG } 23:07:05 DEBUG ] 23:07:05 DEBUG ] 23:07:05 DEBUG ] 23:07:05 DEBUG ] 23:07:05 DEBUG } 23:07:05 DEBUG "8.034234038338546" 23:07:05 DEBUG } 23:07:05 DEBUG 1713305225, 23:07:05 DEBUG "0" 23:07:05 DEBUG ] 23:07:05 DEBUG } 23:07:05 DEBUG } 23:07:05 DEBUG ] 23:07:05 DEBUG ] 23:07:05 DEBUG ] 23:07:05 DEBUG ] 23:07:05 DEBUG ] 23:07:05 DEBUG ] 23:07:05 DEBUG } 23:07:05 DEBUG } 23:07:05 DEBUG ] 23:07:05 DEBUG } 23:07:05 DEBUG ] 23:07:05 DEBUG ] 23:07:05 DEBUG } 23:07:05 DEBUG } 23:07:05 DEBUG } 23:07:05 DEBUG } 23:07:05 DEBUG ] 23:07:05 DEBUG ] 23:07:05 DEBUG ] 23:07:05 DEBUG "0" 23:07:05 DEBUG ] 23:07:05 DEBUG } 23:07:05 DEBUG ] 23:07:05 DEBUG ] 23:07:05 DEBUG } 23:07:05 DEBUG } 23:07:05 DEBUG } 23:07:05 DEBUG } 23:07:05 DEBUG } 23:07:05 DEBUG } 23:07:05 DEBUG } 23:07:05 DEBUG } 23:07:05 DEBUG ] 23:07:05 DEBUG } 23:07:05 DEBUG } 23:07:05 DEBUG ] 23:07:05 DEBUG ] 23:07:05 DEBUG } 23:07:05 DEBUG } 23:07:05 DEBUG } 23:07:05 DEBUG } 23:07:05 DEBUG } 23:07:05 DEBUG ] 23:07:05 DEBUG } 23:07:05 DEBUG ] 23:07:05 DEBUG } 23:07:05 DEBUG } 23:07:05 DEBUG } 23:07:05 DEBUG ] 23:07:05 DEBUG ] 23:07:05 DEBUG ] 23:07:05 DEBUG } 23:07:05 DEBUG ] 23:07:05 DEBUG } 23:07:05 DEBUG } 23:07:05 DEBUG } 23:07:05 DEBUG } 23:07:05 DEBUG } 23:07:05 DEBUG } 23:07:05 DEBUG } 23:07:05 DEBUG ] 23:07:05 DEBUG } 23:07:05 DEBUG } 23:07:05 DEBUG ] 23:07:05 DEBUG } 23:07:05 DEBUG } 23:07:05 DEBUG } 23:07:05 DEBUG } 23:07:05 DEBUG } 23:07:05 DEBUG } 23:07:05 DEBUG } 23:07:05 DEBUG } 23:07:05 DEBUG } 23:07:05 DEBUG } 23:07:05 DEBUG } 23:07:05 DEBUG ] 23:07:05 DEBUG } 23:07:05 DEBUG } 23:07:05 DEBUG } 23:07:05 DEBUG } 23:07:05 DEBUG } 23:07:05 DEBUG } 23:07:05 DEBUG } 23:07:05 DEBUG } 23:07:05 DEBUG } 23:07:05 DEBUG } 23:07:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305279" 23:07:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713305279" 23:07:59 INFO [http_cmd]: http status code OK 23:07:59 ERROR [http_cmd]: ERROR 23:07:59 DEBUG --- status code --- 23:07:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:07:59 ERROR --- status code --- 23:07:59 DEBUG --- http response --- 23:07:59 ERROR http status code is 400 (expected [200, 401, 403, 404]) 23:07:59 ERROR --- http response --- 23:07:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 23:07:59 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 23:07:59 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 23:08:02 INFO 23:08:02 INFO [loop_until]: kubectl --namespace=xlou top pods 23:08:02 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 23:08:02 INFO [loop_until]: OK (rc = 0) 23:08:02 DEBUG --- stdout --- 23:08:02 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 9Mi am-555f4db595-4lbss 9424m 8682Mi am-555f4db595-fz8sn 9682m 8563Mi am-555f4db595-kzxsp 9317m 8972Mi ds-cts-0 19112m 13032Mi ds-cts-1 15066m 12913Mi ds-cts-2 15248m 13100Mi ds-idrepo-0 3961m 5033Mi ds-idrepo-1 3819m 2900Mi ds-idrepo-2 3763m 3222Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 4m 1207Mi idm-b85f9d978-rl76f 4m 1225Mi lodemon-bc756c647-cr7x6 2m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-7c9b667974-9mr2s 8017m 7554Mi 23:08:02 DEBUG --- stderr --- 23:08:02 DEBUG 23:08:05 INFO 23:08:05 INFO [loop_until]: kubectl --namespace=xlou top node 23:08:05 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 23:08:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305285" 23:08:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305285" 23:08:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713305285" 23:08:05 INFO [http_cmd]: http status code OK 23:08:05 DEBUG --- status code --- 23:08:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:08:05 DEBUG --- http response --- 23:08:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 23:08:05 DEBUG ----- output ----- 23:08:05 DEBUG { 23:08:05 DEBUG "status": "success", 23:08:05 DEBUG "data": { 23:08:05 DEBUG "resultType": "vector", 23:08:05 DEBUG "result": [ 23:08:05 DEBUG { 23:08:05 DEBUG "metric": { 23:08:05 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 23:08:05 DEBUG }, 23:08:05 DEBUG "value": [ 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG [...] 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG "pod": "ds-cts-2" 23:08:05 DEBUG }, 23:08:05 DEBUG "value": [ 23:08:05 DEBUG 1713305285, 23:08:05 DEBUG "52247.99330543933" 23:08:05 DEBUG ] 23:08:05 DEBUG } 23:08:05 DEBUG ] 23:08:05 DEBUG } 23:08:05 DEBUG } 23:08:05 INFO [http_cmd]: http status code OK 23:08:05 DEBUG --- status code --- 23:08:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:08:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305285" 23:08:05 DEBUG --- http response --- 23:08:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:08:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305285" 23:08:05 INFO [http_cmd]: http status code OK 23:08:05 DEBUG --- status code --- 23:08:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:08:05 DEBUG --- http response --- 23:08:05 DEBUG ----- output ----- 23:08:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713305285" 23:08:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:08:05 DEBUG ----- output ----- 23:08:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305285" 23:08:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305285" 23:08:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305285" 23:08:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305285" 23:08:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713305285" 23:08:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305285" 23:08:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713305285" 23:08:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305285" 23:08:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713305285" 23:08:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305285" 23:08:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305285" 23:08:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713305285" 23:08:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713305285" 23:08:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713305285" 23:08:05 DEBUG { 23:08:05 DEBUG { 23:08:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305285" 23:08:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305285" 23:08:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305285" 23:08:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305285" 23:08:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713305285" 23:08:05 INFO [http_cmd]: http status code OK 23:08:05 INFO [http_cmd]: http status code OK 23:08:05 INFO [http_cmd]: http status code OK 23:08:05 INFO [loop_until]: OK (rc = 0) 23:08:05 INFO [http_cmd]: http status code OK 23:08:05 INFO [http_cmd]: http status code OK 23:08:05 INFO [http_cmd]: http status code OK 23:08:05 INFO [http_cmd]: http status code OK 23:08:05 INFO [http_cmd]: http status code OK 23:08:05 DEBUG "status": "success", 23:08:05 INFO [http_cmd]: http status code OK 23:08:05 INFO [http_cmd]: http status code OK 23:08:05 DEBUG "status": "success", 23:08:05 INFO [http_cmd]: http status code OK 23:08:05 INFO [http_cmd]: http status code OK 23:08:05 INFO [http_cmd]: http status code OK 23:08:05 INFO [http_cmd]: http status code OK 23:08:05 DEBUG --- status code --- 23:08:05 DEBUG --- status code --- 23:08:05 DEBUG --- status code --- 23:08:05 DEBUG --- stdout --- 23:08:05 DEBUG --- status code --- 23:08:05 DEBUG --- status code --- 23:08:05 DEBUG --- status code --- 23:08:05 DEBUG --- status code --- 23:08:05 DEBUG --- status code --- 23:08:05 DEBUG "data": { 23:08:05 INFO [http_cmd]: http status code OK 23:08:05 INFO [http_cmd]: http status code OK 23:08:05 DEBUG --- status code --- 23:08:05 INFO [http_cmd]: http status code OK 23:08:05 INFO [http_cmd]: http status code OK 23:08:05 DEBUG --- status code --- 23:08:05 DEBUG "data": { 23:08:05 DEBUG --- status code --- 23:08:05 DEBUG --- status code --- 23:08:05 DEBUG --- status code --- 23:08:05 DEBUG --- status code --- 23:08:05 INFO [http_cmd]: http status code OK 23:08:05 INFO [http_cmd]: http status code OK 23:08:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:08:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:08:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:08:05 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 5084Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 15047m 50% 15635Mi 14% gke-xlou-cdm-default-pool-1b0769c8-nhlf 9357m 31% 10810Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 81m 0% 2639Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 3962m 13% 6586Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 20315m 68% 17446Mi 15% gke-xlou-cdm-default-pool-1b0769c8-xnmp 62m 0% 3134Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 9225m 30% 10482Mi 9% gke-xlou-cdm-ds-0f601751-hmm7 4282m 14% 5647Mi 5% gke-xlou-cdm-ds-0f601751-j488 15039m 50% 15065Mi 13% gke-xlou-cdm-ds-0f601751-mc5d 8926m 29% 4188Mi 3% gke-xlou-cdm-ds-0f601751-nj6q 9730m 32% 10861Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 4296m 14% 7359Mi 6% gke-xlou-cdm-frontend-a8771548-i82n 7959m 50% 12238Mi 20% 23:08:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:08:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:08:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:08:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:08:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:08:05 DEBUG "resultType": "vector", 23:08:05 DEBUG --- status code --- 23:08:05 DEBUG --- status code --- 23:08:05 INFO [http_cmd]: http status code OK 23:08:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:08:05 DEBUG --- status code --- 23:08:05 DEBUG --- status code --- 23:08:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:08:05 DEBUG "resultType": "vector", 23:08:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:08:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:08:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:08:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:08:05 DEBUG --- status code --- 23:08:05 DEBUG --- status code --- 23:08:05 DEBUG --- http response --- 23:08:05 DEBUG --- http response --- 23:08:05 DEBUG --- http response --- 23:08:05 DEBUG --- stderr --- 23:08:05 DEBUG --- http response --- 23:08:05 DEBUG --- http response --- 23:08:05 DEBUG --- http response --- 23:08:05 DEBUG --- http response --- 23:08:05 DEBUG --- http response --- 23:08:05 DEBUG "result": [ 23:08:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:08:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:08:05 DEBUG --- status code --- 23:08:05 DEBUG --- http response --- 23:08:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:08:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:08:05 DEBUG --- http response --- 23:08:05 DEBUG "result": [ 23:08:05 DEBUG --- http response --- 23:08:05 DEBUG --- http response --- 23:08:05 DEBUG --- http response --- 23:08:05 DEBUG --- http response --- 23:08:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:08:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:08:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 23:08:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:08:05 DEBUG 23:08:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 23:08:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:08:05 INFO [http_cmd]: http status code OK 23:08:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 23:08:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 23:08:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:08:05 DEBUG { 23:08:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 23:08:05 DEBUG --- http response --- 23:08:05 DEBUG --- http response --- 23:08:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:08:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 23:08:05 DEBUG --- http response --- 23:08:05 DEBUG --- http response --- 23:08:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:08:05 DEBUG { 23:08:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 23:08:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:08:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:08:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:08:05 DEBUG --- http response --- 23:08:05 DEBUG --- http response --- 23:08:05 DEBUG ----- output ----- 23:08:05 DEBUG ----- output ----- 23:08:05 DEBUG ----- output ----- 23:08:05 DEBUG ----- output ----- 23:08:05 DEBUG --- status code --- 23:08:05 DEBUG ----- output ----- 23:08:05 DEBUG ----- output ----- 23:08:05 DEBUG ----- output ----- 23:08:05 DEBUG "metric": { 23:08:05 DEBUG ----- output ----- 23:08:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:08:05 DEBUG --- http response --- 23:08:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:08:05 DEBUG ----- output ----- 23:08:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:08:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:08:05 DEBUG ----- output ----- 23:08:05 DEBUG "metric": { 23:08:05 DEBUG ----- output ----- 23:08:05 DEBUG ----- output ----- 23:08:05 DEBUG ----- output ----- 23:08:05 DEBUG ----- output ----- 23:08:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:08:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:08:05 DEBUG { 23:08:05 DEBUG { 23:08:05 DEBUG { 23:08:05 DEBUG { 23:08:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:08:05 DEBUG { 23:08:05 DEBUG { 23:08:05 DEBUG { 23:08:05 DEBUG "pod": "am-555f4db595-fz8sn" 23:08:05 DEBUG { 23:08:05 DEBUG ----- output ----- 23:08:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:08:05 DEBUG ----- output ----- 23:08:05 DEBUG { 23:08:05 DEBUG ----- output ----- 23:08:05 DEBUG ----- output ----- 23:08:05 DEBUG { 23:08:05 DEBUG "pod": "am-555f4db595-fz8sn" 23:08:05 DEBUG { 23:08:05 DEBUG { 23:08:05 DEBUG { 23:08:05 DEBUG { 23:08:05 DEBUG ----- output ----- 23:08:05 DEBUG ----- output ----- 23:08:05 DEBUG "status": "success", 23:08:05 DEBUG "status": "success", 23:08:05 DEBUG "status": "success", 23:08:05 DEBUG "data": { 23:08:05 DEBUG --- http response --- 23:08:05 DEBUG "status": "success", 23:08:05 DEBUG "status": "success", 23:08:05 DEBUG "status": "success", 23:08:05 DEBUG }, 23:08:05 DEBUG "status": "success", 23:08:05 DEBUG { 23:08:05 DEBUG ----- output ----- 23:08:05 DEBUG { 23:08:05 DEBUG "status": "success", 23:08:05 DEBUG { 23:08:05 DEBUG { 23:08:05 DEBUG "status": "success", 23:08:05 DEBUG }, 23:08:05 DEBUG "status": "success", 23:08:05 DEBUG "status": "success", 23:08:05 DEBUG "status": "success", 23:08:05 DEBUG "status": "success", 23:08:05 DEBUG { 23:08:05 DEBUG { 23:08:05 DEBUG "data": { 23:08:05 DEBUG "data": { 23:08:05 DEBUG "status": "success", 23:08:05 DEBUG "resultType": "vector", 23:08:05 DEBUG "data": { 23:08:05 DEBUG "data": { 23:08:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:08:05 DEBUG "data": { 23:08:05 DEBUG "value": [ 23:08:05 DEBUG "data": { 23:08:05 DEBUG "status": "success", 23:08:05 DEBUG { 23:08:05 DEBUG "status": "success", 23:08:05 DEBUG "data": { 23:08:05 DEBUG "status": "success", 23:08:05 DEBUG "status": "success", 23:08:05 DEBUG "data": { 23:08:05 DEBUG "value": [ 23:08:05 DEBUG "data": { 23:08:05 DEBUG "data": { 23:08:05 DEBUG "data": { 23:08:05 DEBUG "data": { 23:08:05 DEBUG "status": "success", 23:08:05 DEBUG "status": "success", 23:08:05 DEBUG "resultType": "vector", 23:08:05 DEBUG "resultType": "vector", 23:08:05 DEBUG "data": { 23:08:05 DEBUG "result": [ 23:08:05 DEBUG "resultType": "vector", 23:08:05 DEBUG "resultType": "vector", 23:08:05 DEBUG ----- output ----- 23:08:05 DEBUG "resultType": "vector", 23:08:05 DEBUG 23:08:05 DEBUG "resultType": "vector", 23:08:05 DEBUG "data": { 23:08:05 DEBUG "status": "success", 23:08:05 DEBUG "data": { 23:08:05 DEBUG "resultType": "vector", 23:08:05 DEBUG "data": { 23:08:05 DEBUG "data": { 23:08:05 DEBUG "resultType": "vector", 23:08:05 DEBUG 23:08:05 DEBUG "resultType": "vector", 23:08:05 DEBUG "resultType": "vector", 23:08:05 DEBUG "resultType": "vector", 23:08:05 DEBUG "resultType": "vector", 23:08:05 DEBUG "data": { 23:08:05 DEBUG "data": { 23:08:05 DEBUG "result": [ 23:08:05 DEBUG "result": [ 23:08:05 DEBUG "resultType": "vector", 23:08:05 DEBUG { 23:08:05 DEBUG "result": [ 23:08:05 DEBUG "result": [ 23:08:05 DEBUG { 23:08:05 DEBUG "result": [ 23:08:05 DEBUG 23:08:05 DEBUG "result": [ 23:08:05 DEBUG "resultType": "vector", 23:08:05 DEBUG "data": { 23:08:05 DEBUG "resultType": "vector", 23:08:05 DEBUG "result": [ 23:08:05 DEBUG "resultType": "vector", 23:08:05 DEBUG "resultType": "vector", 23:08:05 DEBUG "result": [ 23:08:05 DEBUG 23:08:05 DEBUG "result": [ 23:08:05 DEBUG { 23:08:05 DEBUG "result": [ 23:08:05 DEBUG "result": [ 23:08:05 DEBUG "resultType": "vector", 23:08:05 DEBUG "resultType": "vector", 23:08:05 DEBUG { 23:08:05 DEBUG { 23:08:05 DEBUG "result": [ 23:08:05 DEBUG "metric": { 23:08:05 DEBUG { 23:08:05 DEBUG { 23:08:05 DEBUG "status": "success", 23:08:05 DEBUG { 23:08:05 DEBUG 23:08:05 DEBUG { 23:08:05 DEBUG "result": [ 23:08:05 DEBUG "resultType": "vector", 23:08:05 DEBUG "result": [ 23:08:05 DEBUG { 23:08:05 DEBUG "result": [ 23:08:05 DEBUG "result": [ 23:08:05 DEBUG { 23:08:05 DEBUG 23:08:05 DEBUG "metric": { 23:08:05 DEBUG "result": [ 23:08:05 DEBUG { 23:08:05 DEBUG { 23:08:05 DEBUG "result": [ 23:08:05 DEBUG "result": [ 23:08:05 DEBUG "metric": { 23:08:05 DEBUG "metric": { 23:08:05 DEBUG { 23:08:05 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 23:08:05 DEBUG "metric": { 23:08:05 DEBUG "metric": { 23:08:05 DEBUG "data": { 23:08:05 DEBUG "metric": { 23:08:05 DEBUG [...] 23:08:05 DEBUG "metric": { 23:08:05 DEBUG { 23:08:05 DEBUG "result": [ 23:08:05 DEBUG { 23:08:05 DEBUG "metric": { 23:08:05 DEBUG { 23:08:05 DEBUG { 23:08:05 DEBUG "metric": { 23:08:05 DEBUG [...] 23:08:05 DEBUG "pod": "ds-idrepo-2" 23:08:05 DEBUG { 23:08:05 DEBUG "metric": { 23:08:05 DEBUG "metric": { 23:08:05 DEBUG { 23:08:05 DEBUG { 23:08:05 DEBUG "pod": "ds-idrepo-2" 23:08:05 DEBUG "pod": "am-555f4db595-fz8sn" 23:08:05 DEBUG "metric": { 23:08:05 DEBUG }, 23:08:05 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 23:08:05 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 23:08:05 DEBUG "resultType": "vector", 23:08:05 DEBUG "instance": "10.142.0.107:9100" 23:08:05 DEBUG 23:08:05 DEBUG "pod": "ds-idrepo-2" 23:08:05 DEBUG "metric": { 23:08:05 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 23:08:05 DEBUG "metric": { 23:08:05 DEBUG "pod": "ds-idrepo-2" 23:08:05 DEBUG "metric": { 23:08:05 DEBUG "metric": { 23:08:05 DEBUG "instance": "10.142.0.107:9100" 23:08:05 DEBUG 23:08:05 DEBUG }, 23:08:05 DEBUG "metric": { 23:08:05 DEBUG "pod": "am-555f4db595-fz8sn" 23:08:05 DEBUG "pod": "am-555f4db595-fz8sn" 23:08:05 DEBUG "metric": { 23:08:05 DEBUG "metric": { 23:08:05 DEBUG }, 23:08:05 DEBUG }, 23:08:05 DEBUG "pod": "ds-idrepo-2" 23:08:05 DEBUG "value": [ 23:08:05 DEBUG }, 23:08:05 DEBUG }, 23:08:05 DEBUG "result": [ 23:08:05 DEBUG }, 23:08:05 DEBUG 23:08:05 DEBUG }, 23:08:05 DEBUG }, 23:08:05 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 23:08:05 DEBUG }, 23:08:05 DEBUG { 23:08:05 DEBUG "instance": "10.142.0.107:9100" 23:08:05 DEBUG "pod": "am-555f4db595-fz8sn" 23:08:05 DEBUG }, 23:08:05 DEBUG 23:08:05 DEBUG "value": [ 23:08:05 DEBUG 23:08:05 DEBUG }, 23:08:05 DEBUG }, 23:08:05 DEBUG "pod": "am-555f4db595-fz8sn" 23:08:05 DEBUG "pod": "ds-idrepo-2" 23:08:05 DEBUG "value": [ 23:08:05 DEBUG "value": [ 23:08:05 DEBUG }, 23:08:05 DEBUG 23:08:05 DEBUG "value": [ 23:08:05 DEBUG "value": [ 23:08:05 DEBUG { 23:08:05 DEBUG "value": [ 23:08:05 DEBUG 23:08:05 DEBUG "value": [ 23:08:05 DEBUG "value": [ 23:08:05 DEBUG }, 23:08:05 DEBUG "value": [ 23:08:05 DEBUG "metric": { 23:08:05 DEBUG "pod": "ds-idrepo-2" 23:08:05 DEBUG }, 23:08:05 DEBUG "value": [ 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG "instance": "10.142.0.107:9100" 23:08:05 DEBUG "value": [ 23:08:05 DEBUG "value": [ 23:08:05 DEBUG }, 23:08:05 DEBUG }, 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG "value": [ 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG "metric": { 23:08:05 DEBUG 23:08:05 DEBUG "pod": "am-555f4db595-kzxsp" 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG "value": [ 23:08:05 DEBUG 23:08:05 DEBUG }, 23:08:05 DEBUG }, 23:08:05 DEBUG "value": [ 23:08:05 DEBUG 23:08:05 DEBUG "pod": "am-555f4db595-kzxsp" 23:08:05 DEBUG 23:08:05 DEBUG }, 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG "value": [ 23:08:05 DEBUG "value": [ 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG "instance": "10.142.0.107:9100" 23:08:05 DEBUG 23:08:05 DEBUG }, 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG "value": [ 23:08:05 DEBUG "value": [ 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG }, 23:08:05 DEBUG [...] 23:08:05 DEBUG "value": [ 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG [...] 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG }, 23:08:05 DEBUG 23:08:05 DEBUG "value": [ 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG "value": [ 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG [...] 23:08:05 DEBUG [...] 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG [...] 23:08:05 DEBUG [...] 23:08:05 DEBUG "value": [ 23:08:05 DEBUG [...] 23:08:05 DEBUG 1713305285, 23:08:05 DEBUG [...] 23:08:05 DEBUG [...] 23:08:05 DEBUG 23:08:05 DEBUG [...] 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG [...] 23:08:05 DEBUG 1713305285, 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG [...] 23:08:05 DEBUG [...] 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG [...] 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG "0.027" 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG [...] 23:08:05 DEBUG 23:08:05 DEBUG "2005.7" 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG [...] 23:08:05 DEBUG [...] 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG ] 23:08:05 DEBUG 23:08:05 DEBUG [...] 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG [...] 23:08:05 DEBUG [...] 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG ] 23:08:05 DEBUG "pod": "ds-idrepo-0" 23:08:05 DEBUG [...] 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG "pod": "ds-cts-2" 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG [...] 23:08:05 DEBUG 23:08:05 DEBUG } 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG } 23:08:05 DEBUG }, 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG "pod": "ds-cts-2" 23:08:05 DEBUG "pod": "am-555f4db595-kzxsp" 23:08:05 DEBUG 23:08:05 DEBUG }, 23:08:05 DEBUG "pod": "ds-cts-2" 23:08:05 DEBUG "pod": "ds-cts-2" 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG ] 23:08:05 DEBUG 23:08:05 DEBUG "pod": "ds-idrepo-0" 23:08:05 DEBUG 23:08:05 DEBUG "pod": "ds-idrepo-0" 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG "instance": "10.142.0.43:9100" 23:08:05 DEBUG ] 23:08:05 DEBUG "value": [ 23:08:05 DEBUG 23:08:05 DEBUG "pod": "am-555f4db595-kzxsp" 23:08:05 DEBUG "pod": "am-555f4db595-kzxsp" 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG }, 23:08:05 DEBUG }, 23:08:05 DEBUG "pod": "ds-idrepo-0" 23:08:05 DEBUG "value": [ 23:08:05 DEBUG }, 23:08:05 DEBUG }, 23:08:05 DEBUG 23:08:05 DEBUG "instance": "10.142.0.43:9100" 23:08:05 DEBUG } 23:08:05 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 23:08:05 DEBUG }, 23:08:05 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 23:08:05 DEBUG }, 23:08:05 DEBUG 23:08:05 DEBUG "pod": "ds-idrepo-0" 23:08:05 DEBUG }, 23:08:05 DEBUG } 23:08:05 DEBUG 1713305285, 23:08:05 DEBUG 23:08:05 DEBUG 23:08:05 DEBUG }, 23:08:05 DEBUG }, 23:08:05 DEBUG "pod": "am-555f4db595-kzxsp" 23:08:05 DEBUG "pod": "ds-idrepo-0" 23:08:05 DEBUG "value": [ 23:08:05 DEBUG "value": [ 23:08:05 DEBUG }, 23:08:05 DEBUG 1713305285, 23:08:05 DEBUG "value": [ 23:08:05 DEBUG "value": [ 23:08:05 DEBUG 23:08:05 DEBUG }, 23:08:05 DEBUG } 23:08:05 DEBUG }, 23:08:05 DEBUG "value": [ 23:08:05 DEBUG }, 23:08:05 DEBUG "value": [ 23:08:05 DEBUG "instance": "10.142.0.43:9100" 23:08:05 DEBUG }, 23:08:05 DEBUG "value": [ 23:08:05 DEBUG } 23:08:05 DEBUG "0" 23:08:05 DEBUG "pod": "am-555f4db595-kzxsp" 23:08:05 DEBUG "instance": "10.142.0.43:9100" 23:08:05 DEBUG "value": [ 23:08:05 DEBUG "value": [ 23:08:05 DEBUG }, 23:08:05 DEBUG }, 23:08:05 DEBUG 1713305285, 23:08:05 DEBUG 1713305285, 23:08:05 DEBUG "value": [ 23:08:05 DEBUG "48559523.454393305" 23:08:05 DEBUG 1713305285, 23:08:05 DEBUG 1713305285, 23:08:05 DEBUG "instance": "10.142.0.43:9100" 23:08:05 DEBUG "value": [ 23:08:05 DEBUG "value": [ 23:08:05 DEBUG 1713305285, 23:08:05 DEBUG "value": [ 23:08:05 DEBUG 1713305285, 23:08:05 DEBUG }, 23:08:05 DEBUG "value": [ 23:08:05 DEBUG 1713305285, 23:08:05 DEBUG ] 23:08:05 DEBUG }, 23:08:05 DEBUG }, 23:08:05 DEBUG 1713305285, 23:08:05 DEBUG 1713305285, 23:08:05 DEBUG "value": [ 23:08:05 DEBUG "value": [ 23:08:05 DEBUG "0" 23:08:05 DEBUG "1389.8" 23:08:05 DEBUG 1713305285, 23:08:05 DEBUG ] 23:08:05 DEBUG "10.360712959727962" 23:08:05 DEBUG "41617803.03263599" 23:08:05 DEBUG }, 23:08:05 DEBUG 1713305285, 23:08:05 DEBUG 1713305285, 23:08:05 DEBUG "0" 23:08:05 DEBUG 1713305285, 23:08:05 DEBUG "0" 23:08:05 DEBUG "value": [ 23:08:05 DEBUG 1713305285, 23:08:05 DEBUG "33256777.133333333" 23:08:05 DEBUG } 23:08:05 DEBUG "value": [ 23:08:05 DEBUG "value": [ 23:08:05 DEBUG "0" 23:08:05 DEBUG "3572115.5" 23:08:05 DEBUG 1713305285, 23:08:05 DEBUG 1713305285, 23:08:05 DEBUG ] 23:08:05 DEBUG ] 23:08:05 DEBUG "0" 23:08:05 DEBUG } 23:08:05 DEBUG ] 23:08:05 DEBUG ] 23:08:05 DEBUG "value": [ 23:08:05 DEBUG "19986234368" 23:08:05 DEBUG "8.016133182655443" 23:08:05 DEBUG ] 23:08:05 DEBUG "7727681536" 23:08:05 DEBUG ] 23:08:05 DEBUG 1713305285, 23:08:05 DEBUG "0" 23:08:05 DEBUG ] 23:08:05 DEBUG } 23:08:05 DEBUG 1713305285, 23:08:05 DEBUG 1713305285, 23:08:05 DEBUG ] 23:08:05 DEBUG ] 23:08:05 DEBUG "1389.1666666666667" 23:08:05 DEBUG "0" 23:08:05 DEBUG } 23:08:05 DEBUG } 23:08:05 DEBUG ] 23:08:05 DEBUG ] 23:08:05 DEBUG } 23:08:05 DEBUG } 23:08:05 DEBUG 1713305285, 23:08:05 DEBUG ] 23:08:05 DEBUG ] 23:08:05 DEBUG } 23:08:05 DEBUG ] 23:08:05 DEBUG } 23:08:05 DEBUG "33262675.6" 23:08:05 DEBUG ] 23:08:05 DEBUG ] 23:08:05 DEBUG ] 23:08:05 DEBUG "1389.4" 23:08:05 DEBUG "0" 23:08:05 DEBUG } 23:08:05 DEBUG } 23:08:05 DEBUG ] 23:08:05 DEBUG ] 23:08:05 DEBUG ] 23:08:05 DEBUG ] 23:08:05 DEBUG } 23:08:05 DEBUG } 23:08:05 DEBUG ] 23:08:05 DEBUG ] 23:08:05 DEBUG "0.0005000000000004927" 23:08:05 DEBUG } 23:08:05 DEBUG } 23:08:05 DEBUG ] 23:08:05 DEBUG } 23:08:05 DEBUG ] 23:08:05 DEBUG ] 23:08:05 DEBUG } 23:08:05 DEBUG } 23:08:05 DEBUG } 23:08:05 DEBUG ] 23:08:05 DEBUG ] 23:08:05 DEBUG ] 23:08:05 DEBUG ] 23:08:05 DEBUG } 23:08:05 DEBUG } 23:08:05 DEBUG } 23:08:05 DEBUG } 23:08:05 DEBUG ] 23:08:05 DEBUG } 23:08:05 DEBUG } 23:08:05 DEBUG } 23:08:05 DEBUG ] 23:08:05 DEBUG ] 23:08:05 DEBUG ] 23:08:05 DEBUG } 23:08:05 DEBUG ] 23:08:05 DEBUG } 23:08:05 DEBUG } 23:08:05 DEBUG ] 23:08:05 DEBUG } 23:08:05 DEBUG } 23:08:05 DEBUG } 23:08:05 DEBUG } 23:08:05 DEBUG } 23:08:05 DEBUG } 23:08:05 DEBUG ] 23:08:05 DEBUG ] 23:08:05 DEBUG } 23:08:05 DEBUG } 23:08:05 DEBUG } 23:08:05 DEBUG } 23:08:05 DEBUG } 23:08:05 DEBUG } 23:08:05 DEBUG } 23:08:05 DEBUG } 23:08:05 DEBUG } 23:08:05 DEBUG } 23:08:05 DEBUG } 23:08:05 DEBUG ] 23:08:05 DEBUG } 23:08:05 DEBUG ] 23:08:05 DEBUG ] 23:08:05 DEBUG } 23:08:05 DEBUG } 23:08:05 DEBUG } 23:08:05 DEBUG } 23:08:05 DEBUG } 23:08:05 DEBUG ] 23:08:05 DEBUG } 23:08:05 DEBUG } 23:08:05 DEBUG } 23:08:05 DEBUG } 23:08:05 DEBUG } 23:08:05 DEBUG } 23:08:05 DEBUG } 23:08:05 DEBUG } 23:08:05 DEBUG } 23:08:05 DEBUG } 23:08:05 DEBUG } 23:08:05 DEBUG } 23:08:05 DEBUG } 23:08:05 DEBUG } 23:08:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305339" 23:08:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713305339" 23:08:59 ERROR [http_cmd]: ERROR 23:08:59 INFO [http_cmd]: http status code OK 23:08:59 ERROR --- status code --- 23:08:59 DEBUG --- status code --- 23:08:59 ERROR http status code is 400 (expected [200, 401, 403, 404]) 23:08:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:08:59 ERROR --- http response --- 23:08:59 DEBUG --- http response --- 23:08:59 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 23:08:59 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 23:08:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 23:09:02 INFO 23:09:02 INFO [loop_until]: kubectl --namespace=xlou top pods 23:09:02 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 23:09:02 INFO [loop_until]: OK (rc = 0) 23:09:02 DEBUG --- stdout --- 23:09:02 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 9Mi am-555f4db595-4lbss 8207m 8681Mi am-555f4db595-fz8sn 6938m 8564Mi am-555f4db595-kzxsp 5761m 8973Mi ds-cts-0 15112m 12829Mi ds-cts-1 11747m 12792Mi ds-cts-2 12214m 13386Mi ds-idrepo-0 3423m 5033Mi ds-idrepo-1 3081m 2900Mi ds-idrepo-2 3209m 3222Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 4m 1208Mi idm-b85f9d978-rl76f 4m 1225Mi lodemon-bc756c647-cr7x6 1m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-7c9b667974-9mr2s 4889m 18Mi 23:09:02 DEBUG --- stderr --- 23:09:02 DEBUG 23:09:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305345" 23:09:05 INFO [http_cmd]: http status code OK 23:09:05 DEBUG --- status code --- 23:09:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:09:05 DEBUG --- http response --- 23:09:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 23:09:05 DEBUG ----- output ----- 23:09:05 DEBUG { 23:09:05 DEBUG "status": "success", 23:09:05 DEBUG "data": { 23:09:05 DEBUG "resultType": "vector", 23:09:05 DEBUG "result": [ 23:09:05 DEBUG { 23:09:05 DEBUG "metric": { 23:09:05 DEBUG "pod": "ds-cts-1" 23:09:05 DEBUG }, 23:09:05 DEBUG "value": [ 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG [...] 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 23:09:05 DEBUG }, 23:09:05 DEBUG "value": [ 23:09:05 DEBUG 1713305345, 23:09:05 DEBUG "302456.50512154103" 23:09:05 DEBUG ] 23:09:05 DEBUG } 23:09:05 DEBUG ] 23:09:05 DEBUG } 23:09:05 DEBUG } 23:09:05 INFO 23:09:05 INFO [loop_until]: kubectl --namespace=xlou top node 23:09:05 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 23:09:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305345" 23:09:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305345" 23:09:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305345" 23:09:05 INFO [http_cmd]: http status code OK 23:09:05 DEBUG --- status code --- 23:09:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:09:05 DEBUG --- http response --- 23:09:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:09:05 DEBUG ----- output ----- 23:09:05 DEBUG { 23:09:05 DEBUG "status": "success", 23:09:05 DEBUG "data": { 23:09:05 DEBUG "resultType": "vector", 23:09:05 DEBUG "result": [ 23:09:05 DEBUG { 23:09:05 DEBUG "metric": { 23:09:05 DEBUG "pod": "am-555f4db595-fz8sn" 23:09:05 DEBUG }, 23:09:05 DEBUG "value": [ 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 INFO [loop_until]: OK (rc = 0) 23:09:05 DEBUG 23:09:05 DEBUG --- stdout --- 23:09:05 DEBUG [...] 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG "pod": "am-555f4db595-kzxsp" 23:09:05 DEBUG }, 23:09:05 DEBUG "value": [ 23:09:05 DEBUG 1713305345, 23:09:05 DEBUG "1992.2333333333333" 23:09:05 DEBUG ] 23:09:05 DEBUG } 23:09:05 DEBUG ] 23:09:05 DEBUG } 23:09:05 DEBUG } 23:09:05 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 63m 0% 5095Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 13453m 45% 15296Mi 13% gke-xlou-cdm-default-pool-1b0769c8-nhlf 7450m 24% 10812Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 84m 0% 2643Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 2931m 9% 6580Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 14931m 50% 17349Mi 15% gke-xlou-cdm-default-pool-1b0769c8-xnmp 60m 0% 3132Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 6613m 22% 10515Mi 9% gke-xlou-cdm-ds-0f601751-hmm7 3307m 11% 5657Mi 5% gke-xlou-cdm-ds-0f601751-j488 11861m 39% 15335Mi 13% gke-xlou-cdm-ds-0f601751-mc5d 5217m 17% 4111Mi 3% gke-xlou-cdm-ds-0f601751-nj6q 5702m 19% 10861Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 2996m 10% 7360Mi 6% gke-xlou-cdm-frontend-a8771548-i82n 5099m 32% 5390Mi 9% 23:09:05 DEBUG --- stderr --- 23:09:05 DEBUG 23:09:05 INFO [http_cmd]: http status code OK 23:09:05 DEBUG --- status code --- 23:09:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:09:05 DEBUG --- http response --- 23:09:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 23:09:05 DEBUG ----- output ----- 23:09:05 DEBUG { 23:09:05 DEBUG "status": "success", 23:09:05 DEBUG "data": { 23:09:05 DEBUG "resultType": "vector", 23:09:05 DEBUG "result": [ 23:09:05 DEBUG { 23:09:05 DEBUG "metric": { 23:09:05 DEBUG "pod": "ds-cts-1" 23:09:05 DEBUG }, 23:09:05 DEBUG "value": [ 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG [...] 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 23:09:05 DEBUG }, 23:09:05 INFO [http_cmd]: http status code OK 23:09:05 DEBUG "value": [ 23:09:05 DEBUG --- status code --- 23:09:05 DEBUG 1713305345, 23:09:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:09:05 DEBUG "0" 23:09:05 DEBUG --- http response --- 23:09:05 DEBUG ] 23:09:05 DEBUG } 23:09:05 DEBUG ] 23:09:05 DEBUG } 23:09:05 DEBUG } 23:09:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 23:09:05 DEBUG ----- output ----- 23:09:05 DEBUG { 23:09:05 DEBUG "status": "success", 23:09:05 DEBUG "data": { 23:09:05 DEBUG "resultType": "vector", 23:09:05 DEBUG "result": [ 23:09:05 DEBUG { 23:09:05 DEBUG "metric": { 23:09:05 DEBUG "pod": "ds-cts-1" 23:09:05 DEBUG }, 23:09:05 DEBUG "value": [ 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG [...] 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 23:09:05 DEBUG }, 23:09:05 DEBUG "value": [ 23:09:05 DEBUG 1713305345, 23:09:05 DEBUG "0.000034979360953983606" 23:09:05 DEBUG ] 23:09:05 DEBUG } 23:09:05 DEBUG ] 23:09:05 DEBUG } 23:09:05 DEBUG } 23:09:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713305345" 23:09:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713305345" 23:09:05 INFO [http_cmd]: http status code OK 23:09:05 DEBUG --- status code --- 23:09:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:09:05 DEBUG --- http response --- 23:09:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:09:05 DEBUG ----- output ----- 23:09:05 DEBUG { 23:09:05 DEBUG "status": "success", 23:09:05 DEBUG "data": { 23:09:05 DEBUG "resultType": "vector", 23:09:05 DEBUG "result": [ 23:09:05 DEBUG { 23:09:05 DEBUG "metric": { 23:09:05 DEBUG "instance": "10.142.0.107:9100" 23:09:05 DEBUG }, 23:09:05 DEBUG "value": [ 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG [...] 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG "instance": "10.142.0.43:9100" 23:09:05 DEBUG }, 23:09:05 DEBUG "value": [ 23:09:05 DEBUG 1713305345, 23:09:05 DEBUG "0" 23:09:05 DEBUG ] 23:09:05 DEBUG } 23:09:05 DEBUG ] 23:09:05 DEBUG } 23:09:05 DEBUG } 23:09:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713305345" 23:09:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713305345" 23:09:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305345" 23:09:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305345" 23:09:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305345" 23:09:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713305345" 23:09:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305345" 23:09:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713305345" 23:09:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305345" 23:09:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305345" 23:09:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305345" 23:09:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305345" 23:09:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305345" 23:09:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713305345" 23:09:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713305345" 23:09:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713305345" 23:09:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305345" 23:09:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305345" 23:09:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305345" 23:09:05 INFO [http_cmd]: http status code OK 23:09:05 INFO [http_cmd]: http status code OK 23:09:05 INFO [http_cmd]: http status code OK 23:09:05 INFO [http_cmd]: http status code OK 23:09:05 INFO [http_cmd]: http status code OK 23:09:05 INFO [http_cmd]: http status code OK 23:09:05 INFO [http_cmd]: http status code OK 23:09:05 INFO [http_cmd]: http status code OK 23:09:05 INFO [http_cmd]: http status code OK 23:09:05 INFO [http_cmd]: http status code OK 23:09:05 DEBUG --- status code --- 23:09:05 DEBUG --- status code --- 23:09:05 DEBUG --- status code --- 23:09:05 DEBUG --- status code --- 23:09:05 DEBUG --- status code --- 23:09:05 DEBUG --- status code --- 23:09:05 DEBUG --- status code --- 23:09:05 INFO [http_cmd]: http status code OK 23:09:05 DEBUG --- status code --- 23:09:05 INFO [http_cmd]: http status code OK 23:09:05 INFO [http_cmd]: http status code OK 23:09:05 INFO [http_cmd]: http status code OK 23:09:05 INFO [http_cmd]: http status code OK 23:09:05 DEBUG --- status code --- 23:09:05 DEBUG --- status code --- 23:09:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:09:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:09:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:09:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:09:05 INFO [http_cmd]: http status code OK 23:09:05 INFO [http_cmd]: http status code OK 23:09:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:09:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:09:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:09:05 DEBUG --- status code --- 23:09:05 INFO [http_cmd]: http status code OK 23:09:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:09:05 DEBUG --- status code --- 23:09:05 DEBUG --- status code --- 23:09:05 DEBUG --- status code --- 23:09:05 DEBUG --- status code --- 23:09:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:09:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:09:05 DEBUG --- http response --- 23:09:05 DEBUG --- http response --- 23:09:05 DEBUG --- http response --- 23:09:05 DEBUG --- http response --- 23:09:05 DEBUG --- status code --- 23:09:05 DEBUG --- status code --- 23:09:05 DEBUG --- http response --- 23:09:05 DEBUG --- http response --- 23:09:05 DEBUG --- http response --- 23:09:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:09:05 INFO [http_cmd]: http status code OK 23:09:05 DEBUG --- status code --- 23:09:05 INFO [http_cmd]: http status code OK 23:09:05 DEBUG --- http response --- 23:09:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:09:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:09:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:09:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:09:05 DEBUG --- http response --- 23:09:05 DEBUG --- http response --- 23:09:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:09:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:09:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:09:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 23:09:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:09:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:09:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:09:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:09:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:09:05 DEBUG --- http response --- 23:09:05 DEBUG --- status code --- 23:09:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:09:05 DEBUG --- status code --- 23:09:05 DEBUG --- http response --- 23:09:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:09:05 DEBUG --- http response --- 23:09:05 DEBUG --- http response --- 23:09:05 DEBUG --- http response --- 23:09:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 23:09:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 23:09:05 DEBUG ----- output ----- 23:09:05 DEBUG ----- output ----- 23:09:05 DEBUG ----- output ----- 23:09:05 DEBUG ----- output ----- 23:09:05 DEBUG --- http response --- 23:09:05 DEBUG --- http response --- 23:09:05 DEBUG ----- output ----- 23:09:05 DEBUG ----- output ----- 23:09:05 DEBUG ----- output ----- 23:09:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:09:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:09:05 DEBUG --- http response --- 23:09:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:09:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 23:09:05 DEBUG ----- output ----- 23:09:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:09:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:09:05 DEBUG ----- output ----- 23:09:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 23:09:05 DEBUG ----- output ----- 23:09:05 DEBUG { 23:09:05 DEBUG { 23:09:05 DEBUG { 23:09:05 DEBUG { 23:09:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:09:05 DEBUG { 23:09:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:09:05 DEBUG { 23:09:05 DEBUG { 23:09:05 DEBUG ----- output ----- 23:09:05 DEBUG --- http response --- 23:09:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:09:05 DEBUG --- http response --- 23:09:05 DEBUG ----- output ----- 23:09:05 DEBUG { 23:09:05 DEBUG ----- output ----- 23:09:05 DEBUG ----- output ----- 23:09:05 DEBUG { 23:09:05 DEBUG ----- output ----- 23:09:05 DEBUG { 23:09:05 DEBUG "status": "success", 23:09:05 DEBUG "status": "success", 23:09:05 DEBUG "status": "success", 23:09:05 DEBUG "status": "success", 23:09:05 DEBUG ----- output ----- 23:09:05 DEBUG "status": "success", 23:09:05 DEBUG ----- output ----- 23:09:05 DEBUG "status": "success", 23:09:05 DEBUG "status": "success", 23:09:05 DEBUG { 23:09:05 DEBUG ----- output ----- 23:09:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:09:05 DEBUG { 23:09:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:09:05 DEBUG "status": "success", 23:09:05 DEBUG { 23:09:05 DEBUG { 23:09:05 DEBUG "status": "success", 23:09:05 DEBUG { 23:09:05 DEBUG "status": "success", 23:09:05 DEBUG "data": { 23:09:05 DEBUG "data": { 23:09:05 DEBUG "data": { 23:09:05 DEBUG "data": { 23:09:05 DEBUG { 23:09:05 DEBUG "data": { 23:09:05 DEBUG { 23:09:05 DEBUG "data": { 23:09:05 DEBUG "data": { 23:09:05 DEBUG "status": "success", 23:09:05 DEBUG { 23:09:05 DEBUG ----- output ----- 23:09:05 DEBUG "status": "success", 23:09:05 DEBUG ----- output ----- 23:09:05 DEBUG "data": { 23:09:05 DEBUG "status": "success", 23:09:05 DEBUG "status": "success", 23:09:05 DEBUG "data": { 23:09:05 DEBUG "status": "success", 23:09:05 DEBUG "data": { 23:09:05 DEBUG "resultType": "vector", 23:09:05 DEBUG "resultType": "vector", 23:09:05 DEBUG "resultType": "vector", 23:09:05 DEBUG "resultType": "vector", 23:09:05 DEBUG "status": "success", 23:09:05 DEBUG "resultType": "vector", 23:09:05 DEBUG "status": "success", 23:09:05 DEBUG "resultType": "vector", 23:09:05 DEBUG "resultType": "vector", 23:09:05 DEBUG "data": { 23:09:05 DEBUG "status": "success", 23:09:05 DEBUG { 23:09:05 DEBUG "data": { 23:09:05 DEBUG { 23:09:05 DEBUG "resultType": "vector", 23:09:05 DEBUG "data": { 23:09:05 DEBUG "data": { 23:09:05 DEBUG "resultType": "vector", 23:09:05 DEBUG "data": { 23:09:05 DEBUG "resultType": "vector", 23:09:05 DEBUG "result": [ 23:09:05 DEBUG "result": [ 23:09:05 DEBUG "result": [ 23:09:05 DEBUG "result": [ 23:09:05 DEBUG "data": { 23:09:05 DEBUG "result": [ 23:09:05 DEBUG "data": { 23:09:05 DEBUG "result": [ 23:09:05 DEBUG "result": [ 23:09:05 DEBUG "resultType": "vector", 23:09:05 DEBUG "data": { 23:09:05 DEBUG "status": "success", 23:09:05 DEBUG "resultType": "vector", 23:09:05 DEBUG "status": "success", 23:09:05 DEBUG "result": [ 23:09:05 DEBUG "resultType": "vector", 23:09:05 DEBUG "resultType": "vector", 23:09:05 DEBUG "result": [ 23:09:05 DEBUG "resultType": "vector", 23:09:05 DEBUG "result": [ 23:09:05 DEBUG { 23:09:05 DEBUG { 23:09:05 DEBUG { 23:09:05 DEBUG { 23:09:05 DEBUG "resultType": "vector", 23:09:05 DEBUG { 23:09:05 DEBUG "resultType": "vector", 23:09:05 DEBUG { 23:09:05 DEBUG { 23:09:05 DEBUG "result": [ 23:09:05 DEBUG "resultType": "vector", 23:09:05 DEBUG "data": { 23:09:05 DEBUG "result": [ 23:09:05 DEBUG "data": { 23:09:05 DEBUG { 23:09:05 DEBUG "result": [ 23:09:05 DEBUG "result": [ 23:09:05 DEBUG { 23:09:05 DEBUG "result": [ 23:09:05 DEBUG { 23:09:05 DEBUG "metric": { 23:09:05 DEBUG "metric": { 23:09:05 DEBUG "metric": { 23:09:05 DEBUG "metric": { 23:09:05 DEBUG "result": [ 23:09:05 DEBUG "metric": { 23:09:05 DEBUG "result": [ 23:09:05 DEBUG "metric": { 23:09:05 DEBUG "metric": { 23:09:05 DEBUG { 23:09:05 DEBUG "result": [ 23:09:05 DEBUG "resultType": "vector", 23:09:05 DEBUG { 23:09:05 DEBUG "resultType": "vector", 23:09:05 DEBUG "metric": { 23:09:05 DEBUG { 23:09:05 DEBUG { 23:09:05 DEBUG "metric": { 23:09:05 DEBUG { 23:09:05 DEBUG "metric": { 23:09:05 DEBUG "pod": "am-555f4db595-fz8sn" 23:09:05 DEBUG "instance": "10.142.0.107:9100" 23:09:05 DEBUG "instance": "10.142.0.107:9100" 23:09:05 DEBUG "pod": "ds-idrepo-2" 23:09:05 DEBUG { 23:09:05 DEBUG "pod": "ds-idrepo-2" 23:09:05 DEBUG { 23:09:05 DEBUG "pod": "ds-idrepo-2" 23:09:05 DEBUG "pod": "am-555f4db595-fz8sn" 23:09:05 DEBUG "metric": { 23:09:05 DEBUG { 23:09:05 DEBUG "result": [ 23:09:05 DEBUG "metric": { 23:09:05 DEBUG "result": [ 23:09:05 DEBUG "instance": "10.142.0.107:9100" 23:09:05 DEBUG "metric": { 23:09:05 DEBUG "metric": { 23:09:05 DEBUG "pod": "ds-idrepo-2" 23:09:05 DEBUG "metric": { 23:09:05 DEBUG "pod": "ds-idrepo-2" 23:09:05 DEBUG }, 23:09:05 DEBUG }, 23:09:05 DEBUG }, 23:09:05 DEBUG }, 23:09:05 DEBUG "metric": { 23:09:05 DEBUG }, 23:09:05 DEBUG "metric": { 23:09:05 DEBUG }, 23:09:05 DEBUG }, 23:09:05 DEBUG "pod": "am-555f4db595-fz8sn" 23:09:05 DEBUG "metric": { 23:09:05 DEBUG { 23:09:05 DEBUG "pod": "ds-cts-1" 23:09:05 DEBUG { 23:09:05 DEBUG }, 23:09:05 DEBUG "pod": "am-555f4db595-fz8sn" 23:09:05 DEBUG "instance": "10.142.0.107:9100" 23:09:05 DEBUG }, 23:09:05 DEBUG "pod": "ds-idrepo-2" 23:09:05 DEBUG }, 23:09:05 DEBUG "value": [ 23:09:05 DEBUG "value": [ 23:09:05 DEBUG "value": [ 23:09:05 DEBUG "value": [ 23:09:05 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 23:09:05 DEBUG "value": [ 23:09:05 DEBUG "pod": "ds-idrepo-2" 23:09:05 DEBUG "value": [ 23:09:05 DEBUG "value": [ 23:09:05 DEBUG }, 23:09:05 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 23:09:05 DEBUG "metric": { 23:09:05 DEBUG }, 23:09:05 DEBUG "metric": { 23:09:05 DEBUG "value": [ 23:09:05 DEBUG }, 23:09:05 DEBUG }, 23:09:05 DEBUG "value": [ 23:09:05 DEBUG }, 23:09:05 DEBUG "value": [ 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG }, 23:09:05 DEBUG 23:09:05 DEBUG }, 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG "value": [ 23:09:05 DEBUG }, 23:09:05 DEBUG "pod": "am-555f4db595-fz8sn" 23:09:05 DEBUG "value": [ 23:09:05 DEBUG "pod": "am-555f4db595-fz8sn" 23:09:05 DEBUG 23:09:05 DEBUG "value": [ 23:09:05 DEBUG "value": [ 23:09:05 DEBUG 23:09:05 DEBUG "value": [ 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG "value": [ 23:09:05 DEBUG 23:09:05 DEBUG "value": [ 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG "value": [ 23:09:05 DEBUG }, 23:09:05 DEBUG 23:09:05 DEBUG }, 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG [...] 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG [...] 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG [...] 23:09:05 DEBUG 23:09:05 DEBUG "value": [ 23:09:05 DEBUG 23:09:05 DEBUG "value": [ 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG [...] 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG [...] 23:09:05 DEBUG 23:09:05 DEBUG [...] 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG [...] 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG [...] 23:09:05 DEBUG 23:09:05 DEBUG [...] 23:09:05 DEBUG [...] 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG [...] 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG [...] 23:09:05 DEBUG [...] 23:09:05 DEBUG 23:09:05 DEBUG [...] 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG [...] 23:09:05 DEBUG 23:09:05 DEBUG [...] 23:09:05 DEBUG 23:09:05 DEBUG "pod": "am-555f4db595-kzxsp" 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG "pod": "am-555f4db595-kzxsp" 23:09:05 DEBUG 23:09:05 DEBUG [...] 23:09:05 DEBUG "pod": "ds-idrepo-0" 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG [...] 23:09:05 DEBUG }, 23:09:05 DEBUG [...] 23:09:05 DEBUG 23:09:05 DEBUG [...] 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG }, 23:09:05 DEBUG "instance": "10.142.0.43:9100" 23:09:05 DEBUG 23:09:05 DEBUG }, 23:09:05 DEBUG "pod": "ds-idrepo-0" 23:09:05 DEBUG 23:09:05 DEBUG "pod": "ds-idrepo-0" 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG "value": [ 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG "instance": "10.142.0.43:9100" 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG "pod": "ds-cts-2" 23:09:05 DEBUG 23:09:05 DEBUG "pod": "ds-idrepo-0" 23:09:05 DEBUG "instance": "10.142.0.43:9100" 23:09:05 DEBUG "value": [ 23:09:05 DEBUG }, 23:09:05 DEBUG 23:09:05 DEBUG "value": [ 23:09:05 DEBUG }, 23:09:05 DEBUG "value": [ 23:09:05 DEBUG }, 23:09:05 DEBUG 23:09:05 DEBUG 23:09:05 DEBUG 1713305345, 23:09:05 DEBUG 23:09:05 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 23:09:05 DEBUG 23:09:05 DEBUG }, 23:09:05 DEBUG "pod": "am-555f4db595-kzxsp" 23:09:05 DEBUG "instance": "10.142.0.43:9100" 23:09:05 DEBUG }, 23:09:05 DEBUG "pod": "ds-idrepo-0" 23:09:05 DEBUG }, 23:09:05 DEBUG }, 23:09:05 DEBUG 1713305345, 23:09:05 DEBUG "value": [ 23:09:05 DEBUG 23:09:05 DEBUG 1713305345, 23:09:05 DEBUG 23:09:05 DEBUG 1713305345, 23:09:05 DEBUG "value": [ 23:09:05 DEBUG 1713305345, 23:09:05 DEBUG "pod": "am-555f4db595-kzxsp" 23:09:05 DEBUG "706.1333333333333" 23:09:05 DEBUG 23:09:05 DEBUG }, 23:09:05 DEBUG 23:09:05 DEBUG "value": [ 23:09:05 DEBUG }, 23:09:05 DEBUG }, 23:09:05 DEBUG "value": [ 23:09:05 DEBUG }, 23:09:05 DEBUG "value": [ 23:09:05 DEBUG "value": [ 23:09:05 DEBUG "0.027" 23:09:05 DEBUG 1713305345, 23:09:05 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 23:09:05 DEBUG "0" 23:09:05 DEBUG "pod": "ds-idrepo-0" 23:09:05 DEBUG "0" 23:09:05 DEBUG 23:09:05 DEBUG "0" 23:09:05 DEBUG }, 23:09:05 DEBUG "value": [ 23:09:05 DEBUG ] 23:09:05 DEBUG "value": [ 23:09:05 DEBUG "pod": "am-555f4db595-kzxsp" 23:09:05 DEBUG 1713305345, 23:09:05 DEBUG "value": [ 23:09:05 DEBUG "value": [ 23:09:05 DEBUG 1713305345, 23:09:05 DEBUG "value": [ 23:09:05 DEBUG 1713305345, 23:09:05 DEBUG 1713305345, 23:09:05 DEBUG ] 23:09:05 DEBUG "0.00040000000000001514" 23:09:05 DEBUG }, 23:09:05 DEBUG ] 23:09:05 DEBUG }, 23:09:05 DEBUG ] 23:09:05 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 23:09:05 DEBUG ] 23:09:05 DEBUG "pod": "am-555f4db595-kzxsp" 23:09:05 DEBUG 1713305345, 23:09:05 DEBUG } 23:09:05 DEBUG 1713305345, 23:09:05 DEBUG }, 23:09:05 DEBUG "31437730.499999996" 23:09:05 DEBUG 1713305345, 23:09:05 DEBUG 1713305345, 23:09:05 DEBUG "19987018752" 23:09:05 DEBUG "0" 23:09:05 DEBUG "0" 23:09:05 DEBUG "31444284.066666666" 23:09:05 DEBUG } 23:09:05 DEBUG ] 23:09:05 DEBUG "value": [ 23:09:05 DEBUG } 23:09:05 DEBUG "value": [ 23:09:05 DEBUG } 23:09:05 DEBUG }, 23:09:05 DEBUG } 23:09:05 DEBUG }, 23:09:05 DEBUG "0" 23:09:05 DEBUG ] 23:09:05 DEBUG "0" 23:09:05 DEBUG "value": [ 23:09:05 DEBUG ] 23:09:05 DEBUG "3524302.5" 23:09:05 DEBUG 1713305345, 23:09:05 DEBUG ] 23:09:05 DEBUG ] 23:09:05 DEBUG ] 23:09:05 DEBUG ] 23:09:05 DEBUG ] 23:09:05 DEBUG } 23:09:05 DEBUG 1713305345, 23:09:05 DEBUG ] 23:09:05 DEBUG 1713305345, 23:09:05 DEBUG ] 23:09:05 DEBUG "value": [ 23:09:05 DEBUG ] 23:09:05 DEBUG "value": [ 23:09:05 DEBUG ] 23:09:05 DEBUG } 23:09:05 DEBUG ] 23:09:05 DEBUG 1713305345, 23:09:05 DEBUG } 23:09:05 DEBUG ] 23:09:05 DEBUG "0" 23:09:05 DEBUG } 23:09:05 DEBUG } 23:09:05 DEBUG } 23:09:05 DEBUG } 23:09:05 DEBUG } 23:09:05 DEBUG ] 23:09:05 DEBUG "8354848768" 23:09:05 DEBUG } 23:09:05 DEBUG "0" 23:09:05 DEBUG } 23:09:05 DEBUG 1713305345, 23:09:05 DEBUG } 23:09:05 DEBUG 1713305345, 23:09:05 DEBUG } 23:09:05 DEBUG } 23:09:05 DEBUG } 23:09:05 DEBUG "706.1333333333333" 23:09:05 DEBUG ] 23:09:05 DEBUG } 23:09:05 DEBUG ] 23:09:05 DEBUG ] 23:09:05 DEBUG ] 23:09:05 DEBUG } 23:09:05 DEBUG ] 23:09:05 DEBUG } 23:09:05 DEBUG } 23:09:05 DEBUG ] 23:09:05 DEBUG } 23:09:05 DEBUG ] 23:09:05 DEBUG } 23:09:05 DEBUG "8.025014486927498" 23:09:05 DEBUG } 23:09:05 DEBUG "706.1333333333333" 23:09:05 DEBUG ] 23:09:05 DEBUG ] 23:09:05 DEBUG ] 23:09:05 DEBUG } 23:09:05 DEBUG ] 23:09:05 DEBUG } 23:09:05 DEBUG } 23:09:05 DEBUG ] 23:09:05 DEBUG } 23:09:05 DEBUG } 23:09:05 DEBUG } 23:09:05 DEBUG } 23:09:05 DEBUG } 23:09:05 DEBUG ] 23:09:05 DEBUG ] 23:09:05 DEBUG } 23:09:05 DEBUG } 23:09:05 DEBUG } 23:09:05 DEBUG } 23:09:05 DEBUG } 23:09:05 DEBUG ] 23:09:05 DEBUG } 23:09:05 DEBUG } 23:09:05 DEBUG } 23:09:05 DEBUG ] 23:09:05 DEBUG ] 23:09:05 DEBUG } 23:09:05 DEBUG } 23:09:05 DEBUG } 23:09:05 DEBUG } 23:09:05 DEBUG ] 23:09:05 DEBUG } 23:09:05 DEBUG } 23:09:05 DEBUG } 23:09:05 DEBUG } 23:09:05 DEBUG } 23:09:05 DEBUG ] 23:09:05 DEBUG ] 23:09:05 DEBUG } 23:09:05 DEBUG } 23:09:05 DEBUG } 23:09:05 DEBUG } 23:09:05 DEBUG } 23:09:05 DEBUG } 23:09:05 DEBUG } 23:09:05 DEBUG } 23:09:05 DEBUG } 23:09:06 WARNING File /tmp/lodemon_data-20240416_220944/node_disk_read_bytes_total.json does not exist. Skipping average value computation... 127.0.0.1 - - [16/Apr/2024 23:09:06] "GET /monitoring/average?start_time=20240416_214408&stop_time=20240416_220705 HTTP/1.1" 200 - 23:09:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713305399" 23:09:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305399" 23:09:59 ERROR [http_cmd]: ERROR 23:09:59 INFO [http_cmd]: http status code OK 23:09:59 ERROR --- status code --- 23:09:59 DEBUG --- status code --- 23:09:59 ERROR http status code is 400 (expected [200, 401, 403, 404]) 23:09:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:09:59 ERROR --- http response --- 23:09:59 DEBUG --- http response --- 23:09:59 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 23:09:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 23:09:59 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 23:10:02 INFO 23:10:02 INFO [loop_until]: kubectl --namespace=xlou top pods 23:10:02 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 23:10:02 INFO [loop_until]: OK (rc = 0) 23:10:02 DEBUG --- stdout --- 23:10:02 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 9Mi am-555f4db595-4lbss 1736m 8669Mi am-555f4db595-fz8sn 1867m 8561Mi am-555f4db595-kzxsp 1779m 8574Mi ds-cts-0 8854m 12764Mi ds-cts-1 7711m 13053Mi ds-cts-2 7866m 13823Mi ds-idrepo-0 805m 5039Mi ds-idrepo-1 570m 2900Mi ds-idrepo-2 626m 3341Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 4m 1207Mi idm-b85f9d978-rl76f 4m 1226Mi lodemon-bc756c647-cr7x6 2m 64Mi login-ui-5fdf49fc66-2wnx7 1m 5Mi overseer-0-7c9b667974-9mr2s 1m 18Mi 23:10:02 DEBUG --- stderr --- 23:10:02 DEBUG 23:10:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305405" 23:10:05 INFO [http_cmd]: http status code OK 23:10:05 DEBUG --- status code --- 23:10:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:10:05 DEBUG --- http response --- 23:10:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 23:10:05 DEBUG ----- output ----- 23:10:05 DEBUG { 23:10:05 DEBUG "status": "success", 23:10:05 DEBUG "data": { 23:10:05 DEBUG "resultType": "vector", 23:10:05 DEBUG "result": [ 23:10:05 DEBUG { 23:10:05 DEBUG "metric": { 23:10:05 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 23:10:05 DEBUG }, 23:10:05 DEBUG "value": [ 23:10:05 DEBUG 23:10:05 DEBUG 23:10:05 DEBUG 23:10:05 DEBUG [...] 23:10:05 DEBUG 23:10:05 DEBUG 23:10:05 DEBUG 23:10:05 DEBUG "pod": "idm-b85f9d978-j5pbm" 23:10:05 DEBUG }, 23:10:05 DEBUG "value": [ 23:10:05 DEBUG 1713305405, 23:10:05 DEBUG "0" 23:10:05 DEBUG ] 23:10:05 DEBUG } 23:10:05 DEBUG ] 23:10:05 DEBUG } 23:10:05 DEBUG } 23:10:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305405" 23:10:05 INFO [http_cmd]: http status code OK 23:10:05 DEBUG --- status code --- 23:10:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:10:05 DEBUG --- http response --- 23:10:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:10:05 DEBUG ----- output ----- 23:10:05 DEBUG { 23:10:05 DEBUG "status": "success", 23:10:05 DEBUG "data": { 23:10:05 DEBUG "resultType": "vector", 23:10:05 DEBUG "result": [ 23:10:05 DEBUG { 23:10:05 DEBUG "metric": { 23:10:05 DEBUG "pod": "am-555f4db595-fz8sn" 23:10:05 DEBUG }, 23:10:05 DEBUG "value": [ 23:10:05 DEBUG 23:10:05 DEBUG 23:10:05 DEBUG 23:10:05 DEBUG [...] 23:10:05 DEBUG 23:10:05 DEBUG 23:10:05 DEBUG 23:10:05 DEBUG "pod": "am-555f4db595-kzxsp" 23:10:05 DEBUG }, 23:10:05 DEBUG "value": [ 23:10:05 DEBUG 1713305405, 23:10:05 DEBUG "1992.9666666666667" 23:10:05 DEBUG ] 23:10:05 DEBUG } 23:10:05 DEBUG ] 23:10:05 DEBUG } 23:10:05 DEBUG } 23:10:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305405" 23:10:05 INFO 23:10:05 INFO [loop_until]: kubectl --namespace=xlou top node 23:10:05 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 23:10:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713305405" 23:10:05 INFO [http_cmd]: http status code OK 23:10:05 DEBUG --- status code --- 23:10:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:10:05 DEBUG --- http response --- 23:10:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:10:05 DEBUG ----- output ----- 23:10:05 DEBUG { 23:10:05 DEBUG "status": "success", 23:10:05 DEBUG "data": { 23:10:05 DEBUG "resultType": "vector", 23:10:05 DEBUG "result": [ 23:10:05 DEBUG { 23:10:05 DEBUG "metric": { 23:10:05 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 23:10:05 DEBUG }, 23:10:05 DEBUG "value": [ 23:10:05 DEBUG 23:10:05 DEBUG 23:10:05 DEBUG 23:10:05 DEBUG [...] 23:10:05 DEBUG 23:10:05 DEBUG 23:10:05 DEBUG 23:10:05 DEBUG "pod": "end-user-ui-58878d7668-zg4h2" 23:10:05 DEBUG }, 23:10:05 DEBUG "value": [ 23:10:05 DEBUG 1713305405, 23:10:05 DEBUG "0" 23:10:05 DEBUG ] 23:10:05 DEBUG } 23:10:05 DEBUG ] 23:10:05 DEBUG } 23:10:05 DEBUG } 23:10:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305405" 23:10:05 INFO [http_cmd]: http status code OK 23:10:05 DEBUG --- status code --- 23:10:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:10:05 DEBUG --- http response --- 23:10:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:10:05 DEBUG ----- output ----- 23:10:05 DEBUG { 23:10:05 DEBUG "status": "success", 23:10:05 DEBUG "data": { 23:10:05 DEBUG "resultType": "vector", 23:10:05 DEBUG "result": [ 23:10:05 DEBUG { 23:10:05 DEBUG "metric": { 23:10:05 DEBUG "instance": "10.142.0.107:9100" 23:10:05 DEBUG }, 23:10:05 DEBUG "value": [ 23:10:05 DEBUG 23:10:05 DEBUG 23:10:05 DEBUG 23:10:05 DEBUG [...] 23:10:05 INFO [http_cmd]: http status code OK 23:10:05 DEBUG 23:10:05 DEBUG --- status code --- 23:10:05 DEBUG 23:10:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:10:05 DEBUG 23:10:05 DEBUG --- http response --- 23:10:05 DEBUG "instance": "10.142.0.43:9100" 23:10:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 23:10:05 DEBUG }, 23:10:05 DEBUG ----- output ----- 23:10:05 DEBUG "value": [ 23:10:05 DEBUG { 23:10:05 DEBUG 1713305405, 23:10:05 DEBUG "status": "success", 23:10:05 DEBUG "0" 23:10:05 DEBUG "data": { 23:10:05 DEBUG ] 23:10:05 DEBUG "resultType": "vector", 23:10:05 DEBUG } 23:10:05 DEBUG "result": [ 23:10:05 DEBUG ] 23:10:05 DEBUG { 23:10:05 DEBUG } 23:10:05 DEBUG "metric": { 23:10:05 DEBUG } 23:10:05 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 23:10:05 DEBUG }, 23:10:05 DEBUG "value": [ 23:10:05 DEBUG 23:10:05 DEBUG 23:10:05 DEBUG 23:10:05 DEBUG [...] 23:10:05 DEBUG 23:10:05 DEBUG 23:10:05 DEBUG 23:10:05 DEBUG "pod": "idm-b85f9d978-j5pbm" 23:10:05 DEBUG }, 23:10:05 DEBUG "value": [ 23:10:05 DEBUG 1713305405, 23:10:05 DEBUG "0" 23:10:05 DEBUG ] 23:10:05 DEBUG } 23:10:05 DEBUG ] 23:10:05 DEBUG } 23:10:05 DEBUG } 23:10:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305405" 23:10:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713305405" 23:10:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713305405" 23:10:05 INFO [http_cmd]: http status code OK 23:10:05 DEBUG --- status code --- 23:10:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:10:05 DEBUG --- http response --- 23:10:05 INFO [http_cmd]: http status code OK 23:10:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:10:05 DEBUG --- status code --- 23:10:05 DEBUG ----- output ----- 23:10:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:10:05 DEBUG { 23:10:05 DEBUG --- http response --- 23:10:05 DEBUG "status": "success", 23:10:05 DEBUG "data": { 23:10:05 DEBUG "resultType": "vector", 23:10:05 DEBUG "result": [ 23:10:05 DEBUG { 23:10:05 DEBUG "metric": { 23:10:05 DEBUG "pod": "ds-idrepo-2" 23:10:05 DEBUG }, 23:10:05 DEBUG "value": [ 23:10:05 DEBUG 23:10:05 DEBUG 23:10:05 DEBUG 23:10:05 DEBUG [...] 23:10:05 DEBUG 23:10:05 DEBUG 23:10:05 DEBUG 23:10:05 DEBUG "pod": "ds-idrepo-0" 23:10:05 DEBUG }, 23:10:05 DEBUG "value": [ 23:10:05 DEBUG 1713305405, 23:10:05 DEBUG "0" 23:10:05 DEBUG ] 23:10:05 DEBUG } 23:10:05 DEBUG ] 23:10:05 DEBUG } 23:10:05 DEBUG } 23:10:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:10:05 DEBUG ----- output ----- 23:10:05 DEBUG { 23:10:05 DEBUG "status": "success", 23:10:05 DEBUG "data": { 23:10:05 DEBUG "resultType": "vector", 23:10:05 DEBUG "result": [ 23:10:05 DEBUG { 23:10:05 DEBUG "metric": { 23:10:05 DEBUG "instance": "10.142.0.107:9100" 23:10:05 DEBUG }, 23:10:05 DEBUG "value": [ 23:10:05 DEBUG 23:10:05 DEBUG 23:10:05 DEBUG 23:10:05 DEBUG [...] 23:10:05 DEBUG 23:10:05 DEBUG 23:10:05 DEBUG 23:10:05 DEBUG "instance": "10.142.0.43:9100" 23:10:05 DEBUG }, 23:10:05 DEBUG "value": [ 23:10:05 DEBUG 1713305405, 23:10:05 DEBUG "17497.166666666668" 23:10:05 DEBUG ] 23:10:05 DEBUG } 23:10:05 DEBUG ] 23:10:05 DEBUG } 23:10:05 DEBUG } 23:10:05 INFO [loop_until]: OK (rc = 0) 23:10:05 DEBUG --- stdout --- 23:10:05 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 72m 0% 5100Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7793m 26% 15600Mi 13% gke-xlou-cdm-default-pool-1b0769c8-nhlf 1832m 6% 10822Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 89m 0% 2670Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 651m 2% 6613Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 9443m 31% 17208Mi 15% gke-xlou-cdm-default-pool-1b0769c8-xnmp 100m 0% 3158Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 1807m 6% 10510Mi 9% gke-xlou-cdm-ds-0f601751-hmm7 642m 2% 5677Mi 5% gke-xlou-cdm-ds-0f601751-j488 7728m 25% 15697Mi 14% gke-xlou-cdm-ds-0f601751-mc5d 142m 0% 3931Mi 3% gke-xlou-cdm-ds-0f601751-nj6q 1914m 6% 10481Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 827m 2% 7370Mi 6% gke-xlou-cdm-frontend-a8771548-i82n 555m 3% 5634Mi 9% 23:10:05 DEBUG --- stderr --- 23:10:05 DEBUG 23:10:05 INFO [http_cmd]: http status code OK 23:10:05 DEBUG --- status code --- 23:10:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:10:05 DEBUG --- http response --- 23:10:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:10:05 DEBUG ----- output ----- 23:10:05 DEBUG { 23:10:05 DEBUG "status": "success", 23:10:05 DEBUG "data": { 23:10:05 DEBUG "resultType": "vector", 23:10:05 DEBUG "result": [ 23:10:05 DEBUG { 23:10:05 DEBUG "metric": { 23:10:05 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 23:10:05 DEBUG }, 23:10:05 DEBUG "value": [ 23:10:05 DEBUG 23:10:05 DEBUG 23:10:05 DEBUG 23:10:05 DEBUG [...] 23:10:05 DEBUG 23:10:05 DEBUG 23:10:05 DEBUG 23:10:05 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 23:10:05 DEBUG }, 23:10:05 DEBUG "value": [ 23:10:05 DEBUG 1713305405, 23:10:05 DEBUG "0.023272684229175854" 23:10:05 DEBUG ] 23:10:05 DEBUG } 23:10:05 DEBUG ] 23:10:05 DEBUG } 23:10:05 DEBUG } 23:10:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305405" 23:10:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305405" 23:10:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713305405" 23:10:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305405" 23:10:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305405" 23:10:05 INFO [http_cmd]: http status code OK 23:10:05 INFO [http_cmd]: http status code OK 23:10:05 INFO [http_cmd]: http status code OK 23:10:05 DEBUG --- status code --- 23:10:05 DEBUG --- status code --- 23:10:05 DEBUG --- status code --- 23:10:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:10:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:10:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:10:05 DEBUG --- http response --- 23:10:05 DEBUG --- http response --- 23:10:05 DEBUG --- http response --- 23:10:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:10:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:10:05 DEBUG ----- output ----- 23:10:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:10:05 DEBUG ----- output ----- 23:10:05 DEBUG { 23:10:05 DEBUG ----- output ----- 23:10:05 DEBUG { 23:10:05 DEBUG "status": "success", 23:10:05 DEBUG { 23:10:05 DEBUG "status": "success", 23:10:05 DEBUG "data": { 23:10:05 DEBUG "status": "success", 23:10:05 DEBUG "data": { 23:10:05 DEBUG "resultType": "vector", 23:10:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305405" 23:10:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713305405" 23:10:05 DEBUG "data": { 23:10:05 DEBUG "resultType": "vector", 23:10:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305405" 23:10:05 DEBUG "result": [ 23:10:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305405" 23:10:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713305405" 23:10:05 DEBUG "resultType": "vector", 23:10:05 DEBUG "result": [ 23:10:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713305405" 23:10:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713305405" 23:10:05 DEBUG { 23:10:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713305405" 23:10:05 INFO [http_cmd]: http status code OK 23:10:05 DEBUG "result": [ 23:10:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305405" 23:10:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305405" 23:10:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305405" 23:10:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305405" 23:10:05 DEBUG { 23:10:05 INFO [http_cmd]: http status code OK 23:10:05 DEBUG "metric": { 23:10:05 DEBUG --- status code --- 23:10:05 DEBUG { 23:10:05 INFO [http_cmd]: http status code OK 23:10:05 INFO [http_cmd]: http status code OK 23:10:05 INFO [http_cmd]: http status code OK 23:10:05 INFO [http_cmd]: http status code OK 23:10:05 INFO [http_cmd]: http status code OK 23:10:05 DEBUG "metric": { 23:10:05 DEBUG --- status code --- 23:10:05 DEBUG "instance": "10.142.0.107:9100" 23:10:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:10:05 DEBUG "metric": { 23:10:05 DEBUG --- status code --- 23:10:05 DEBUG --- status code --- 23:10:05 DEBUG --- status code --- 23:10:05 DEBUG --- status code --- 23:10:05 DEBUG --- status code --- 23:10:05 DEBUG "pod": "am-555f4db595-fz8sn" 23:10:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:10:05 DEBUG }, 23:10:05 DEBUG --- http response --- 23:10:05 DEBUG "pod": "am-555f4db595-fz8sn" 23:10:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:10:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:10:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:10:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:10:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:10:05 DEBUG }, 23:10:05 DEBUG --- http response --- 23:10:05 DEBUG "value": [ 23:10:05 DEBUG }, 23:10:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:10:05 DEBUG --- http response --- 23:10:05 INFO [http_cmd]: http status code OK 23:10:05 DEBUG --- http response --- 23:10:05 DEBUG --- http response --- 23:10:05 INFO [http_cmd]: http status code OK 23:10:05 INFO [http_cmd]: http status code OK 23:10:05 DEBUG --- http response --- 23:10:05 DEBUG --- http response --- 23:10:05 DEBUG "value": [ 23:10:05 DEBUG 23:10:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:10:05 DEBUG "value": [ 23:10:05 DEBUG ----- output ----- 23:10:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:10:05 DEBUG --- status code --- 23:10:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:10:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:10:05 DEBUG --- status code --- 23:10:05 DEBUG --- status code --- 23:10:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 23:10:05 DEBUG 23:10:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:10:05 DEBUG 23:10:05 DEBUG ----- output ----- 23:10:05 DEBUG 23:10:05 DEBUG { 23:10:05 DEBUG ----- output ----- 23:10:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:10:05 DEBUG ----- output ----- 23:10:05 DEBUG ----- output ----- 23:10:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:10:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:10:05 DEBUG ----- output ----- 23:10:05 DEBUG 23:10:05 DEBUG ----- output ----- 23:10:05 DEBUG 23:10:05 DEBUG { 23:10:05 DEBUG 23:10:05 DEBUG "status": "success", 23:10:05 DEBUG { 23:10:05 DEBUG --- http response --- 23:10:05 DEBUG { 23:10:05 DEBUG { 23:10:05 DEBUG --- http response --- 23:10:05 DEBUG --- http response --- 23:10:05 DEBUG { 23:10:05 DEBUG 23:10:05 DEBUG { 23:10:05 DEBUG [...] 23:10:05 DEBUG "status": "success", 23:10:05 DEBUG "data": { 23:10:05 DEBUG "data": { 23:10:05 DEBUG "status": "success", 23:10:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:10:05 DEBUG "status": "success", 23:10:05 DEBUG "status": "success", 23:10:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 23:10:05 DEBUG "status": "success", 23:10:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 23:10:05 DEBUG [...] 23:10:05 DEBUG "status": "success", 23:10:05 DEBUG 23:10:05 DEBUG "resultType": "vector", 23:10:05 DEBUG 23:10:05 DEBUG "resultType": "vector", 23:10:05 DEBUG "data": { 23:10:05 DEBUG ----- output ----- 23:10:05 DEBUG "data": { 23:10:05 DEBUG "data": { 23:10:05 DEBUG ----- output ----- 23:10:05 DEBUG "data": { 23:10:05 DEBUG ----- output ----- 23:10:05 DEBUG 23:10:05 DEBUG "data": { 23:10:05 DEBUG 23:10:05 DEBUG "result": [ 23:10:05 DEBUG [...] 23:10:05 DEBUG "result": [ 23:10:05 DEBUG "resultType": "vector", 23:10:05 DEBUG { 23:10:05 DEBUG "resultType": "vector", 23:10:05 DEBUG "resultType": "vector", 23:10:05 DEBUG { 23:10:05 DEBUG "resultType": "vector", 23:10:05 DEBUG { 23:10:05 DEBUG 23:10:05 DEBUG "resultType": "vector", 23:10:05 DEBUG 23:10:05 DEBUG { 23:10:05 DEBUG 23:10:05 DEBUG { 23:10:05 DEBUG "result": [ 23:10:05 DEBUG "status": "success", 23:10:05 DEBUG "result": [ 23:10:05 DEBUG "result": [ 23:10:05 DEBUG "status": "success", 23:10:05 DEBUG "result": [ 23:10:05 DEBUG "status": "success", 23:10:05 DEBUG 23:10:05 DEBUG "result": [ 23:10:05 DEBUG "instance": "10.142.0.43:9100" 23:10:05 DEBUG "metric": { 23:10:05 DEBUG 23:10:05 DEBUG "metric": { 23:10:05 DEBUG { 23:10:05 DEBUG "data": { 23:10:05 DEBUG { 23:10:05 DEBUG { 23:10:05 DEBUG "data": { 23:10:05 DEBUG { 23:10:05 DEBUG "data": { 23:10:05 DEBUG "pod": "am-555f4db595-kzxsp" 23:10:05 DEBUG { 23:10:05 DEBUG }, 23:10:05 DEBUG "pod": "ds-idrepo-2" 23:10:05 DEBUG 23:10:05 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 23:10:05 DEBUG "metric": { 23:10:05 DEBUG "resultType": "vector", 23:10:05 DEBUG "metric": { 23:10:05 DEBUG "metric": { 23:10:05 DEBUG "resultType": "vector", 23:10:05 DEBUG "metric": { 23:10:05 DEBUG "resultType": "vector", 23:10:05 DEBUG }, 23:10:05 DEBUG "metric": { 23:10:05 DEBUG "value": [ 23:10:05 DEBUG }, 23:10:05 DEBUG "pod": "am-555f4db595-kzxsp" 23:10:05 DEBUG }, 23:10:05 DEBUG "pod": "am-555f4db595-fz8sn" 23:10:05 DEBUG "result": [ 23:10:05 DEBUG "pod": "ds-idrepo-2" 23:10:05 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 23:10:05 DEBUG "result": [ 23:10:05 DEBUG "pod": "ds-idrepo-2" 23:10:05 DEBUG "result": [ 23:10:05 DEBUG "value": [ 23:10:05 DEBUG "pod": "am-555f4db595-fz8sn" 23:10:05 DEBUG 1713305405, 23:10:05 DEBUG "value": [ 23:10:05 DEBUG }, 23:10:05 DEBUG "value": [ 23:10:05 DEBUG }, 23:10:05 DEBUG { 23:10:05 DEBUG }, 23:10:05 DEBUG }, 23:10:05 DEBUG { 23:10:05 DEBUG }, 23:10:05 DEBUG { 23:10:05 DEBUG 1713305405, 23:10:05 DEBUG }, 23:10:05 DEBUG "19987577856" 23:10:05 DEBUG 23:10:05 DEBUG "value": [ 23:10:05 DEBUG 23:10:05 DEBUG "value": [ 23:10:05 DEBUG "metric": { 23:10:05 DEBUG "value": [ 23:10:05 DEBUG "value": [ 23:10:05 DEBUG "metric": { 23:10:05 DEBUG "value": [ 23:10:05 DEBUG "metric": { 23:10:05 DEBUG "3415067.5" 23:10:05 DEBUG "value": [ 23:10:05 DEBUG ] 23:10:05 DEBUG 23:10:05 DEBUG 23:10:05 DEBUG 23:10:05 DEBUG 23:10:05 DEBUG "instance": "10.142.0.107:9100" 23:10:05 DEBUG 23:10:05 DEBUG 23:10:05 DEBUG "pod": "ds-idrepo-2" 23:10:05 DEBUG 23:10:05 DEBUG "pod": "ds-idrepo-2" 23:10:05 DEBUG ] 23:10:05 DEBUG 23:10:05 DEBUG } 23:10:05 DEBUG 1713305405, 23:10:05 DEBUG [...] 23:10:05 DEBUG 23:10:05 DEBUG 23:10:05 DEBUG }, 23:10:05 DEBUG 23:10:05 DEBUG 23:10:05 DEBUG }, 23:10:05 DEBUG 23:10:05 DEBUG }, 23:10:05 INFO [http_cmd]: http status code OK 23:10:05 INFO [http_cmd]: http status code OK 23:10:05 INFO [http_cmd]: http status code OK 23:10:05 DEBUG } 23:10:05 INFO [http_cmd]: http status code OK 23:10:05 DEBUG 23:10:05 DEBUG ] 23:10:05 DEBUG "0.06666666666666667" 23:10:05 DEBUG 23:10:05 DEBUG [...] 23:10:05 DEBUG 23:10:05 DEBUG "value": [ 23:10:05 DEBUG 23:10:05 DEBUG 23:10:05 DEBUG "value": [ 23:10:05 DEBUG 23:10:05 DEBUG "value": [ 23:10:05 DEBUG --- status code --- 23:10:05 DEBUG --- status code --- 23:10:05 DEBUG --- status code --- 23:10:05 DEBUG ] 23:10:05 DEBUG --- status code --- 23:10:05 DEBUG 23:10:05 DEBUG } 23:10:05 DEBUG ] 23:10:05 DEBUG 23:10:05 DEBUG 23:10:05 DEBUG [...] 23:10:05 DEBUG 23:10:05 DEBUG [...] 23:10:05 DEBUG [...] 23:10:05 DEBUG 23:10:05 DEBUG [...] 23:10:05 DEBUG 23:10:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:10:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:10:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:10:05 DEBUG } 23:10:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:10:05 DEBUG [...] 23:10:05 DEBUG } 23:10:05 DEBUG } 23:10:05 DEBUG 23:10:05 DEBUG 23:10:05 DEBUG 23:10:05 DEBUG 23:10:05 DEBUG 23:10:05 DEBUG 23:10:05 DEBUG 23:10:05 DEBUG 23:10:05 DEBUG 23:10:05 DEBUG --- http response --- 23:10:05 DEBUG --- http response --- 23:10:05 DEBUG --- http response --- 23:10:05 DEBUG } 23:10:05 DEBUG --- http response --- 23:10:05 DEBUG 23:10:05 DEBUG ] 23:10:05 DEBUG "pod": "ds-idrepo-0" 23:10:05 DEBUG 23:10:05 DEBUG 23:10:05 DEBUG 23:10:05 DEBUG 23:10:05 DEBUG 23:10:05 DEBUG 23:10:05 DEBUG 23:10:05 DEBUG 23:10:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:10:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 23:10:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:10:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:10:05 DEBUG 23:10:05 DEBUG } 23:10:05 DEBUG }, 23:10:05 DEBUG "pod": "end-user-ui-58878d7668-zg4h2" 23:10:05 DEBUG 23:10:05 DEBUG [...] 23:10:05 DEBUG 23:10:05 DEBUG 23:10:05 DEBUG [...] 23:10:05 DEBUG 23:10:05 DEBUG [...] 23:10:05 DEBUG ----- output ----- 23:10:05 DEBUG ----- output ----- 23:10:05 DEBUG ----- output ----- 23:10:05 DEBUG ----- output ----- 23:10:05 DEBUG 23:10:05 DEBUG } 23:10:05 DEBUG "value": [ 23:10:05 DEBUG }, 23:10:05 DEBUG "pod": "am-555f4db595-kzxsp" 23:10:05 DEBUG 23:10:05 DEBUG "pod": "ds-idrepo-0" 23:10:05 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 23:10:05 DEBUG 23:10:05 DEBUG "pod": "ds-idrepo-0" 23:10:05 DEBUG 23:10:05 DEBUG { 23:10:05 DEBUG { 23:10:05 DEBUG { 23:10:05 DEBUG { 23:10:05 DEBUG "pod": "am-555f4db595-kzxsp" 23:10:05 DEBUG 1713305405, 23:10:05 DEBUG "value": [ 23:10:05 DEBUG }, 23:10:05 DEBUG 23:10:05 DEBUG }, 23:10:05 DEBUG }, 23:10:05 DEBUG 23:10:05 DEBUG }, 23:10:05 DEBUG 23:10:05 DEBUG "status": "success", 23:10:05 DEBUG "status": "success", 23:10:05 DEBUG "status": "success", 23:10:05 DEBUG "status": "success", 23:10:05 DEBUG }, 23:10:05 DEBUG "0" 23:10:05 DEBUG 1713305405, 23:10:05 DEBUG "value": [ 23:10:05 DEBUG 23:10:05 DEBUG "value": [ 23:10:05 DEBUG "value": [ 23:10:05 DEBUG 23:10:05 DEBUG "value": [ 23:10:05 DEBUG 23:10:05 DEBUG "data": { 23:10:05 DEBUG "data": { 23:10:05 DEBUG "data": { 23:10:05 DEBUG "data": { 23:10:05 DEBUG "value": [ 23:10:05 DEBUG ] 23:10:05 DEBUG "0" 23:10:05 DEBUG 1713305405, 23:10:05 DEBUG "instance": "10.142.0.43:9100" 23:10:05 DEBUG 1713305405, 23:10:05 DEBUG 1713305405, 23:10:05 DEBUG "pod": "ds-cts-2" 23:10:05 DEBUG 1713305405, 23:10:05 DEBUG "pod": "ds-idrepo-0" 23:10:05 DEBUG "resultType": "vector", 23:10:05 DEBUG "resultType": "vector", 23:10:05 DEBUG "resultType": "vector", 23:10:05 DEBUG "resultType": "vector", 23:10:05 DEBUG 1713305405, 23:10:05 DEBUG } 23:10:05 DEBUG ] 23:10:05 DEBUG "0.06666666666666667" 23:10:05 DEBUG }, 23:10:05 DEBUG "0" 23:10:05 DEBUG "118353920" 23:10:05 DEBUG }, 23:10:05 DEBUG "0" 23:10:05 DEBUG }, 23:10:05 DEBUG "result": [ 23:10:05 DEBUG "result": [ 23:10:05 DEBUG "result": [ 23:10:05 DEBUG "result": [ 23:10:05 DEBUG "0.0275" 23:10:05 DEBUG ] 23:10:05 DEBUG } 23:10:05 DEBUG ] 23:10:05 DEBUG "value": [ 23:10:05 DEBUG ] 23:10:05 DEBUG ] 23:10:05 DEBUG "value": [ 23:10:05 DEBUG ] 23:10:05 DEBUG "value": [ 23:10:05 DEBUG { 23:10:05 DEBUG { 23:10:05 DEBUG { 23:10:05 DEBUG { 23:10:05 DEBUG ] 23:10:05 DEBUG } 23:10:05 DEBUG ] 23:10:05 DEBUG } 23:10:05 DEBUG 1713305405, 23:10:05 DEBUG } 23:10:05 DEBUG } 23:10:05 DEBUG 1713305405, 23:10:05 DEBUG } 23:10:05 DEBUG 1713305405, 23:10:05 DEBUG "metric": { 23:10:05 DEBUG "metric": { 23:10:05 DEBUG "metric": { 23:10:05 DEBUG "metric": { 23:10:05 DEBUG } 23:10:05 DEBUG } 23:10:05 DEBUG } 23:10:05 DEBUG ] 23:10:05 DEBUG "261299.06666666665" 23:10:05 DEBUG ] 23:10:05 DEBUG ] 23:10:05 DEBUG "0" 23:10:05 DEBUG ] 23:10:05 DEBUG "0" 23:10:05 DEBUG "pod": "am-555f4db595-fz8sn" 23:10:05 DEBUG "pod": "ds-idrepo-2" 23:10:05 DEBUG "instance": "10.142.0.107:9100" 23:10:05 DEBUG "pod": "am-555f4db595-fz8sn" 23:10:05 DEBUG ] 23:10:05 DEBUG } 23:10:05 DEBUG } 23:10:05 DEBUG ] 23:10:05 DEBUG } 23:10:05 DEBUG } 23:10:05 DEBUG ] 23:10:05 DEBUG } 23:10:05 DEBUG ] 23:10:05 DEBUG }, 23:10:05 DEBUG }, 23:10:05 DEBUG }, 23:10:05 DEBUG }, 23:10:05 DEBUG } 23:10:05 DEBUG } 23:10:05 DEBUG } 23:10:05 DEBUG } 23:10:05 DEBUG } 23:10:05 DEBUG } 23:10:05 DEBUG } 23:10:05 DEBUG } 23:10:05 DEBUG "value": [ 23:10:05 DEBUG "value": [ 23:10:05 DEBUG "value": [ 23:10:05 DEBUG "value": [ 23:10:05 DEBUG } 23:10:05 DEBUG ] 23:10:05 DEBUG ] 23:10:05 DEBUG ] 23:10:05 DEBUG 23:10:05 DEBUG 23:10:05 DEBUG 23:10:05 DEBUG 23:10:05 DEBUG } 23:10:05 DEBUG } 23:10:05 DEBUG } 23:10:05 DEBUG 23:10:05 DEBUG 23:10:05 DEBUG 23:10:05 DEBUG 23:10:05 DEBUG } 23:10:05 DEBUG } 23:10:05 DEBUG } 23:10:05 DEBUG 23:10:05 DEBUG 23:10:05 DEBUG 23:10:05 DEBUG 23:10:05 DEBUG [...] 23:10:05 DEBUG [...] 23:10:05 DEBUG [...] 23:10:05 DEBUG [...] 23:10:05 DEBUG 23:10:05 DEBUG 23:10:05 DEBUG 23:10:05 DEBUG 23:10:05 DEBUG 23:10:05 DEBUG 23:10:05 DEBUG 23:10:05 DEBUG 23:10:05 DEBUG 23:10:05 DEBUG 23:10:05 DEBUG 23:10:05 DEBUG 23:10:05 DEBUG "pod": "am-555f4db595-kzxsp" 23:10:05 DEBUG "pod": "ds-idrepo-0" 23:10:05 DEBUG "instance": "10.142.0.43:9100" 23:10:05 DEBUG "pod": "am-555f4db595-kzxsp" 23:10:05 DEBUG }, 23:10:05 DEBUG }, 23:10:05 DEBUG }, 23:10:05 DEBUG }, 23:10:05 DEBUG "value": [ 23:10:05 DEBUG "value": [ 23:10:05 DEBUG "value": [ 23:10:05 DEBUG "value": [ 23:10:05 DEBUG 1713305405, 23:10:05 DEBUG 1713305405, 23:10:05 DEBUG 1713305405, 23:10:05 DEBUG 1713305405, 23:10:05 DEBUG "0.03333333333333333" 23:10:05 DEBUG "0" 23:10:05 DEBUG "0.0004333333333325603" 23:10:05 DEBUG "0.06666666666666667" 23:10:05 DEBUG ] 23:10:05 DEBUG ] 23:10:05 DEBUG ] 23:10:05 DEBUG ] 23:10:05 DEBUG } 23:10:05 DEBUG } 23:10:05 DEBUG } 23:10:05 DEBUG } 23:10:05 DEBUG ] 23:10:05 DEBUG ] 23:10:05 DEBUG ] 23:10:05 DEBUG ] 23:10:05 DEBUG } 23:10:05 DEBUG } 23:10:05 DEBUG } 23:10:05 DEBUG } 23:10:05 DEBUG } 23:10:05 DEBUG } 23:10:05 DEBUG } 23:10:05 DEBUG } 23:10:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713305459" 23:10:59 ERROR [http_cmd]: ERROR 23:10:59 ERROR --- status code --- 23:10:59 ERROR http status code is 400 (expected [200, 401, 403, 404]) 23:10:59 ERROR --- http response --- 23:10:59 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 23:10:59 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 23:10:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305459" 23:10:59 INFO [http_cmd]: http status code OK 23:10:59 DEBUG --- status code --- 23:10:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:10:59 DEBUG --- http response --- 23:10:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 23:11:02 INFO 23:11:02 INFO [loop_until]: kubectl --namespace=xlou top pods 23:11:02 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 23:11:02 INFO [loop_until]: OK (rc = 0) 23:11:02 DEBUG --- stdout --- 23:11:02 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 9Mi am-555f4db595-4lbss 1765m 8671Mi am-555f4db595-fz8sn 1881m 8535Mi am-555f4db595-kzxsp 1783m 8562Mi ds-cts-0 8359m 13513Mi ds-cts-1 7299m 13295Mi ds-cts-2 7509m 13924Mi ds-idrepo-0 825m 5044Mi ds-idrepo-1 663m 2913Mi ds-idrepo-2 615m 3239Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 4m 1208Mi idm-b85f9d978-rl76f 4m 1227Mi lodemon-bc756c647-cr7x6 2m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-7c9b667974-9mr2s 1002m 492Mi 23:11:02 DEBUG --- stderr --- 23:11:02 DEBUG 23:11:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305465" 23:11:05 INFO [http_cmd]: http status code OK 23:11:05 DEBUG --- status code --- 23:11:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:11:05 DEBUG --- http response --- 23:11:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:11:05 DEBUG ----- output ----- 23:11:05 DEBUG { 23:11:05 DEBUG "status": "success", 23:11:05 DEBUG "data": { 23:11:05 DEBUG "resultType": "vector", 23:11:05 DEBUG "result": [ 23:11:05 DEBUG { 23:11:05 DEBUG "metric": { 23:11:05 DEBUG "pod": "ds-cts-1" 23:11:05 DEBUG }, 23:11:05 DEBUG "value": [ 23:11:05 DEBUG 23:11:05 DEBUG 23:11:05 DEBUG 23:11:05 DEBUG [...] 23:11:05 DEBUG 23:11:05 DEBUG 23:11:05 DEBUG 23:11:05 DEBUG "pod": "ds-cts-2" 23:11:05 DEBUG }, 23:11:05 DEBUG "value": [ 23:11:05 DEBUG 1713305465, 23:11:05 DEBUG "0" 23:11:05 DEBUG ] 23:11:05 DEBUG } 23:11:05 DEBUG ] 23:11:05 DEBUG } 23:11:05 DEBUG } 23:11:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305465" 23:11:05 INFO [http_cmd]: http status code OK 23:11:05 DEBUG --- status code --- 23:11:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:11:05 DEBUG --- http response --- 23:11:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:11:05 DEBUG ----- output ----- 23:11:05 DEBUG { 23:11:05 DEBUG "status": "success", 23:11:05 DEBUG "data": { 23:11:05 DEBUG "resultType": "vector", 23:11:05 DEBUG "result": [ 23:11:05 DEBUG { 23:11:05 DEBUG "metric": { 23:11:05 DEBUG "pod": "am-555f4db595-fz8sn" 23:11:05 DEBUG }, 23:11:05 DEBUG "value": [ 23:11:05 DEBUG 23:11:05 DEBUG 23:11:05 DEBUG 23:11:05 DEBUG [...] 23:11:05 DEBUG 23:11:05 DEBUG 23:11:05 DEBUG 23:11:05 DEBUG "pod": "am-555f4db595-kzxsp" 23:11:05 DEBUG }, 23:11:05 DEBUG "value": [ 23:11:05 DEBUG 1713305465, 23:11:05 DEBUG "1996.0666666666666" 23:11:05 DEBUG ] 23:11:05 DEBUG } 23:11:05 DEBUG ] 23:11:05 DEBUG } 23:11:05 DEBUG } 23:11:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305465" 23:11:05 INFO [http_cmd]: http status code OK 23:11:05 DEBUG --- status code --- 23:11:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:11:05 DEBUG --- http response --- 23:11:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:11:05 DEBUG ----- output ----- 23:11:05 DEBUG { 23:11:05 DEBUG "status": "success", 23:11:05 DEBUG "data": { 23:11:05 DEBUG "resultType": "vector", 23:11:05 DEBUG "result": [ 23:11:05 DEBUG { 23:11:05 DEBUG "metric": { 23:11:05 DEBUG "pod": "ds-cts-1" 23:11:05 DEBUG }, 23:11:05 DEBUG "value": [ 23:11:05 DEBUG 23:11:05 DEBUG 23:11:05 DEBUG 23:11:05 DEBUG [...] 23:11:05 DEBUG 23:11:05 DEBUG 23:11:05 DEBUG 23:11:05 DEBUG "pod": "ds-cts-2" 23:11:05 DEBUG }, 23:11:05 DEBUG "value": [ 23:11:05 DEBUG 1713305465, 23:11:05 DEBUG "0" 23:11:05 DEBUG ] 23:11:05 DEBUG } 23:11:05 DEBUG ] 23:11:05 DEBUG } 23:11:05 DEBUG } 23:11:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305465" 23:11:05 INFO [http_cmd]: http status code OK 23:11:05 DEBUG --- status code --- 23:11:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:11:05 DEBUG --- http response --- 23:11:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 23:11:05 DEBUG ----- output ----- 23:11:05 DEBUG { 23:11:05 DEBUG "status": "success", 23:11:05 DEBUG "data": { 23:11:05 DEBUG "resultType": "vector", 23:11:05 DEBUG "result": [ 23:11:05 DEBUG { 23:11:05 DEBUG "metric": { 23:11:05 DEBUG "pod": "ds-cts-1" 23:11:05 DEBUG }, 23:11:05 DEBUG "value": [ 23:11:05 DEBUG 23:11:05 DEBUG 23:11:05 DEBUG 23:11:05 DEBUG [...] 23:11:05 DEBUG 23:11:05 DEBUG 23:11:05 DEBUG 23:11:05 DEBUG "pod": "lodemon-bc756c647-cr7x6" 23:11:05 DEBUG }, 23:11:05 DEBUG "value": [ 23:11:05 DEBUG 1713305465, 23:11:05 DEBUG "0" 23:11:05 DEBUG ] 23:11:05 DEBUG } 23:11:05 DEBUG ] 23:11:05 DEBUG } 23:11:05 DEBUG } 23:11:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713305465" 23:11:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713305465" 23:11:05 INFO [http_cmd]: http status code OK 23:11:05 DEBUG --- status code --- 23:11:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:11:05 DEBUG --- http response --- 23:11:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:11:05 DEBUG ----- output ----- 23:11:05 DEBUG { 23:11:05 DEBUG "status": "success", 23:11:05 DEBUG "data": { 23:11:05 DEBUG "resultType": "vector", 23:11:05 DEBUG "result": [ 23:11:05 DEBUG { 23:11:05 DEBUG "metric": { 23:11:05 DEBUG "instance": "10.142.0.107:9100" 23:11:05 DEBUG }, 23:11:05 DEBUG "value": [ 23:11:05 DEBUG 23:11:05 DEBUG 23:11:05 DEBUG 23:11:05 DEBUG [...] 23:11:05 DEBUG 23:11:05 DEBUG 23:11:05 DEBUG 23:11:05 DEBUG "instance": "10.142.0.43:9100" 23:11:05 DEBUG }, 23:11:05 DEBUG "value": [ 23:11:05 DEBUG 1713305465, 23:11:05 DEBUG "7258.8" 23:11:05 DEBUG ] 23:11:05 DEBUG } 23:11:05 DEBUG ] 23:11:05 DEBUG } 23:11:05 DEBUG } 23:11:05 INFO [http_cmd]: http status code OK 23:11:05 DEBUG --- status code --- 23:11:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:11:05 DEBUG --- http response --- 23:11:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:11:05 DEBUG ----- output ----- 23:11:05 DEBUG { 23:11:05 DEBUG "status": "success", 23:11:05 DEBUG "data": { 23:11:05 DEBUG "resultType": "vector", 23:11:05 DEBUG "result": [ 23:11:05 DEBUG { 23:11:05 DEBUG "metric": { 23:11:05 DEBUG "instance": "10.142.0.107:9100" 23:11:05 DEBUG }, 23:11:05 DEBUG "value": [ 23:11:05 DEBUG 23:11:05 DEBUG 23:11:05 DEBUG 23:11:05 DEBUG [...] 23:11:05 DEBUG 23:11:05 DEBUG 23:11:05 DEBUG 23:11:05 DEBUG "instance": "10.142.0.43:9100" 23:11:05 DEBUG }, 23:11:05 DEBUG "value": [ 23:11:05 DEBUG 1713305465, 23:11:05 DEBUG "0" 23:11:05 DEBUG ] 23:11:05 DEBUG } 23:11:05 DEBUG ] 23:11:05 DEBUG } 23:11:05 DEBUG } 23:11:05 INFO 23:11:05 INFO [loop_until]: kubectl --namespace=xlou top node 23:11:05 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 23:11:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305465" 23:11:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713305465" 23:11:05 INFO [http_cmd]: http status code OK 23:11:05 DEBUG --- status code --- 23:11:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:11:05 DEBUG --- http response --- 23:11:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:11:05 DEBUG ----- output ----- 23:11:05 DEBUG { 23:11:05 DEBUG "status": "success", 23:11:05 DEBUG "data": { 23:11:05 DEBUG "resultType": "vector", 23:11:05 DEBUG "result": [ 23:11:05 DEBUG { 23:11:05 DEBUG "metric": { 23:11:05 DEBUG "pod": "ds-idrepo-2" 23:11:05 DEBUG }, 23:11:05 DEBUG "value": [ 23:11:05 DEBUG 23:11:05 DEBUG 23:11:05 DEBUG 23:11:05 DEBUG [...] 23:11:05 DEBUG 23:11:05 DEBUG 23:11:05 DEBUG 23:11:05 DEBUG "pod": "ds-idrepo-0" 23:11:05 DEBUG }, 23:11:05 DEBUG "value": [ 23:11:05 DEBUG 1713305465, 23:11:05 DEBUG "0" 23:11:05 DEBUG ] 23:11:05 DEBUG } 23:11:05 DEBUG ] 23:11:05 DEBUG } 23:11:05 DEBUG } 23:11:05 INFO [http_cmd]: http status code OK 23:11:05 DEBUG --- status code --- 23:11:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:11:05 DEBUG --- http response --- 23:11:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:11:05 DEBUG ----- output ----- 23:11:05 DEBUG { 23:11:05 DEBUG "status": "success", 23:11:05 DEBUG "data": { 23:11:05 DEBUG "resultType": "vector", 23:11:05 DEBUG "result": [ 23:11:05 DEBUG { 23:11:05 DEBUG "metric": { 23:11:05 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 23:11:05 DEBUG }, 23:11:05 DEBUG "value": [ 23:11:05 DEBUG 23:11:05 DEBUG 23:11:05 DEBUG 23:11:05 DEBUG [...] 23:11:05 DEBUG 23:11:05 DEBUG 23:11:05 DEBUG 23:11:05 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 23:11:05 DEBUG }, 23:11:05 DEBUG "value": [ 23:11:05 DEBUG 1713305465, 23:11:05 DEBUG "1.2618040563517052" 23:11:05 DEBUG ] 23:11:05 DEBUG } 23:11:05 DEBUG ] 23:11:05 DEBUG } 23:11:05 DEBUG } 23:11:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713305465" 23:11:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305465" 23:11:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305465" 23:11:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713305465" 23:11:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305465" 23:11:05 INFO [http_cmd]: http status code OK 23:11:05 DEBUG --- status code --- 23:11:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:11:05 DEBUG --- http response --- 23:11:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:11:05 DEBUG ----- output ----- 23:11:05 DEBUG { 23:11:05 DEBUG "status": "success", 23:11:05 DEBUG "data": { 23:11:05 DEBUG "resultType": "vector", 23:11:05 DEBUG "result": [ 23:11:05 DEBUG { 23:11:05 DEBUG "metric": { 23:11:05 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 23:11:05 DEBUG }, 23:11:05 DEBUG "value": [ 23:11:05 DEBUG 23:11:05 DEBUG 23:11:05 DEBUG 23:11:05 DEBUG [...] 23:11:05 DEBUG 23:11:05 DEBUG 23:11:05 DEBUG 23:11:05 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 23:11:05 DEBUG }, 23:11:05 DEBUG "value": [ 23:11:05 DEBUG 1713305465, 23:11:05 DEBUG "517488640" 23:11:05 DEBUG ] 23:11:05 DEBUG } 23:11:05 DEBUG ] 23:11:05 DEBUG } 23:11:05 DEBUG } 23:11:05 INFO [http_cmd]: http status code OK 23:11:05 INFO [http_cmd]: http status code OK 23:11:05 DEBUG --- status code --- 23:11:05 INFO [http_cmd]: http status code OK 23:11:05 DEBUG --- status code --- 23:11:05 INFO [http_cmd]: http status code OK 23:11:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:11:05 DEBUG --- status code --- 23:11:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:11:05 DEBUG --- status code --- 23:11:05 DEBUG --- http response --- 23:11:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:11:05 DEBUG --- http response --- 23:11:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:11:05 DEBUG --- http response --- 23:11:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:11:05 DEBUG --- http response --- 23:11:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:11:05 DEBUG ----- output ----- 23:11:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:11:05 DEBUG ----- output ----- 23:11:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:11:05 DEBUG { 23:11:05 DEBUG ----- output ----- 23:11:05 DEBUG { 23:11:05 DEBUG ----- output ----- 23:11:05 DEBUG "status": "success", 23:11:05 DEBUG { 23:11:05 DEBUG "status": "success", 23:11:05 DEBUG { 23:11:05 DEBUG "data": { 23:11:05 DEBUG "status": "success", 23:11:05 DEBUG "data": { 23:11:05 DEBUG "status": "success", 23:11:05 DEBUG "resultType": "vector", 23:11:05 DEBUG "data": { 23:11:05 DEBUG "resultType": "vector", 23:11:05 DEBUG "data": { 23:11:05 DEBUG "result": [ 23:11:05 DEBUG "resultType": "vector", 23:11:05 DEBUG "result": [ 23:11:05 DEBUG "resultType": "vector", 23:11:05 DEBUG { 23:11:05 DEBUG "result": [ 23:11:05 DEBUG { 23:11:05 DEBUG "result": [ 23:11:05 DEBUG "metric": { 23:11:05 DEBUG { 23:11:05 DEBUG "metric": { 23:11:05 DEBUG { 23:11:05 DEBUG "pod": "am-555f4db595-fz8sn" 23:11:05 DEBUG "metric": { 23:11:05 DEBUG "instance": "10.142.0.107:9100" 23:11:05 DEBUG "metric": { 23:11:05 DEBUG }, 23:11:05 DEBUG "pod": "am-555f4db595-fz8sn" 23:11:05 DEBUG }, 23:11:05 DEBUG "pod": "ds-idrepo-2" 23:11:05 DEBUG "value": [ 23:11:05 DEBUG }, 23:11:05 DEBUG "value": [ 23:11:05 DEBUG }, 23:11:05 DEBUG 23:11:05 DEBUG "value": [ 23:11:05 DEBUG 23:11:05 DEBUG "value": [ 23:11:05 DEBUG 23:11:05 DEBUG 23:11:05 DEBUG 23:11:05 DEBUG 23:11:05 DEBUG 23:11:05 DEBUG 23:11:05 DEBUG 23:11:05 DEBUG 23:11:05 DEBUG [...] 23:11:05 DEBUG 23:11:05 DEBUG [...] 23:11:05 DEBUG 23:11:05 DEBUG 23:11:05 DEBUG [...] 23:11:05 DEBUG 23:11:05 DEBUG [...] 23:11:05 DEBUG 23:11:05 DEBUG 23:11:05 DEBUG 23:11:05 DEBUG 23:11:05 DEBUG 23:11:05 DEBUG 23:11:05 DEBUG 23:11:05 DEBUG 23:11:05 DEBUG "pod": "am-555f4db595-kzxsp" 23:11:05 DEBUG 23:11:05 DEBUG "instance": "10.142.0.43:9100" 23:11:05 DEBUG 23:11:05 DEBUG }, 23:11:05 DEBUG "pod": "am-555f4db595-kzxsp" 23:11:05 DEBUG }, 23:11:05 DEBUG "pod": "ds-idrepo-0" 23:11:05 DEBUG "value": [ 23:11:05 DEBUG }, 23:11:05 DEBUG "value": [ 23:11:05 DEBUG }, 23:11:05 DEBUG 1713305465, 23:11:05 DEBUG "value": [ 23:11:05 DEBUG 1713305465, 23:11:05 DEBUG "value": [ 23:11:05 DEBUG "0" 23:11:05 DEBUG 1713305465, 23:11:05 DEBUG "0.00036666666666652263" 23:11:05 DEBUG 1713305465, 23:11:05 DEBUG ] 23:11:05 DEBUG "0" 23:11:05 DEBUG ] 23:11:05 DEBUG "0" 23:11:05 DEBUG } 23:11:05 DEBUG ] 23:11:05 DEBUG } 23:11:05 DEBUG ] 23:11:05 DEBUG ] 23:11:05 DEBUG } 23:11:05 DEBUG ] 23:11:05 DEBUG } 23:11:05 DEBUG } 23:11:05 DEBUG ] 23:11:05 DEBUG } 23:11:05 DEBUG ] 23:11:05 DEBUG } 23:11:05 DEBUG } 23:11:05 DEBUG } 23:11:05 DEBUG } 23:11:05 DEBUG } 23:11:05 DEBUG } 23:11:05 INFO [loop_until]: OK (rc = 0) 23:11:05 DEBUG --- stdout --- 23:11:05 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 5100Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7501m 25% 15935Mi 14% gke-xlou-cdm-default-pool-1b0769c8-nhlf 1827m 6% 10800Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 83m 0% 2670Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 678m 2% 6591Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 8841m 29% 18016Mi 16% gke-xlou-cdm-default-pool-1b0769c8-xnmp 55m 0% 3167Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 1830m 6% 10506Mi 9% gke-xlou-cdm-ds-0f601751-hmm7 724m 2% 5643Mi 5% gke-xlou-cdm-ds-0f601751-j488 7851m 26% 15708Mi 14% gke-xlou-cdm-ds-0f601751-mc5d 85m 0% 3851Mi 3% gke-xlou-cdm-ds-0f601751-nj6q 1893m 6% 10461Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 890m 2% 7371Mi 6% gke-xlou-cdm-frontend-a8771548-i82n 1082m 6% 5925Mi 10% 23:11:05 DEBUG --- stderr --- 23:11:05 DEBUG 23:11:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713305465" 23:11:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305465" 23:11:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305465" 23:11:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305465" 23:11:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305465" 23:11:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305465" 23:11:05 INFO [http_cmd]: http status code OK 23:11:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305465" 23:11:05 INFO [http_cmd]: http status code OK 23:11:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713305465" 23:11:05 INFO [http_cmd]: http status code OK 23:11:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305465" 23:11:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713305465" 23:11:05 DEBUG --- status code --- 23:11:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713305465" 23:11:05 DEBUG --- status code --- 23:11:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305465" 23:11:05 DEBUG --- status code --- 23:11:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:11:05 INFO [http_cmd]: http status code OK 23:11:05 INFO [http_cmd]: http status code OK 23:11:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:11:05 INFO [http_cmd]: http status code OK 23:11:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:11:05 DEBUG --- http response --- 23:11:05 DEBUG --- status code --- 23:11:05 INFO [http_cmd]: http status code OK 23:11:05 DEBUG --- status code --- 23:11:05 DEBUG --- http response --- 23:11:05 DEBUG --- status code --- 23:11:05 DEBUG --- http response --- 23:11:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:11:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:11:05 DEBUG --- status code --- 23:11:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:11:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:11:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:11:05 DEBUG --- http response --- 23:11:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:11:05 DEBUG ----- output ----- 23:11:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:11:05 DEBUG --- http response --- 23:11:05 DEBUG --- http response --- 23:11:05 DEBUG ----- output ----- 23:11:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 23:11:05 DEBUG ----- output ----- 23:11:05 DEBUG { 23:11:05 DEBUG --- http response --- 23:11:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:11:05 DEBUG { 23:11:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:11:05 DEBUG ----- output ----- 23:11:05 DEBUG { 23:11:05 INFO [http_cmd]: http status code OK 23:11:05 INFO [http_cmd]: http status code OK 23:11:05 DEBUG "status": "success", 23:11:05 INFO [http_cmd]: http status code OK 23:11:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 23:11:05 DEBUG ----- output ----- 23:11:05 DEBUG "status": "success", 23:11:05 DEBUG ----- output ----- 23:11:05 DEBUG { 23:11:05 DEBUG "status": "success", 23:11:05 DEBUG --- status code --- 23:11:05 DEBUG --- status code --- 23:11:05 DEBUG "data": { 23:11:05 DEBUG --- status code --- 23:11:05 DEBUG ----- output ----- 23:11:05 DEBUG { 23:11:05 DEBUG "data": { 23:11:05 DEBUG { 23:11:05 DEBUG "status": "success", 23:11:05 DEBUG "data": { 23:11:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:11:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:11:05 DEBUG "resultType": "vector", 23:11:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:11:05 DEBUG { 23:11:05 DEBUG "status": "success", 23:11:05 DEBUG "resultType": "vector", 23:11:05 DEBUG "status": "success", 23:11:05 DEBUG "data": { 23:11:05 DEBUG "resultType": "vector", 23:11:05 DEBUG --- http response --- 23:11:05 DEBUG --- http response --- 23:11:05 DEBUG "result": [ 23:11:05 DEBUG --- http response --- 23:11:05 DEBUG "status": "success", 23:11:05 DEBUG "data": { 23:11:05 DEBUG "result": [ 23:11:05 DEBUG "data": { 23:11:05 DEBUG "resultType": "vector", 23:11:05 DEBUG "result": [ 23:11:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:11:05 DEBUG { 23:11:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 23:11:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 23:11:05 DEBUG "data": { 23:11:05 INFO [http_cmd]: http status code OK 23:11:05 INFO [http_cmd]: http status code OK 23:11:05 DEBUG "resultType": "vector", 23:11:05 DEBUG { 23:11:05 DEBUG "resultType": "vector", 23:11:05 DEBUG "result": [ 23:11:05 DEBUG { 23:11:05 DEBUG ----- output ----- 23:11:05 DEBUG "metric": { 23:11:05 DEBUG ----- output ----- 23:11:05 DEBUG ----- output ----- 23:11:05 DEBUG "resultType": "vector", 23:11:05 DEBUG --- status code --- 23:11:05 DEBUG --- status code --- 23:11:05 DEBUG "result": [ 23:11:05 DEBUG "metric": { 23:11:05 DEBUG "result": [ 23:11:05 DEBUG { 23:11:05 DEBUG "metric": { 23:11:05 DEBUG { 23:11:05 DEBUG "pod": "am-555f4db595-fz8sn" 23:11:05 DEBUG { 23:11:05 DEBUG { 23:11:05 DEBUG "result": [ 23:11:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:11:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:11:05 DEBUG { 23:11:05 DEBUG "pod": "am-555f4db595-fz8sn" 23:11:05 DEBUG { 23:11:05 DEBUG "metric": { 23:11:05 DEBUG "instance": "10.142.0.107:9100" 23:11:05 DEBUG "status": "success", 23:11:05 DEBUG }, 23:11:05 DEBUG "status": "success", 23:11:05 DEBUG "status": "success", 23:11:05 DEBUG { 23:11:05 DEBUG --- http response --- 23:11:05 DEBUG --- http response --- 23:11:05 DEBUG "metric": { 23:11:05 DEBUG }, 23:11:05 DEBUG "metric": { 23:11:05 DEBUG "pod": "ds-cts-1" 23:11:05 DEBUG }, 23:11:05 DEBUG "data": { 23:11:05 DEBUG "value": [ 23:11:05 DEBUG "data": { 23:11:05 DEBUG "data": { 23:11:05 DEBUG "metric": { 23:11:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 23:11:05 DEBUG "pod": "ds-idrepo-2" 23:11:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:11:05 DEBUG "value": [ 23:11:05 DEBUG "pod": "am-555f4db595-fz8sn" 23:11:05 DEBUG }, 23:11:05 DEBUG "value": [ 23:11:05 DEBUG "resultType": "vector", 23:11:05 DEBUG 23:11:05 DEBUG "resultType": "vector", 23:11:05 DEBUG "resultType": "vector", 23:11:05 DEBUG "pod": "ds-idrepo-2" 23:11:05 DEBUG ----- output ----- 23:11:05 DEBUG }, 23:11:05 DEBUG ----- output ----- 23:11:05 DEBUG 23:11:05 DEBUG }, 23:11:05 DEBUG "value": [ 23:11:05 DEBUG 23:11:05 DEBUG "result": [ 23:11:05 DEBUG 23:11:05 DEBUG "result": [ 23:11:05 DEBUG "result": [ 23:11:05 DEBUG }, 23:11:05 DEBUG { 23:11:05 DEBUG "value": [ 23:11:05 DEBUG { 23:11:05 DEBUG 23:11:05 DEBUG "value": [ 23:11:05 DEBUG 23:11:05 DEBUG 23:11:05 DEBUG { 23:11:05 DEBUG 23:11:05 DEBUG { 23:11:05 DEBUG { 23:11:05 DEBUG "value": [ 23:11:05 DEBUG "status": "success", 23:11:05 DEBUG 23:11:05 DEBUG "status": "success", 23:11:05 DEBUG 23:11:05 DEBUG 23:11:05 DEBUG 23:11:05 DEBUG 23:11:05 DEBUG "metric": { 23:11:05 DEBUG [...] 23:11:05 DEBUG "metric": { 23:11:05 DEBUG "metric": { 23:11:05 DEBUG 23:11:05 DEBUG "data": { 23:11:05 DEBUG 23:11:05 DEBUG "data": { 23:11:05 DEBUG [...] 23:11:05 DEBUG 23:11:05 DEBUG 23:11:05 DEBUG [...] 23:11:05 DEBUG "pod": "am-555f4db595-fz8sn" 23:11:05 DEBUG 23:11:05 DEBUG "pod": "ds-idrepo-2" 23:11:05 DEBUG "pod": "ds-idrepo-2" 23:11:05 DEBUG 23:11:05 DEBUG "resultType": "vector", 23:11:05 DEBUG 23:11:05 DEBUG "resultType": "vector", 23:11:05 DEBUG 23:11:05 DEBUG 23:11:05 DEBUG [...] 23:11:05 DEBUG 23:11:05 DEBUG }, 23:11:05 DEBUG 23:11:05 DEBUG }, 23:11:05 DEBUG }, 23:11:05 DEBUG 23:11:05 DEBUG "result": [ 23:11:05 DEBUG [...] 23:11:05 DEBUG "result": [ 23:11:05 DEBUG 23:11:05 DEBUG [...] 23:11:05 DEBUG 23:11:05 DEBUG 23:11:05 DEBUG "value": [ 23:11:05 DEBUG 23:11:05 DEBUG "value": [ 23:11:05 DEBUG "value": [ 23:11:05 DEBUG [...] 23:11:05 DEBUG { 23:11:05 DEBUG 23:11:05 DEBUG { 23:11:05 DEBUG 23:11:05 DEBUG 23:11:05 DEBUG 23:11:05 DEBUG 23:11:05 DEBUG 23:11:05 DEBUG "pod": "am-555f4db595-kzxsp" 23:11:05 DEBUG 23:11:05 DEBUG 23:11:05 DEBUG 23:11:05 DEBUG "metric": { 23:11:05 DEBUG 23:11:05 DEBUG "metric": { 23:11:05 DEBUG "pod": "am-555f4db595-kzxsp" 23:11:05 DEBUG 23:11:05 DEBUG 23:11:05 DEBUG "pod": "lodemon-bc756c647-cr7x6" 23:11:05 DEBUG 23:11:05 DEBUG }, 23:11:05 DEBUG 23:11:05 DEBUG 23:11:05 DEBUG 23:11:05 DEBUG "pod": "ds-idrepo-2" 23:11:05 DEBUG 23:11:05 DEBUG "instance": "10.142.0.107:9100" 23:11:05 DEBUG }, 23:11:05 DEBUG 23:11:05 DEBUG "instance": "10.142.0.43:9100" 23:11:05 DEBUG }, 23:11:05 DEBUG 23:11:05 DEBUG "value": [ 23:11:05 DEBUG 23:11:05 DEBUG 23:11:05 DEBUG 23:11:05 DEBUG }, 23:11:05 DEBUG "pod": "ds-idrepo-0" 23:11:05 DEBUG }, 23:11:05 DEBUG "value": [ 23:11:05 DEBUG "pod": "am-555f4db595-kzxsp" 23:11:05 DEBUG }, 23:11:05 DEBUG "value": [ 23:11:05 DEBUG [...] 23:11:05 DEBUG 1713305465, 23:11:05 DEBUG [...] 23:11:05 DEBUG [...] 23:11:05 DEBUG "pod": "ds-idrepo-0" 23:11:05 DEBUG "value": [ 23:11:05 DEBUG }, 23:11:05 DEBUG "value": [ 23:11:05 DEBUG 1713305465, 23:11:05 DEBUG }, 23:11:05 DEBUG "value": [ 23:11:05 DEBUG 1713305465, 23:11:05 DEBUG 23:11:05 DEBUG "3295223.5" 23:11:05 DEBUG 23:11:05 DEBUG 23:11:05 DEBUG }, 23:11:05 DEBUG 23:11:05 DEBUG "value": [ 23:11:05 DEBUG 23:11:05 DEBUG "0" 23:11:05 DEBUG "value": [ 23:11:05 DEBUG 1713305465, 23:11:05 DEBUG "126.45193645551429" 23:11:05 DEBUG 23:11:05 DEBUG ] 23:11:05 DEBUG 23:11:05 DEBUG 23:11:05 DEBUG "value": [ 23:11:05 DEBUG 23:11:05 DEBUG 1713305465, 23:11:05 DEBUG 23:11:05 DEBUG ] 23:11:05 DEBUG 1713305465, 23:11:05 DEBUG "19988333568" 23:11:05 DEBUG ] 23:11:05 DEBUG 23:11:05 DEBUG } 23:11:05 DEBUG 23:11:05 DEBUG 23:11:05 DEBUG 1713305465, 23:11:05 DEBUG 23:11:05 DEBUG "0" 23:11:05 DEBUG 23:11:05 DEBUG } 23:11:05 DEBUG "0" 23:11:05 DEBUG ] 23:11:05 DEBUG } 23:11:05 DEBUG "pod": "am-555f4db595-kzxsp" 23:11:05 DEBUG ] 23:11:05 DEBUG "pod": "ds-cts-2" 23:11:05 DEBUG "pod": "ds-idrepo-0" 23:11:05 DEBUG "0" 23:11:05 DEBUG [...] 23:11:05 DEBUG ] 23:11:05 DEBUG [...] 23:11:05 DEBUG ] 23:11:05 DEBUG ] 23:11:05 DEBUG } 23:11:05 DEBUG ] 23:11:05 DEBUG }, 23:11:05 DEBUG } 23:11:05 DEBUG }, 23:11:05 DEBUG }, 23:11:05 DEBUG ] 23:11:05 DEBUG 23:11:05 DEBUG } 23:11:05 DEBUG 23:11:05 DEBUG } 23:11:05 DEBUG } 23:11:05 DEBUG ] 23:11:05 DEBUG } 23:11:05 DEBUG "value": [ 23:11:05 DEBUG } 23:11:05 DEBUG "value": [ 23:11:05 DEBUG "value": [ 23:11:05 DEBUG } 23:11:05 DEBUG 23:11:05 DEBUG ] 23:11:05 DEBUG 23:11:05 DEBUG } 23:11:05 DEBUG ] 23:11:05 DEBUG } 23:11:05 DEBUG } 23:11:05 DEBUG 1713305465, 23:11:05 DEBUG 1713305465, 23:11:05 DEBUG 1713305465, 23:11:05 DEBUG ] 23:11:05 DEBUG 23:11:05 DEBUG } 23:11:05 DEBUG 23:11:05 DEBUG } 23:11:05 DEBUG } 23:11:05 DEBUG "0.0285" 23:11:05 DEBUG "0" 23:11:05 DEBUG "0" 23:11:05 DEBUG } 23:11:05 DEBUG "pod": "ds-idrepo-0" 23:11:05 DEBUG } 23:11:05 DEBUG "instance": "10.142.0.43:9100" 23:11:05 DEBUG } 23:11:05 DEBUG ] 23:11:05 DEBUG ] 23:11:05 DEBUG ] 23:11:05 DEBUG } 23:11:05 DEBUG }, 23:11:05 DEBUG }, 23:11:05 DEBUG } 23:11:05 DEBUG } 23:11:05 DEBUG } 23:11:05 DEBUG "value": [ 23:11:05 DEBUG "value": [ 23:11:05 DEBUG ] 23:11:05 DEBUG ] 23:11:05 DEBUG ] 23:11:05 DEBUG 1713305465, 23:11:05 DEBUG 1713305465, 23:11:05 DEBUG } 23:11:05 DEBUG } 23:11:05 DEBUG } 23:11:05 DEBUG "0" 23:11:05 DEBUG "16951.23333333333" 23:11:05 DEBUG } 23:11:05 DEBUG } 23:11:05 DEBUG } 23:11:05 DEBUG ] 23:11:05 DEBUG ] 23:11:05 DEBUG } 23:11:05 DEBUG } 23:11:05 DEBUG ] 23:11:05 DEBUG ] 23:11:05 DEBUG } 23:11:05 DEBUG } 23:11:05 DEBUG } 23:11:05 DEBUG } 23:11:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713305519" 23:11:59 ERROR [http_cmd]: ERROR 23:11:59 ERROR --- status code --- 23:11:59 ERROR http status code is 400 (expected [200, 401, 403, 404]) 23:11:59 ERROR --- http response --- 23:11:59 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 23:11:59 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 23:11:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305519" 23:11:59 INFO [http_cmd]: http status code OK 23:11:59 DEBUG --- status code --- 23:11:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:11:59 DEBUG --- http response --- 23:11:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 23:12:02 INFO 23:12:02 INFO [loop_until]: kubectl --namespace=xlou top pods 23:12:02 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 23:12:02 INFO [loop_until]: OK (rc = 0) 23:12:02 DEBUG --- stdout --- 23:12:02 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 9Mi am-555f4db595-4lbss 1652m 8659Mi am-555f4db595-fz8sn 1782m 8535Mi am-555f4db595-kzxsp 1687m 8558Mi ds-cts-0 8180m 12893Mi ds-cts-1 5829m 13144Mi ds-cts-2 7459m 12932Mi ds-idrepo-0 537m 5043Mi ds-idrepo-1 479m 2913Mi ds-idrepo-2 538m 3239Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 4m 1208Mi idm-b85f9d978-rl76f 4m 1227Mi lodemon-bc756c647-cr7x6 2m 64Mi login-ui-5fdf49fc66-2wnx7 1m 5Mi overseer-0-7c9b667974-9mr2s 1005m 856Mi 23:12:02 DEBUG --- stderr --- 23:12:02 DEBUG 23:12:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305525" 23:12:05 INFO [http_cmd]: http status code OK 23:12:05 DEBUG --- status code --- 23:12:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:12:05 DEBUG --- http response --- 23:12:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 23:12:05 DEBUG ----- output ----- 23:12:05 DEBUG { 23:12:05 DEBUG "status": "success", 23:12:05 DEBUG "data": { 23:12:05 DEBUG "resultType": "vector", 23:12:05 DEBUG "result": [ 23:12:05 DEBUG { 23:12:05 DEBUG "metric": { 23:12:05 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 23:12:05 DEBUG }, 23:12:05 DEBUG "value": [ 23:12:05 DEBUG 23:12:05 DEBUG 23:12:05 DEBUG 23:12:05 DEBUG [...] 23:12:05 DEBUG 23:12:05 DEBUG 23:12:05 DEBUG 23:12:05 DEBUG "pod": "ds-cts-2" 23:12:05 DEBUG }, 23:12:05 DEBUG "value": [ 23:12:05 DEBUG 1713305525, 23:12:05 DEBUG "0" 23:12:05 DEBUG ] 23:12:05 DEBUG } 23:12:05 DEBUG ] 23:12:05 DEBUG } 23:12:05 DEBUG } 23:12:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305525" 23:12:05 INFO [http_cmd]: http status code OK 23:12:05 DEBUG --- status code --- 23:12:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:12:05 DEBUG --- http response --- 23:12:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:12:05 DEBUG ----- output ----- 23:12:05 DEBUG { 23:12:05 DEBUG "status": "success", 23:12:05 DEBUG "data": { 23:12:05 DEBUG "resultType": "vector", 23:12:05 DEBUG "result": [ 23:12:05 DEBUG { 23:12:05 DEBUG "metric": { 23:12:05 DEBUG "pod": "am-555f4db595-fz8sn" 23:12:05 DEBUG }, 23:12:05 DEBUG "value": [ 23:12:05 DEBUG 23:12:05 DEBUG 23:12:05 DEBUG 23:12:05 DEBUG [...] 23:12:05 DEBUG 23:12:05 DEBUG 23:12:05 DEBUG 23:12:05 DEBUG "pod": "am-555f4db595-kzxsp" 23:12:05 DEBUG }, 23:12:05 DEBUG "value": [ 23:12:05 DEBUG 1713305525, 23:12:05 DEBUG "1320.2666666666667" 23:12:05 DEBUG ] 23:12:05 DEBUG } 23:12:05 DEBUG ] 23:12:05 DEBUG } 23:12:05 DEBUG } 23:12:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305525" 23:12:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713305525" 23:12:05 INFO [http_cmd]: http status code OK 23:12:05 DEBUG --- status code --- 23:12:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:12:05 DEBUG --- http response --- 23:12:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 23:12:05 DEBUG ----- output ----- 23:12:05 DEBUG { 23:12:05 DEBUG "status": "success", 23:12:05 DEBUG "data": { 23:12:05 DEBUG "resultType": "vector", 23:12:05 DEBUG "result": [ 23:12:05 DEBUG { 23:12:05 DEBUG "metric": { 23:12:05 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 23:12:05 DEBUG }, 23:12:05 DEBUG "value": [ 23:12:05 DEBUG 23:12:05 DEBUG 23:12:05 DEBUG 23:12:05 DEBUG [...] 23:12:05 DEBUG 23:12:05 DEBUG 23:12:05 DEBUG 23:12:05 DEBUG "pod": "am-555f4db595-4lbss" 23:12:05 DEBUG }, 23:12:05 DEBUG "value": [ 23:12:05 DEBUG 1713305525, 23:12:05 DEBUG "0" 23:12:05 DEBUG ] 23:12:05 DEBUG } 23:12:05 DEBUG ] 23:12:05 DEBUG } 23:12:05 DEBUG } 23:12:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305525" 23:12:05 INFO [http_cmd]: http status code OK 23:12:05 DEBUG --- status code --- 23:12:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:12:05 DEBUG --- http response --- 23:12:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:12:05 DEBUG ----- output ----- 23:12:05 DEBUG { 23:12:05 DEBUG "status": "success", 23:12:05 DEBUG "data": { 23:12:05 DEBUG "resultType": "vector", 23:12:05 DEBUG "result": [ 23:12:05 DEBUG { 23:12:05 DEBUG "metric": { 23:12:05 DEBUG "instance": "10.142.0.107:9100" 23:12:05 DEBUG }, 23:12:05 DEBUG "value": [ 23:12:05 DEBUG 23:12:05 DEBUG 23:12:05 DEBUG 23:12:05 DEBUG [...] 23:12:05 DEBUG 23:12:05 DEBUG 23:12:05 DEBUG 23:12:05 DEBUG "instance": "10.142.0.43:9100" 23:12:05 DEBUG }, 23:12:05 DEBUG "value": [ 23:12:05 DEBUG 1713305525, 23:12:05 DEBUG "6724.966666666667" 23:12:05 DEBUG ] 23:12:05 DEBUG } 23:12:05 DEBUG ] 23:12:05 DEBUG } 23:12:05 DEBUG } 23:12:05 INFO [http_cmd]: http status code OK 23:12:05 DEBUG --- status code --- 23:12:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:12:05 DEBUG --- http response --- 23:12:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 23:12:05 DEBUG ----- output ----- 23:12:05 DEBUG { 23:12:05 DEBUG "status": "success", 23:12:05 DEBUG "data": { 23:12:05 DEBUG "resultType": "vector", 23:12:05 DEBUG "result": [ 23:12:05 DEBUG { 23:12:05 DEBUG "metric": { 23:12:05 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 23:12:05 DEBUG }, 23:12:05 DEBUG "value": [ 23:12:05 DEBUG 23:12:05 DEBUG 23:12:05 DEBUG 23:12:05 DEBUG [...] 23:12:05 DEBUG 23:12:05 DEBUG 23:12:05 DEBUG 23:12:05 DEBUG "pod": "ds-cts-2" 23:12:05 DEBUG }, 23:12:05 DEBUG "value": [ 23:12:05 DEBUG 1713305525, 23:12:05 DEBUG "0" 23:12:05 DEBUG ] 23:12:05 DEBUG } 23:12:05 DEBUG ] 23:12:05 DEBUG } 23:12:05 DEBUG } 23:12:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713305525" 23:12:05 INFO [http_cmd]: http status code OK 23:12:05 DEBUG --- status code --- 23:12:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:12:05 DEBUG --- http response --- 23:12:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:12:05 DEBUG ----- output ----- 23:12:05 DEBUG { 23:12:05 DEBUG "status": "success", 23:12:05 DEBUG "data": { 23:12:05 DEBUG "resultType": "vector", 23:12:05 DEBUG "result": [ 23:12:05 DEBUG { 23:12:05 DEBUG "metric": { 23:12:05 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 23:12:05 DEBUG }, 23:12:05 DEBUG "value": [ 23:12:05 DEBUG 23:12:05 DEBUG 23:12:05 DEBUG 23:12:05 DEBUG [...] 23:12:05 DEBUG 23:12:05 DEBUG 23:12:05 DEBUG 23:12:05 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 23:12:05 DEBUG }, 23:12:05 DEBUG "value": [ 23:12:05 DEBUG 1713305525, 23:12:05 DEBUG "1.0235870429378764" 23:12:05 DEBUG ] 23:12:05 DEBUG } 23:12:05 DEBUG ] 23:12:05 DEBUG } 23:12:05 DEBUG } 23:12:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713305525" 23:12:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713305525" 23:12:05 INFO [http_cmd]: http status code OK 23:12:05 DEBUG --- status code --- 23:12:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:12:05 DEBUG --- http response --- 23:12:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:12:05 DEBUG ----- output ----- 23:12:05 DEBUG { 23:12:05 DEBUG "status": "success", 23:12:05 DEBUG "data": { 23:12:05 DEBUG "resultType": "vector", 23:12:05 DEBUG "result": [ 23:12:05 DEBUG { 23:12:05 DEBUG "metric": { 23:12:05 DEBUG "instance": "10.142.0.107:9100" 23:12:05 DEBUG }, 23:12:05 DEBUG "value": [ 23:12:05 DEBUG 23:12:05 DEBUG 23:12:05 DEBUG 23:12:05 DEBUG [...] 23:12:05 DEBUG 23:12:05 DEBUG 23:12:05 DEBUG 23:12:05 DEBUG "instance": "10.142.0.43:9100" 23:12:05 DEBUG }, 23:12:05 DEBUG "value": [ 23:12:05 DEBUG 1713305525, 23:12:05 DEBUG "0.0003333333333333262" 23:12:05 DEBUG ] 23:12:05 DEBUG } 23:12:05 DEBUG ] 23:12:05 DEBUG } 23:12:05 DEBUG } 23:12:05 INFO [http_cmd]: http status code OK 23:12:05 DEBUG --- status code --- 23:12:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:12:05 DEBUG --- http response --- 23:12:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:12:05 DEBUG ----- output ----- 23:12:05 DEBUG { 23:12:05 DEBUG "status": "success", 23:12:05 DEBUG "data": { 23:12:05 DEBUG "resultType": "vector", 23:12:05 DEBUG "result": [ 23:12:05 DEBUG { 23:12:05 DEBUG "metric": { 23:12:05 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 23:12:05 DEBUG }, 23:12:05 DEBUG "value": [ 23:12:05 DEBUG 23:12:05 DEBUG 23:12:05 DEBUG 23:12:05 DEBUG [...] 23:12:05 DEBUG 23:12:05 DEBUG 23:12:05 DEBUG 23:12:05 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 23:12:05 DEBUG }, 23:12:05 DEBUG "value": [ 23:12:05 DEBUG 1713305525, 23:12:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305525" 23:12:05 DEBUG "790466560" 23:12:05 DEBUG ] 23:12:05 DEBUG } 23:12:05 DEBUG ] 23:12:05 DEBUG } 23:12:05 DEBUG } 23:12:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305525" 23:12:05 INFO [http_cmd]: http status code OK 23:12:05 DEBUG --- status code --- 23:12:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:12:05 DEBUG --- http response --- 23:12:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:12:05 DEBUG ----- output ----- 23:12:05 DEBUG { 23:12:05 DEBUG "status": "success", 23:12:05 DEBUG "data": { 23:12:05 DEBUG "resultType": "vector", 23:12:05 DEBUG "result": [ 23:12:05 DEBUG { 23:12:05 DEBUG "metric": { 23:12:05 DEBUG "pod": "am-555f4db595-fz8sn" 23:12:05 DEBUG }, 23:12:05 DEBUG "value": [ 23:12:05 DEBUG 23:12:05 DEBUG 23:12:05 DEBUG 23:12:05 DEBUG [...] 23:12:05 DEBUG 23:12:05 DEBUG 23:12:05 DEBUG 23:12:05 DEBUG "pod": "am-555f4db595-kzxsp" 23:12:05 DEBUG }, 23:12:05 DEBUG "value": [ 23:12:05 DEBUG 1713305525, 23:12:05 DEBUG "0" 23:12:05 DEBUG ] 23:12:05 DEBUG } 23:12:05 DEBUG ] 23:12:05 DEBUG } 23:12:05 DEBUG } 23:12:05 INFO 23:12:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713305525" 23:12:05 INFO [loop_until]: kubectl --namespace=xlou top node 23:12:05 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 23:12:05 INFO [http_cmd]: http status code OK 23:12:05 DEBUG --- status code --- 23:12:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:12:05 DEBUG --- http response --- 23:12:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:12:05 DEBUG ----- output ----- 23:12:05 DEBUG { 23:12:05 DEBUG "status": "success", 23:12:05 DEBUG "data": { 23:12:05 DEBUG "resultType": "vector", 23:12:05 DEBUG "result": [ 23:12:05 DEBUG { 23:12:05 DEBUG "metric": { 23:12:05 DEBUG "pod": "ds-idrepo-2" 23:12:05 DEBUG }, 23:12:05 DEBUG "value": [ 23:12:05 DEBUG 23:12:05 DEBUG 23:12:05 DEBUG 23:12:05 DEBUG [...] 23:12:05 DEBUG 23:12:05 DEBUG 23:12:05 DEBUG 23:12:05 DEBUG "pod": "ds-idrepo-0" 23:12:05 DEBUG }, 23:12:05 DEBUG "value": [ 23:12:05 DEBUG 1713305525, 23:12:05 DEBUG "0" 23:12:05 DEBUG ] 23:12:05 DEBUG } 23:12:05 DEBUG ] 23:12:05 DEBUG } 23:12:05 DEBUG } 23:12:05 INFO [http_cmd]: http status code OK 23:12:05 DEBUG --- status code --- 23:12:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:12:05 DEBUG --- http response --- 23:12:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:12:05 DEBUG ----- output ----- 23:12:05 DEBUG { 23:12:05 DEBUG "status": "success", 23:12:05 DEBUG "data": { 23:12:05 DEBUG "resultType": "vector", 23:12:05 DEBUG "result": [ 23:12:05 DEBUG { 23:12:05 DEBUG "metric": { 23:12:05 DEBUG "instance": "10.142.0.107:9100" 23:12:05 DEBUG }, 23:12:05 DEBUG "value": [ 23:12:05 DEBUG 23:12:05 DEBUG 23:12:05 DEBUG 23:12:05 DEBUG [...] 23:12:05 DEBUG 23:12:05 DEBUG 23:12:05 DEBUG 23:12:05 DEBUG "instance": "10.142.0.43:9100" 23:12:05 DEBUG }, 23:12:05 DEBUG "value": [ 23:12:05 DEBUG 1713305525, 23:12:05 DEBUG "0.0006333333333335152" 23:12:05 DEBUG ] 23:12:05 DEBUG } 23:12:05 DEBUG ] 23:12:05 DEBUG } 23:12:05 DEBUG } 23:12:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305525" 23:12:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305525" 23:12:05 INFO [http_cmd]: http status code OK 23:12:05 INFO [http_cmd]: http status code OK 23:12:05 DEBUG --- status code --- 23:12:05 DEBUG --- status code --- 23:12:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:12:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:12:05 DEBUG --- http response --- 23:12:05 DEBUG --- http response --- 23:12:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:12:05 DEBUG ----- output ----- 23:12:05 DEBUG { 23:12:05 DEBUG "status": "success", 23:12:05 DEBUG "data": { 23:12:05 DEBUG "resultType": "vector", 23:12:05 DEBUG "result": [ 23:12:05 DEBUG { 23:12:05 DEBUG "metric": { 23:12:05 DEBUG "pod": "ds-idrepo-2" 23:12:05 DEBUG }, 23:12:05 DEBUG "value": [ 23:12:05 DEBUG 23:12:05 DEBUG 23:12:05 DEBUG 23:12:05 DEBUG [...] 23:12:05 DEBUG 23:12:05 DEBUG 23:12:05 DEBUG 23:12:05 DEBUG "pod": "ds-idrepo-0" 23:12:05 DEBUG }, 23:12:05 DEBUG "value": [ 23:12:05 DEBUG 1713305525, 23:12:05 DEBUG "0" 23:12:05 DEBUG ] 23:12:05 DEBUG } 23:12:05 DEBUG ] 23:12:05 DEBUG } 23:12:05 DEBUG } 23:12:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305525" 23:12:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713305525" 23:12:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:12:05 DEBUG ----- output ----- 23:12:05 DEBUG { 23:12:05 DEBUG "status": "success", 23:12:05 DEBUG "data": { 23:12:05 DEBUG "resultType": "vector", 23:12:05 DEBUG "result": [ 23:12:05 DEBUG { 23:12:05 DEBUG "metric": { 23:12:05 DEBUG "pod": "am-555f4db595-fz8sn" 23:12:05 DEBUG }, 23:12:05 DEBUG "value": [ 23:12:05 DEBUG 23:12:05 DEBUG 23:12:05 DEBUG 23:12:05 DEBUG [...] 23:12:05 DEBUG 23:12:05 DEBUG 23:12:05 DEBUG 23:12:05 DEBUG "pod": "am-555f4db595-kzxsp" 23:12:05 DEBUG }, 23:12:05 DEBUG "value": [ 23:12:05 DEBUG 1713305525, 23:12:05 DEBUG "0" 23:12:05 DEBUG ] 23:12:05 DEBUG } 23:12:05 DEBUG ] 23:12:05 DEBUG } 23:12:05 DEBUG } 23:12:05 INFO [http_cmd]: http status code OK 23:12:05 INFO [http_cmd]: http status code OK 23:12:05 DEBUG --- status code --- 23:12:05 DEBUG --- status code --- 23:12:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:12:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:12:05 DEBUG --- http response --- 23:12:05 DEBUG --- http response --- 23:12:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:12:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:12:05 DEBUG ----- output ----- 23:12:05 DEBUG ----- output ----- 23:12:05 DEBUG { 23:12:05 DEBUG { 23:12:05 DEBUG "status": "success", 23:12:05 DEBUG "status": "success", 23:12:05 DEBUG "data": { 23:12:05 DEBUG "data": { 23:12:05 DEBUG "resultType": "vector", 23:12:05 DEBUG "resultType": "vector", 23:12:05 DEBUG "result": [ 23:12:05 DEBUG "result": [ 23:12:05 DEBUG { 23:12:05 DEBUG { 23:12:05 DEBUG "metric": { 23:12:05 DEBUG "metric": { 23:12:05 DEBUG "pod": "am-555f4db595-fz8sn" 23:12:05 DEBUG "instance": "10.142.0.107:9100" 23:12:05 DEBUG }, 23:12:05 DEBUG }, 23:12:05 DEBUG "value": [ 23:12:05 DEBUG "value": [ 23:12:05 DEBUG 23:12:05 DEBUG 23:12:05 DEBUG 23:12:05 DEBUG 23:12:05 DEBUG 23:12:05 DEBUG 23:12:05 DEBUG [...] 23:12:05 DEBUG [...] 23:12:05 DEBUG 23:12:05 DEBUG 23:12:05 DEBUG 23:12:05 DEBUG 23:12:05 DEBUG 23:12:05 DEBUG 23:12:05 DEBUG "pod": "am-555f4db595-kzxsp" 23:12:05 DEBUG "instance": "10.142.0.43:9100" 23:12:05 DEBUG }, 23:12:05 DEBUG }, 23:12:05 DEBUG "value": [ 23:12:05 DEBUG "value": [ 23:12:05 DEBUG 1713305525, 23:12:05 DEBUG 1713305525, 23:12:05 DEBUG "0" 23:12:05 DEBUG "19989246976" 23:12:05 DEBUG ] 23:12:05 DEBUG ] 23:12:05 DEBUG } 23:12:05 DEBUG } 23:12:05 DEBUG ] 23:12:05 DEBUG ] 23:12:05 DEBUG } 23:12:05 DEBUG } 23:12:05 DEBUG } 23:12:05 DEBUG } 23:12:05 INFO [loop_until]: OK (rc = 0) 23:12:05 DEBUG --- stdout --- 23:12:05 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 5081Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 6727m 22% 15724Mi 14% gke-xlou-cdm-default-pool-1b0769c8-nhlf 1649m 5% 10809Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 83m 0% 2656Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 503m 1% 6611Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 7191m 24% 17057Mi 15% gke-xlou-cdm-default-pool-1b0769c8-xnmp 59m 0% 3165Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 1406m 4% 10489Mi 9% gke-xlou-cdm-ds-0f601751-hmm7 597m 1% 5647Mi 5% gke-xlou-cdm-ds-0f601751-j488 5363m 17% 14626Mi 13% gke-xlou-cdm-ds-0f601751-mc5d 83m 0% 3778Mi 3% gke-xlou-cdm-ds-0f601751-nj6q 1240m 4% 10457Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 509m 1% 7393Mi 6% gke-xlou-cdm-frontend-a8771548-i82n 1082m 6% 6237Mi 10% 23:12:05 DEBUG --- stderr --- 23:12:05 DEBUG 23:12:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713305525" 23:12:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713305525" 23:12:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305525" 23:12:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305525" 23:12:05 INFO [http_cmd]: http status code OK 23:12:05 DEBUG --- status code --- 23:12:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:12:05 DEBUG --- http response --- 23:12:05 INFO [http_cmd]: http status code OK 23:12:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 23:12:05 DEBUG --- status code --- 23:12:05 DEBUG ----- output ----- 23:12:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:12:05 DEBUG { 23:12:05 DEBUG --- http response --- 23:12:05 DEBUG "status": "success", 23:12:05 DEBUG "data": { 23:12:05 DEBUG "resultType": "vector", 23:12:05 DEBUG "result": [ 23:12:05 DEBUG { 23:12:05 DEBUG "metric": { 23:12:05 DEBUG "pod": "ds-idrepo-2" 23:12:05 DEBUG }, 23:12:05 DEBUG "value": [ 23:12:05 DEBUG 23:12:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:12:05 DEBUG 23:12:05 DEBUG ----- output ----- 23:12:05 DEBUG 23:12:05 DEBUG { 23:12:05 DEBUG [...] 23:12:05 DEBUG "status": "success", 23:12:05 DEBUG 23:12:05 DEBUG "data": { 23:12:05 DEBUG 23:12:05 DEBUG "resultType": "vector", 23:12:05 DEBUG 23:12:05 DEBUG "result": [ 23:12:05 DEBUG "pod": "ds-cts-2" 23:12:05 DEBUG { 23:12:05 DEBUG }, 23:12:05 DEBUG "metric": { 23:12:05 DEBUG "value": [ 23:12:05 DEBUG "instance": "10.142.0.107:9100" 23:12:05 DEBUG 1713305525, 23:12:05 DEBUG }, 23:12:05 DEBUG "0" 23:12:05 DEBUG "value": [ 23:12:05 DEBUG ] 23:12:05 DEBUG 23:12:05 DEBUG } 23:12:05 DEBUG 23:12:05 DEBUG ] 23:12:05 DEBUG 23:12:05 DEBUG } 23:12:05 DEBUG [...] 23:12:05 DEBUG } 23:12:05 DEBUG 23:12:05 DEBUG 23:12:05 DEBUG 23:12:05 DEBUG "instance": "10.142.0.43:9100" 23:12:05 DEBUG }, 23:12:05 DEBUG "value": [ 23:12:05 DEBUG 1713305525, 23:12:05 DEBUG "16615.13333333333" 23:12:05 DEBUG ] 23:12:05 DEBUG } 23:12:05 DEBUG ] 23:12:05 DEBUG } 23:12:05 DEBUG } 23:12:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305525" 23:12:05 INFO [http_cmd]: http status code OK 23:12:05 DEBUG --- status code --- 23:12:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:12:05 DEBUG --- http response --- 23:12:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 23:12:05 DEBUG ----- output ----- 23:12:05 DEBUG { 23:12:05 DEBUG "status": "success", 23:12:05 DEBUG "data": { 23:12:05 DEBUG "resultType": "vector", 23:12:05 DEBUG "result": [ 23:12:05 DEBUG { 23:12:05 DEBUG "metric": { 23:12:05 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 23:12:05 DEBUG }, 23:12:05 DEBUG "value": [ 23:12:05 DEBUG 23:12:05 DEBUG 23:12:05 DEBUG 23:12:05 DEBUG [...] 23:12:05 DEBUG 23:12:05 DEBUG 23:12:05 DEBUG 23:12:05 DEBUG "pod": "am-555f4db595-4lbss" 23:12:05 DEBUG }, 23:12:05 DEBUG "value": [ 23:12:05 DEBUG 1713305525, 23:12:05 DEBUG "0" 23:12:05 DEBUG ] 23:12:05 DEBUG } 23:12:05 DEBUG ] 23:12:05 DEBUG } 23:12:05 DEBUG } 23:12:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305525" 23:12:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713305525" 23:12:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305525" 23:12:05 INFO [http_cmd]: http status code OK 23:12:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305525" 23:12:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305525" 23:12:05 DEBUG --- status code --- 23:12:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:12:05 DEBUG --- http response --- 23:12:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:12:05 DEBUG ----- output ----- 23:12:05 DEBUG { 23:12:05 DEBUG "status": "success", 23:12:05 DEBUG "data": { 23:12:05 DEBUG "resultType": "vector", 23:12:05 DEBUG "result": [ 23:12:05 DEBUG { 23:12:05 DEBUG "metric": { 23:12:05 DEBUG "pod": "ds-idrepo-2" 23:12:05 DEBUG }, 23:12:05 DEBUG "value": [ 23:12:05 DEBUG 23:12:05 DEBUG 23:12:05 DEBUG 23:12:05 DEBUG [...] 23:12:05 DEBUG 23:12:05 DEBUG 23:12:05 DEBUG 23:12:05 DEBUG "pod": "ds-idrepo-0" 23:12:05 DEBUG }, 23:12:05 DEBUG "value": [ 23:12:05 DEBUG 1713305525, 23:12:05 DEBUG "0" 23:12:05 DEBUG ] 23:12:05 DEBUG } 23:12:05 DEBUG ] 23:12:05 DEBUG } 23:12:05 DEBUG } 23:12:05 INFO [http_cmd]: http status code OK 23:12:05 DEBUG --- status code --- 23:12:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:12:05 DEBUG --- http response --- 23:12:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:12:05 DEBUG ----- output ----- 23:12:05 DEBUG { 23:12:05 DEBUG "status": "success", 23:12:05 DEBUG "data": { 23:12:05 DEBUG "resultType": "vector", 23:12:05 DEBUG "result": [ 23:12:05 DEBUG { 23:12:05 DEBUG "metric": { 23:12:05 DEBUG "pod": "am-555f4db595-fz8sn" 23:12:05 DEBUG }, 23:12:05 DEBUG "value": [ 23:12:05 DEBUG 23:12:05 INFO [http_cmd]: http status code OK 23:12:05 DEBUG 23:12:05 INFO [http_cmd]: http status code OK 23:12:05 INFO [http_cmd]: http status code OK 23:12:05 DEBUG --- status code --- 23:12:05 DEBUG 23:12:05 DEBUG --- status code --- 23:12:05 DEBUG --- status code --- 23:12:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:12:05 DEBUG [...] 23:12:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:12:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:12:05 DEBUG --- http response --- 23:12:05 DEBUG 23:12:05 DEBUG --- http response --- 23:12:05 DEBUG --- http response --- 23:12:05 DEBUG 23:12:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:12:05 DEBUG 23:12:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 23:12:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:12:05 DEBUG ----- output ----- 23:12:05 DEBUG "pod": "am-555f4db595-kzxsp" 23:12:05 DEBUG ----- output ----- 23:12:05 DEBUG ----- output ----- 23:12:05 DEBUG { 23:12:05 DEBUG }, 23:12:05 DEBUG { 23:12:05 DEBUG { 23:12:05 DEBUG "status": "success", 23:12:05 DEBUG "value": [ 23:12:06 DEBUG "status": "success", 23:12:06 DEBUG "status": "success", 23:12:06 DEBUG "data": { 23:12:06 DEBUG 1713305525, 23:12:06 DEBUG "data": { 23:12:06 DEBUG "data": { 23:12:06 DEBUG "resultType": "vector", 23:12:06 INFO [http_cmd]: http status code OK 23:12:06 DEBUG "3185604" 23:12:06 DEBUG "resultType": "vector", 23:12:06 DEBUG "resultType": "vector", 23:12:06 DEBUG "result": [ 23:12:06 DEBUG --- status code --- 23:12:06 DEBUG ] 23:12:06 DEBUG "result": [ 23:12:06 DEBUG "result": [ 23:12:06 DEBUG { 23:12:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:12:06 DEBUG } 23:12:06 DEBUG { 23:12:06 DEBUG { 23:12:06 DEBUG "metric": { 23:12:06 DEBUG --- http response --- 23:12:06 DEBUG ] 23:12:06 DEBUG "metric": { 23:12:06 DEBUG "metric": { 23:12:06 DEBUG "pod": "am-555f4db595-fz8sn" 23:12:06 DEBUG } 23:12:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 23:12:06 DEBUG "pod": "ds-idrepo-2" 23:12:06 DEBUG "pod": "am-555f4db595-fz8sn" 23:12:06 DEBUG }, 23:12:06 DEBUG } 23:12:06 DEBUG ----- output ----- 23:12:06 DEBUG }, 23:12:06 DEBUG }, 23:12:06 DEBUG "value": [ 23:12:06 DEBUG { 23:12:06 DEBUG "value": [ 23:12:06 DEBUG "value": [ 23:12:06 DEBUG 23:12:06 DEBUG "status": "success", 23:12:06 DEBUG 23:12:06 DEBUG 23:12:06 INFO [http_cmd]: http status code OK 23:12:06 DEBUG 23:12:06 DEBUG "data": { 23:12:06 DEBUG 23:12:06 DEBUG 23:12:06 DEBUG --- status code --- 23:12:06 DEBUG 23:12:06 DEBUG "resultType": "vector", 23:12:06 DEBUG 23:12:06 DEBUG 23:12:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:12:06 DEBUG [...] 23:12:06 DEBUG "result": [ 23:12:06 DEBUG [...] 23:12:06 DEBUG [...] 23:12:06 DEBUG --- http response --- 23:12:06 DEBUG 23:12:06 DEBUG { 23:12:06 DEBUG 23:12:06 DEBUG 23:12:06 DEBUG 23:12:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 23:12:06 DEBUG "metric": { 23:12:06 DEBUG 23:12:06 DEBUG 23:12:06 DEBUG 23:12:06 DEBUG ----- output ----- 23:12:06 DEBUG "pod": "ds-idrepo-2" 23:12:06 DEBUG 23:12:06 DEBUG 23:12:06 DEBUG "pod": "am-555f4db595-kzxsp" 23:12:06 DEBUG { 23:12:06 DEBUG }, 23:12:06 DEBUG "pod": "ds-idrepo-0" 23:12:06 DEBUG "pod": "am-555f4db595-kzxsp" 23:12:06 DEBUG }, 23:12:06 DEBUG "status": "success", 23:12:06 DEBUG "value": [ 23:12:06 DEBUG }, 23:12:06 DEBUG }, 23:12:06 DEBUG "value": [ 23:12:06 DEBUG "data": { 23:12:06 DEBUG 23:12:06 DEBUG "value": [ 23:12:06 DEBUG "value": [ 23:12:06 DEBUG 1713305525, 23:12:06 DEBUG "resultType": "vector", 23:12:06 DEBUG 23:12:06 DEBUG 1713305525, 23:12:06 DEBUG 1713305525, 23:12:06 DEBUG "0.029" 23:12:06 DEBUG "result": [ 23:12:06 DEBUG 23:12:06 DEBUG "0" 23:12:06 DEBUG "0" 23:12:06 DEBUG ] 23:12:06 DEBUG { 23:12:06 DEBUG [...] 23:12:06 DEBUG ] 23:12:06 DEBUG ] 23:12:06 DEBUG } 23:12:06 DEBUG "metric": { 23:12:06 DEBUG 23:12:06 DEBUG } 23:12:06 DEBUG } 23:12:06 DEBUG ] 23:12:06 DEBUG "pod": "ds-idrepo-2" 23:12:06 DEBUG 23:12:06 DEBUG ] 23:12:06 DEBUG ] 23:12:06 DEBUG } 23:12:06 DEBUG }, 23:12:06 DEBUG 23:12:06 DEBUG } 23:12:06 DEBUG } 23:12:06 DEBUG } 23:12:06 DEBUG "value": [ 23:12:06 DEBUG "pod": "ds-idrepo-0" 23:12:06 DEBUG } 23:12:06 DEBUG } 23:12:06 DEBUG 23:12:06 DEBUG }, 23:12:06 DEBUG 23:12:06 DEBUG "value": [ 23:12:06 DEBUG 23:12:06 DEBUG 1713305525, 23:12:06 DEBUG [...] 23:12:06 DEBUG "0" 23:12:06 DEBUG 23:12:06 DEBUG ] 23:12:06 DEBUG 23:12:06 DEBUG } 23:12:06 DEBUG 23:12:06 DEBUG ] 23:12:06 DEBUG "pod": "ds-idrepo-0" 23:12:06 DEBUG } 23:12:06 DEBUG }, 23:12:06 DEBUG } 23:12:06 DEBUG "value": [ 23:12:06 DEBUG 1713305525, 23:12:06 DEBUG "0" 23:12:06 DEBUG ] 23:12:06 DEBUG } 23:12:06 DEBUG ] 23:12:06 DEBUG } 23:12:06 DEBUG } 23:12:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713305579" 23:12:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305579" 23:12:59 ERROR [http_cmd]: ERROR 23:12:59 ERROR --- status code --- 23:12:59 ERROR http status code is 400 (expected [200, 401, 403, 404]) 23:12:59 ERROR --- http response --- 23:12:59 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 23:12:59 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 23:12:59 INFO [http_cmd]: http status code OK 23:12:59 DEBUG --- status code --- 23:12:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:12:59 DEBUG --- http response --- 23:12:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 23:13:02 INFO 23:13:02 INFO [loop_until]: kubectl --namespace=xlou top pods 23:13:02 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 23:13:02 INFO [loop_until]: OK (rc = 0) 23:13:02 DEBUG --- stdout --- 23:13:02 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 9Mi am-555f4db595-4lbss 492m 8655Mi am-555f4db595-fz8sn 570m 8532Mi am-555f4db595-kzxsp 747m 8550Mi ds-cts-0 1974m 12607Mi ds-cts-1 3203m 13333Mi ds-cts-2 2851m 12776Mi ds-idrepo-0 241m 5044Mi ds-idrepo-1 188m 2913Mi ds-idrepo-2 82m 3239Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 4m 1208Mi idm-b85f9d978-rl76f 3m 1227Mi lodemon-bc756c647-cr7x6 1m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-7c9b667974-9mr2s 1019m 1181Mi 23:13:02 DEBUG --- stderr --- 23:13:02 DEBUG 23:13:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305585" 23:13:05 INFO [http_cmd]: http status code OK 23:13:05 DEBUG --- status code --- 23:13:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:13:05 DEBUG --- http response --- 23:13:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 23:13:05 DEBUG ----- output ----- 23:13:05 DEBUG { 23:13:05 DEBUG "status": "success", 23:13:05 DEBUG "data": { 23:13:05 DEBUG "resultType": "vector", 23:13:05 DEBUG "result": [ 23:13:05 DEBUG { 23:13:05 DEBUG "metric": { 23:13:05 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 23:13:05 DEBUG }, 23:13:05 DEBUG "value": [ 23:13:05 DEBUG 23:13:05 DEBUG 23:13:05 DEBUG 23:13:05 DEBUG [...] 23:13:05 DEBUG 23:13:05 DEBUG 23:13:05 DEBUG 23:13:05 DEBUG "pod": "overseer-0-7c9b667974-9mr2s" 23:13:05 DEBUG }, 23:13:05 DEBUG "value": [ 23:13:05 DEBUG 1713305585, 23:13:05 DEBUG "10896110.369373212" 23:13:05 DEBUG ] 23:13:05 DEBUG } 23:13:05 DEBUG ] 23:13:05 DEBUG } 23:13:05 DEBUG } 23:13:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305585" 23:13:05 INFO [http_cmd]: http status code OK 23:13:05 DEBUG --- status code --- 23:13:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:13:05 DEBUG --- http response --- 23:13:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:13:05 DEBUG ----- output ----- 23:13:05 DEBUG { 23:13:05 DEBUG "status": "success", 23:13:05 DEBUG "data": { 23:13:05 DEBUG "resultType": "vector", 23:13:05 DEBUG "result": [ 23:13:05 DEBUG { 23:13:05 DEBUG "metric": { 23:13:05 DEBUG "pod": "am-555f4db595-fz8sn" 23:13:05 DEBUG }, 23:13:05 DEBUG "value": [ 23:13:05 DEBUG 23:13:05 DEBUG 23:13:05 DEBUG 23:13:05 DEBUG [...] 23:13:05 DEBUG 23:13:05 DEBUG 23:13:05 DEBUG 23:13:05 DEBUG "pod": "am-555f4db595-kzxsp" 23:13:05 DEBUG }, 23:13:05 DEBUG "value": [ 23:13:05 DEBUG 1713305585, 23:13:05 DEBUG "897.4333333333333" 23:13:05 DEBUG ] 23:13:05 DEBUG } 23:13:05 DEBUG ] 23:13:05 DEBUG } 23:13:05 DEBUG } 23:13:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713305585" 23:13:05 INFO [http_cmd]: http status code OK 23:13:05 DEBUG --- status code --- 23:13:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:13:05 DEBUG --- http response --- 23:13:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:13:05 DEBUG ----- output ----- 23:13:05 DEBUG { 23:13:05 DEBUG "status": "success", 23:13:05 DEBUG "data": { 23:13:05 DEBUG "resultType": "vector", 23:13:05 DEBUG "result": [ 23:13:05 DEBUG { 23:13:05 DEBUG "metric": { 23:13:05 DEBUG "instance": "10.142.0.107:9100" 23:13:05 DEBUG }, 23:13:05 DEBUG "value": [ 23:13:05 DEBUG 23:13:05 DEBUG 23:13:05 DEBUG 23:13:05 DEBUG [...] 23:13:05 DEBUG 23:13:05 DEBUG 23:13:05 DEBUG 23:13:05 DEBUG "instance": "10.142.0.43:9100" 23:13:05 DEBUG }, 23:13:05 DEBUG "value": [ 23:13:05 DEBUG 1713305585, 23:13:05 DEBUG "7159.333333333333" 23:13:05 DEBUG ] 23:13:05 DEBUG } 23:13:05 DEBUG ] 23:13:05 DEBUG } 23:13:05 DEBUG } 23:13:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305585" 23:13:05 INFO [http_cmd]: http status code OK 23:13:05 DEBUG --- status code --- 23:13:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:13:05 DEBUG --- http response --- 23:13:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 23:13:05 DEBUG ----- output ----- 23:13:05 DEBUG { 23:13:05 DEBUG "status": "success", 23:13:05 DEBUG "data": { 23:13:05 DEBUG "resultType": "vector", 23:13:05 DEBUG "result": [ 23:13:05 DEBUG { 23:13:05 DEBUG "metric": { 23:13:05 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 23:13:05 DEBUG }, 23:13:05 DEBUG "value": [ 23:13:05 DEBUG 23:13:05 DEBUG 23:13:05 DEBUG 23:13:05 DEBUG [...] 23:13:05 DEBUG 23:13:05 DEBUG 23:13:05 DEBUG 23:13:05 DEBUG "pod": "overseer-0-7c9b667974-9mr2s" 23:13:05 DEBUG }, 23:13:05 DEBUG "value": [ 23:13:05 DEBUG 1713305585, 23:13:05 DEBUG "0" 23:13:05 DEBUG ] 23:13:05 DEBUG } 23:13:05 DEBUG ] 23:13:05 DEBUG } 23:13:05 DEBUG } 23:13:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305585" 23:13:05 INFO [http_cmd]: http status code OK 23:13:05 DEBUG --- status code --- 23:13:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:13:05 DEBUG --- http response --- 23:13:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 23:13:05 DEBUG ----- output ----- 23:13:05 DEBUG { 23:13:05 DEBUG "status": "success", 23:13:05 DEBUG "data": { 23:13:05 DEBUG "resultType": "vector", 23:13:05 DEBUG "result": [ 23:13:05 DEBUG { 23:13:05 DEBUG "metric": { 23:13:05 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 23:13:05 DEBUG }, 23:13:05 DEBUG "value": [ 23:13:05 DEBUG 23:13:05 DEBUG 23:13:05 DEBUG 23:13:05 DEBUG [...] 23:13:05 DEBUG 23:13:05 DEBUG 23:13:05 DEBUG 23:13:05 DEBUG "pod": "overseer-0-7c9b667974-9mr2s" 23:13:05 DEBUG }, 23:13:05 DEBUG "value": [ 23:13:05 DEBUG 1713305585, 23:13:05 DEBUG "2.0057325782849995" 23:13:05 DEBUG ] 23:13:05 DEBUG } 23:13:05 DEBUG ] 23:13:05 DEBUG } 23:13:05 DEBUG } 23:13:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713305585" 23:13:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713305585" 23:13:05 INFO [http_cmd]: http status code OK 23:13:05 DEBUG --- status code --- 23:13:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:13:05 DEBUG --- http response --- 23:13:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:13:05 DEBUG ----- output ----- 23:13:05 DEBUG { 23:13:05 DEBUG "status": "success", 23:13:05 DEBUG "data": { 23:13:05 DEBUG "resultType": "vector", 23:13:05 DEBUG "result": [ 23:13:05 DEBUG { 23:13:05 DEBUG "metric": { 23:13:05 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 23:13:05 DEBUG }, 23:13:05 DEBUG "value": [ 23:13:05 DEBUG 23:13:05 DEBUG 23:13:05 DEBUG 23:13:05 DEBUG [...] 23:13:05 DEBUG 23:13:05 DEBUG 23:13:05 DEBUG 23:13:05 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 23:13:05 DEBUG }, 23:13:05 DEBUG "value": [ 23:13:05 DEBUG 1713305585, 23:13:05 DEBUG "1178398720" 23:13:05 DEBUG ] 23:13:05 DEBUG } 23:13:05 DEBUG ] 23:13:05 DEBUG } 23:13:05 DEBUG } 23:13:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713305585" 23:13:05 INFO [http_cmd]: http status code OK 23:13:05 DEBUG --- status code --- 23:13:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:13:05 DEBUG --- http response --- 23:13:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:13:05 DEBUG ----- output ----- 23:13:05 DEBUG { 23:13:05 DEBUG "status": "success", 23:13:05 DEBUG "data": { 23:13:05 DEBUG "resultType": "vector", 23:13:05 DEBUG "result": [ 23:13:05 DEBUG { 23:13:05 DEBUG "metric": { 23:13:05 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 23:13:05 DEBUG }, 23:13:05 DEBUG "value": [ 23:13:05 DEBUG 23:13:05 DEBUG 23:13:05 DEBUG 23:13:05 DEBUG [...] 23:13:05 DEBUG 23:13:05 DEBUG 23:13:05 DEBUG 23:13:05 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 23:13:05 DEBUG }, 23:13:05 DEBUG "value": [ 23:13:05 DEBUG 1713305585, 23:13:05 DEBUG "1.0319704534884944" 23:13:05 DEBUG ] 23:13:05 DEBUG } 23:13:05 DEBUG ] 23:13:05 DEBUG } 23:13:05 DEBUG } 23:13:05 INFO [http_cmd]: http status code OK 23:13:05 DEBUG --- status code --- 23:13:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:13:05 DEBUG --- http response --- 23:13:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:13:05 DEBUG ----- output ----- 23:13:05 DEBUG { 23:13:05 DEBUG "status": "success", 23:13:05 DEBUG "data": { 23:13:05 DEBUG "resultType": "vector", 23:13:05 DEBUG "result": [ 23:13:05 DEBUG { 23:13:05 DEBUG "metric": { 23:13:05 DEBUG "instance": "10.142.0.107:9100" 23:13:05 DEBUG }, 23:13:05 DEBUG "value": [ 23:13:05 DEBUG 23:13:05 DEBUG 23:13:05 DEBUG 23:13:05 DEBUG [...] 23:13:05 DEBUG 23:13:05 DEBUG 23:13:05 DEBUG 23:13:05 DEBUG "instance": "10.142.0.43:9100" 23:13:05 DEBUG }, 23:13:05 DEBUG "value": [ 23:13:05 DEBUG 1713305585, 23:13:05 DEBUG "0" 23:13:05 DEBUG ] 23:13:05 DEBUG } 23:13:05 DEBUG ] 23:13:05 DEBUG } 23:13:05 DEBUG } 23:13:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305585" 23:13:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305585" 23:13:05 INFO [http_cmd]: http status code OK 23:13:05 DEBUG --- status code --- 23:13:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:13:05 DEBUG --- http response --- 23:13:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:13:05 DEBUG ----- output ----- 23:13:05 DEBUG { 23:13:05 DEBUG "status": "success", 23:13:05 DEBUG "data": { 23:13:05 DEBUG "resultType": "vector", 23:13:05 DEBUG "result": [ 23:13:05 DEBUG { 23:13:05 DEBUG "metric": { 23:13:05 DEBUG "pod": "ds-idrepo-2" 23:13:05 DEBUG }, 23:13:05 DEBUG "value": [ 23:13:05 DEBUG 23:13:05 DEBUG 23:13:05 DEBUG 23:13:05 DEBUG [...] 23:13:05 DEBUG 23:13:05 DEBUG 23:13:05 DEBUG 23:13:05 DEBUG "pod": "ds-idrepo-0" 23:13:05 DEBUG }, 23:13:05 DEBUG "value": [ 23:13:05 DEBUG 1713305585, 23:13:05 DEBUG "0" 23:13:05 DEBUG ] 23:13:05 DEBUG } 23:13:05 DEBUG ] 23:13:05 DEBUG } 23:13:05 DEBUG } 23:13:05 INFO [http_cmd]: http status code OK 23:13:05 DEBUG --- status code --- 23:13:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:13:05 DEBUG --- http response --- 23:13:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:13:05 DEBUG ----- output ----- 23:13:05 DEBUG { 23:13:05 DEBUG "status": "success", 23:13:05 DEBUG "data": { 23:13:05 DEBUG "resultType": "vector", 23:13:05 DEBUG "result": [ 23:13:05 DEBUG { 23:13:05 DEBUG "metric": { 23:13:05 DEBUG "pod": "am-555f4db595-fz8sn" 23:13:05 DEBUG }, 23:13:05 DEBUG "value": [ 23:13:05 DEBUG 23:13:05 DEBUG 23:13:05 DEBUG 23:13:05 DEBUG [...] 23:13:05 DEBUG 23:13:05 DEBUG 23:13:05 DEBUG 23:13:05 DEBUG "pod": "am-555f4db595-kzxsp" 23:13:05 DEBUG }, 23:13:05 DEBUG "value": [ 23:13:05 DEBUG 1713305585, 23:13:05 DEBUG "0" 23:13:05 DEBUG ] 23:13:05 DEBUG } 23:13:05 DEBUG ] 23:13:05 DEBUG } 23:13:05 DEBUG } 23:13:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305585" 23:13:05 INFO [http_cmd]: http status code OK 23:13:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713305585" 23:13:05 DEBUG --- status code --- 23:13:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:13:05 DEBUG --- http response --- 23:13:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:13:05 DEBUG ----- output ----- 23:13:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305585" 23:13:05 DEBUG { 23:13:05 DEBUG "status": "success", 23:13:05 DEBUG "data": { 23:13:05 DEBUG "resultType": "vector", 23:13:05 DEBUG "result": [ 23:13:05 DEBUG { 23:13:05 DEBUG "metric": { 23:13:05 DEBUG "pod": "ds-idrepo-2" 23:13:05 DEBUG }, 23:13:05 DEBUG "value": [ 23:13:05 DEBUG 23:13:05 DEBUG 23:13:05 DEBUG 23:13:05 DEBUG [...] 23:13:05 DEBUG 23:13:05 DEBUG 23:13:05 DEBUG 23:13:05 DEBUG "pod": "ds-idrepo-0" 23:13:05 DEBUG }, 23:13:05 DEBUG "value": [ 23:13:05 DEBUG 1713305585, 23:13:05 DEBUG "0" 23:13:05 DEBUG ] 23:13:05 DEBUG } 23:13:05 DEBUG ] 23:13:05 DEBUG } 23:13:05 DEBUG } 23:13:05 INFO [http_cmd]: http status code OK 23:13:05 DEBUG --- status code --- 23:13:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:13:05 DEBUG --- http response --- 23:13:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:13:05 DEBUG ----- output ----- 23:13:05 DEBUG { 23:13:05 DEBUG "status": "success", 23:13:05 DEBUG "data": { 23:13:05 DEBUG "resultType": "vector", 23:13:05 DEBUG "result": [ 23:13:05 DEBUG { 23:13:05 DEBUG "metric": { 23:13:05 DEBUG "pod": "am-555f4db595-fz8sn" 23:13:05 DEBUG }, 23:13:05 DEBUG "value": [ 23:13:05 DEBUG 23:13:05 DEBUG 23:13:05 DEBUG 23:13:05 DEBUG [...] 23:13:05 DEBUG 23:13:05 INFO [http_cmd]: http status code OK 23:13:05 DEBUG 23:13:05 DEBUG --- status code --- 23:13:05 DEBUG 23:13:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:13:05 DEBUG "pod": "am-555f4db595-kzxsp" 23:13:05 DEBUG --- http response --- 23:13:05 DEBUG }, 23:13:05 DEBUG "value": [ 23:13:05 DEBUG 1713305585, 23:13:05 DEBUG "0" 23:13:05 DEBUG ] 23:13:05 DEBUG } 23:13:05 DEBUG ] 23:13:05 DEBUG } 23:13:05 DEBUG } 23:13:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:13:05 DEBUG ----- output ----- 23:13:05 DEBUG { 23:13:05 DEBUG "status": "success", 23:13:05 DEBUG "data": { 23:13:05 DEBUG "resultType": "vector", 23:13:05 DEBUG "result": [ 23:13:05 DEBUG { 23:13:05 DEBUG "metric": { 23:13:05 DEBUG "instance": "10.142.0.107:9100" 23:13:05 DEBUG }, 23:13:05 DEBUG "value": [ 23:13:05 DEBUG 23:13:05 DEBUG 23:13:05 DEBUG 23:13:05 DEBUG [...] 23:13:05 DEBUG 23:13:05 DEBUG 23:13:05 DEBUG 23:13:05 DEBUG "instance": "10.142.0.43:9100" 23:13:05 DEBUG }, 23:13:05 DEBUG "value": [ 23:13:05 DEBUG 1713305585, 23:13:05 DEBUG "0.00046666666666700016" 23:13:05 DEBUG ] 23:13:05 DEBUG } 23:13:05 DEBUG ] 23:13:05 DEBUG } 23:13:05 DEBUG } 23:13:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305585" 23:13:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713305585" 23:13:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305585" 23:13:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713305586" 23:13:06 INFO [http_cmd]: http status code OK 23:13:06 INFO [http_cmd]: http status code OK 23:13:06 DEBUG --- status code --- 23:13:06 DEBUG --- status code --- 23:13:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:13:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:13:06 DEBUG --- http response --- 23:13:06 DEBUG --- http response --- 23:13:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 23:13:06 DEBUG ----- output ----- 23:13:06 DEBUG { 23:13:06 DEBUG "status": "success", 23:13:06 DEBUG "data": { 23:13:06 DEBUG "resultType": "vector", 23:13:06 DEBUG "result": [ 23:13:06 DEBUG { 23:13:06 DEBUG "metric": { 23:13:06 INFO [http_cmd]: http status code OK 23:13:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:13:06 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 23:13:06 DEBUG --- status code --- 23:13:06 DEBUG ----- output ----- 23:13:06 DEBUG }, 23:13:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:13:06 DEBUG { 23:13:06 DEBUG "value": [ 23:13:06 DEBUG --- http response --- 23:13:06 DEBUG "status": "success", 23:13:06 DEBUG 23:13:06 DEBUG "data": { 23:13:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:13:06 DEBUG 23:13:06 DEBUG "resultType": "vector", 23:13:06 DEBUG "result": [ 23:13:06 DEBUG ----- output ----- 23:13:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713305586" 23:13:06 DEBUG 23:13:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305586" 23:13:06 DEBUG { 23:13:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713305586" 23:13:06 DEBUG { 23:13:06 DEBUG [...] 23:13:06 DEBUG "metric": { 23:13:06 INFO 23:13:06 DEBUG "status": "success", 23:13:06 DEBUG 23:13:06 INFO [http_cmd]: http status code OK 23:13:06 DEBUG "instance": "10.142.0.107:9100" 23:13:06 INFO [loop_until]: kubectl --namespace=xlou top node 23:13:06 DEBUG "data": { 23:13:06 DEBUG 23:13:06 DEBUG --- status code --- 23:13:06 DEBUG }, 23:13:06 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 23:13:06 DEBUG "resultType": "vector", 23:13:06 DEBUG 23:13:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:13:06 DEBUG "value": [ 23:13:06 DEBUG "result": [ 23:13:06 DEBUG "pod": "overseer-0-7c9b667974-9mr2s" 23:13:06 DEBUG --- http response --- 23:13:06 DEBUG 23:13:06 DEBUG { 23:13:06 DEBUG }, 23:13:06 DEBUG 23:13:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 23:13:06 DEBUG "metric": { 23:13:06 DEBUG "value": [ 23:13:06 DEBUG 23:13:06 DEBUG ----- output ----- 23:13:06 DEBUG "pod": "am-555f4db595-fz8sn" 23:13:06 DEBUG 1713305585, 23:13:06 DEBUG [...] 23:13:06 DEBUG { 23:13:06 DEBUG }, 23:13:06 DEBUG "0" 23:13:06 DEBUG 23:13:06 DEBUG "status": "success", 23:13:06 DEBUG "value": [ 23:13:06 DEBUG ] 23:13:06 DEBUG 23:13:06 DEBUG "data": { 23:13:06 DEBUG 23:13:06 DEBUG } 23:13:06 DEBUG 23:13:06 DEBUG "resultType": "vector", 23:13:06 DEBUG 23:13:06 DEBUG ] 23:13:06 DEBUG "instance": "10.142.0.43:9100" 23:13:06 DEBUG "result": [ 23:13:06 DEBUG 23:13:06 DEBUG } 23:13:06 DEBUG }, 23:13:06 DEBUG { 23:13:06 DEBUG [...] 23:13:06 DEBUG } 23:13:06 DEBUG "value": [ 23:13:06 DEBUG "metric": { 23:13:06 DEBUG 23:13:06 DEBUG 1713305585, 23:13:06 DEBUG "pod": "ds-idrepo-2" 23:13:06 DEBUG 23:13:06 DEBUG "16472.033333333333" 23:13:06 DEBUG }, 23:13:06 DEBUG 23:13:06 DEBUG ] 23:13:06 DEBUG "value": [ 23:13:06 DEBUG "pod": "am-555f4db595-kzxsp" 23:13:06 DEBUG } 23:13:06 DEBUG 23:13:06 DEBUG }, 23:13:06 DEBUG ] 23:13:06 DEBUG 23:13:06 DEBUG "value": [ 23:13:06 DEBUG } 23:13:06 DEBUG 23:13:06 DEBUG 1713305585, 23:13:06 DEBUG } 23:13:06 DEBUG [...] 23:13:06 DEBUG "0" 23:13:06 DEBUG 23:13:06 DEBUG ] 23:13:06 DEBUG 23:13:06 DEBUG } 23:13:06 DEBUG 23:13:06 DEBUG ] 23:13:06 DEBUG "pod": "ds-cts-2" 23:13:06 DEBUG } 23:13:06 INFO [http_cmd]: http status code OK 23:13:06 DEBUG }, 23:13:06 INFO [http_cmd]: http status code OK 23:13:06 INFO [http_cmd]: http status code OK 23:13:06 DEBUG } 23:13:06 DEBUG --- status code --- 23:13:06 DEBUG "value": [ 23:13:06 DEBUG --- status code --- 23:13:06 DEBUG --- status code --- 23:13:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:13:06 DEBUG 1713305586, 23:13:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:13:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:13:06 DEBUG --- http response --- 23:13:06 DEBUG "0" 23:13:06 DEBUG --- http response --- 23:13:06 DEBUG --- http response --- 23:13:06 DEBUG ] 23:13:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:13:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:13:06 DEBUG } 23:13:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 23:13:06 DEBUG ----- output ----- 23:13:06 DEBUG ----- output ----- 23:13:06 DEBUG ] 23:13:06 DEBUG ----- output ----- 23:13:06 DEBUG { 23:13:06 DEBUG { 23:13:06 DEBUG } 23:13:06 DEBUG { 23:13:06 DEBUG "status": "success", 23:13:06 DEBUG "status": "success", 23:13:06 DEBUG } 23:13:06 DEBUG "status": "success", 23:13:06 DEBUG "data": { 23:13:06 DEBUG "data": { 23:13:06 DEBUG "data": { 23:13:06 DEBUG "resultType": "vector", 23:13:06 DEBUG "resultType": "vector", 23:13:06 DEBUG "resultType": "vector", 23:13:06 DEBUG "result": [ 23:13:06 DEBUG "result": [ 23:13:06 DEBUG "result": [ 23:13:06 DEBUG { 23:13:06 DEBUG { 23:13:06 DEBUG { 23:13:06 DEBUG "metric": { 23:13:06 DEBUG "metric": { 23:13:06 DEBUG "metric": { 23:13:06 DEBUG "pod": "am-555f4db595-fz8sn" 23:13:06 DEBUG "instance": "10.142.0.107:9100" 23:13:06 DEBUG "pod": "ds-idrepo-2" 23:13:06 DEBUG }, 23:13:06 DEBUG }, 23:13:06 DEBUG }, 23:13:06 DEBUG "value": [ 23:13:06 DEBUG "value": [ 23:13:06 DEBUG "value": [ 23:13:06 DEBUG 23:13:06 DEBUG 23:13:06 DEBUG 23:13:06 DEBUG 23:13:06 DEBUG 23:13:06 DEBUG 23:13:06 DEBUG 23:13:06 DEBUG 23:13:06 DEBUG 23:13:06 DEBUG [...] 23:13:06 DEBUG [...] 23:13:06 DEBUG [...] 23:13:06 DEBUG 23:13:06 DEBUG 23:13:06 DEBUG 23:13:06 DEBUG 23:13:06 DEBUG 23:13:06 DEBUG 23:13:06 DEBUG 23:13:06 DEBUG 23:13:06 DEBUG 23:13:06 DEBUG "pod": "am-555f4db595-kzxsp" 23:13:06 DEBUG "instance": "10.142.0.43:9100" 23:13:06 DEBUG "pod": "ds-idrepo-0" 23:13:06 DEBUG }, 23:13:06 DEBUG }, 23:13:06 DEBUG }, 23:13:06 DEBUG "value": [ 23:13:06 DEBUG "value": [ 23:13:06 DEBUG "value": [ 23:13:06 DEBUG 1713305586, 23:13:06 DEBUG 1713305586, 23:13:06 DEBUG 1713305586, 23:13:06 DEBUG "0.029" 23:13:06 DEBUG "19989990400" 23:13:06 DEBUG "0" 23:13:06 DEBUG ] 23:13:06 DEBUG ] 23:13:06 DEBUG ] 23:13:06 DEBUG } 23:13:06 DEBUG } 23:13:06 DEBUG } 23:13:06 DEBUG ] 23:13:06 DEBUG ] 23:13:06 DEBUG ] 23:13:06 DEBUG } 23:13:06 DEBUG } 23:13:06 DEBUG } 23:13:06 DEBUG } 23:13:06 DEBUG } 23:13:06 DEBUG } 23:13:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305586" 23:13:06 INFO [http_cmd]: http status code OK 23:13:06 DEBUG --- status code --- 23:13:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:13:06 DEBUG --- http response --- 23:13:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:13:06 DEBUG ----- output ----- 23:13:06 DEBUG { 23:13:06 DEBUG "status": "success", 23:13:06 DEBUG "data": { 23:13:06 DEBUG "resultType": "vector", 23:13:06 DEBUG "result": [ 23:13:06 DEBUG { 23:13:06 DEBUG "metric": { 23:13:06 DEBUG "pod": "am-555f4db595-fz8sn" 23:13:06 DEBUG }, 23:13:06 DEBUG "value": [ 23:13:06 DEBUG 23:13:06 DEBUG 23:13:06 DEBUG 23:13:06 DEBUG [...] 23:13:06 DEBUG 23:13:06 DEBUG 23:13:06 DEBUG 23:13:06 DEBUG "pod": "am-555f4db595-kzxsp" 23:13:06 DEBUG }, 23:13:06 DEBUG "value": [ 23:13:06 DEBUG 1713305586, 23:13:06 DEBUG "3152344" 23:13:06 DEBUG ] 23:13:06 DEBUG } 23:13:06 DEBUG ] 23:13:06 DEBUG } 23:13:06 DEBUG } 23:13:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305586" 23:13:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305586" 23:13:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305586" 23:13:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305586" 23:13:06 INFO [http_cmd]: http status code OK 23:13:06 DEBUG --- status code --- 23:13:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:13:06 DEBUG --- http response --- 23:13:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:13:06 DEBUG ----- output ----- 23:13:06 DEBUG { 23:13:06 DEBUG "status": "success", 23:13:06 DEBUG "data": { 23:13:06 DEBUG "resultType": "vector", 23:13:06 DEBUG "result": [ 23:13:06 DEBUG { 23:13:06 DEBUG "metric": { 23:13:06 DEBUG "pod": "ds-idrepo-2" 23:13:06 DEBUG }, 23:13:06 DEBUG "value": [ 23:13:06 DEBUG 23:13:06 DEBUG 23:13:06 DEBUG 23:13:06 DEBUG [...] 23:13:06 DEBUG 23:13:06 DEBUG 23:13:06 DEBUG 23:13:06 DEBUG "pod": "ds-idrepo-0" 23:13:06 DEBUG }, 23:13:06 DEBUG "value": [ 23:13:06 DEBUG 1713305586, 23:13:06 DEBUG "0" 23:13:06 DEBUG ] 23:13:06 DEBUG } 23:13:06 DEBUG ] 23:13:06 DEBUG } 23:13:06 DEBUG } 23:13:06 INFO [http_cmd]: http status code OK 23:13:06 INFO [http_cmd]: http status code OK 23:13:06 DEBUG --- status code --- 23:13:06 DEBUG --- status code --- 23:13:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:13:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:13:06 DEBUG --- http response --- 23:13:06 DEBUG --- http response --- 23:13:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 23:13:06 DEBUG ----- output ----- 23:13:06 DEBUG { 23:13:06 DEBUG "status": "success", 23:13:06 DEBUG "data": { 23:13:06 DEBUG "resultType": "vector", 23:13:06 DEBUG "result": [ 23:13:06 DEBUG { 23:13:06 DEBUG "metric": { 23:13:06 DEBUG "pod": "ds-idrepo-2" 23:13:06 DEBUG }, 23:13:06 DEBUG "value": [ 23:13:06 DEBUG 23:13:06 DEBUG 23:13:06 DEBUG 23:13:06 DEBUG [...] 23:13:06 DEBUG 23:13:06 DEBUG 23:13:06 DEBUG 23:13:06 DEBUG "pod": "ds-idrepo-0" 23:13:06 DEBUG }, 23:13:06 DEBUG "value": [ 23:13:06 DEBUG 1713305586, 23:13:06 DEBUG "0" 23:13:06 DEBUG ] 23:13:06 DEBUG } 23:13:06 DEBUG ] 23:13:06 DEBUG } 23:13:06 DEBUG } 23:13:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:13:06 DEBUG ----- output ----- 23:13:06 DEBUG { 23:13:06 DEBUG "status": "success", 23:13:06 DEBUG "data": { 23:13:06 DEBUG "resultType": "vector", 23:13:06 DEBUG "result": [ 23:13:06 DEBUG { 23:13:06 DEBUG "metric": { 23:13:06 DEBUG "pod": "am-555f4db595-fz8sn" 23:13:06 DEBUG }, 23:13:06 DEBUG "value": [ 23:13:06 DEBUG 23:13:06 DEBUG 23:13:06 DEBUG 23:13:06 DEBUG [...] 23:13:06 DEBUG 23:13:06 DEBUG 23:13:06 DEBUG 23:13:06 DEBUG "pod": "am-555f4db595-kzxsp" 23:13:06 DEBUG }, 23:13:06 DEBUG "value": [ 23:13:06 DEBUG 1713305586, 23:13:06 DEBUG "0" 23:13:06 DEBUG ] 23:13:06 DEBUG } 23:13:06 DEBUG ] 23:13:06 DEBUG } 23:13:06 DEBUG } 23:13:06 INFO [loop_until]: OK (rc = 0) 23:13:06 DEBUG --- stdout --- 23:13:06 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 62m 0% 5096Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 2286m 7% 15881Mi 14% gke-xlou-cdm-default-pool-1b0769c8-nhlf 476m 1% 10799Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 82m 0% 2659Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 243m 0% 6606Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 2898m 9% 17115Mi 15% gke-xlou-cdm-default-pool-1b0769c8-xnmp 57m 0% 3159Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 707m 2% 10472Mi 9% gke-xlou-cdm-ds-0f601751-hmm7 196m 0% 5669Mi 5% gke-xlou-cdm-ds-0f601751-j488 3606m 12% 14744Mi 13% gke-xlou-cdm-ds-0f601751-mc5d 83m 0% 3769Mi 3% gke-xlou-cdm-ds-0f601751-nj6q 959m 3% 10472Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 236m 0% 7389Mi 6% gke-xlou-cdm-frontend-a8771548-i82n 1080m 6% 6548Mi 11% 23:13:06 DEBUG --- stderr --- 23:13:06 DEBUG 23:13:06 INFO [http_cmd]: http status code OK 23:13:06 DEBUG --- status code --- 23:13:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:13:06 DEBUG --- http response --- 23:13:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 23:13:06 DEBUG ----- output ----- 23:13:06 DEBUG { 23:13:06 DEBUG "status": "success", 23:13:06 DEBUG "data": { 23:13:06 DEBUG "resultType": "vector", 23:13:06 DEBUG "result": [ 23:13:06 DEBUG { 23:13:06 DEBUG "metric": { 23:13:06 DEBUG "pod": "ds-idrepo-2" 23:13:06 DEBUG }, 23:13:06 DEBUG "value": [ 23:13:06 DEBUG 23:13:06 DEBUG 23:13:06 DEBUG 23:13:06 DEBUG [...] 23:13:06 DEBUG 23:13:06 DEBUG 23:13:06 DEBUG 23:13:06 DEBUG "pod": "ds-idrepo-0" 23:13:06 DEBUG }, 23:13:06 DEBUG "value": [ 23:13:06 DEBUG 1713305586, 23:13:06 DEBUG "0" 23:13:06 DEBUG ] 23:13:06 DEBUG } 23:13:06 DEBUG ] 23:13:06 DEBUG } 23:13:06 DEBUG } 23:13:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305639" 23:13:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713305639" 23:13:59 ERROR [http_cmd]: ERROR 23:13:59 INFO [http_cmd]: http status code OK 23:13:59 ERROR --- status code --- 23:13:59 DEBUG --- status code --- 23:13:59 ERROR http status code is 400 (expected [200, 401, 403, 404]) 23:13:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:13:59 ERROR --- http response --- 23:13:59 DEBUG --- http response --- 23:13:59 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 23:13:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 23:13:59 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 23:14:02 INFO 23:14:02 INFO [loop_until]: kubectl --namespace=xlou top pods 23:14:02 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 23:14:02 INFO [loop_until]: OK (rc = 0) 23:14:02 DEBUG --- stdout --- 23:14:02 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 9Mi am-555f4db595-4lbss 2062m 8658Mi am-555f4db595-fz8sn 2062m 8532Mi am-555f4db595-kzxsp 2029m 8558Mi ds-cts-0 9324m 13133Mi ds-cts-1 8336m 13161Mi ds-cts-2 8676m 13033Mi ds-idrepo-0 544m 5044Mi ds-idrepo-1 598m 2913Mi ds-idrepo-2 604m 3240Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 4m 1208Mi idm-b85f9d978-rl76f 3m 1227Mi lodemon-bc756c647-cr7x6 2m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-7c9b667974-9mr2s 1001m 1494Mi 23:14:02 DEBUG --- stderr --- 23:14:02 DEBUG 23:14:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305645" 23:14:05 INFO [http_cmd]: http status code OK 23:14:05 DEBUG --- status code --- 23:14:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:14:05 DEBUG --- http response --- 23:14:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 23:14:05 DEBUG ----- output ----- 23:14:05 DEBUG { 23:14:05 DEBUG "status": "success", 23:14:05 DEBUG "data": { 23:14:05 DEBUG "resultType": "vector", 23:14:05 DEBUG "result": [ 23:14:05 DEBUG { 23:14:05 DEBUG "metric": { 23:14:05 DEBUG "pod": "ds-cts-1" 23:14:05 DEBUG }, 23:14:05 DEBUG "value": [ 23:14:05 DEBUG 23:14:05 DEBUG 23:14:05 DEBUG 23:14:05 DEBUG [...] 23:14:05 DEBUG 23:14:05 DEBUG 23:14:05 DEBUG 23:14:05 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 23:14:05 DEBUG }, 23:14:05 DEBUG "value": [ 23:14:05 DEBUG 1713305645, 23:14:05 DEBUG "0" 23:14:05 DEBUG ] 23:14:05 DEBUG } 23:14:05 DEBUG ] 23:14:05 DEBUG } 23:14:05 DEBUG } 23:14:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305645" 23:14:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713305645" 23:14:05 INFO [http_cmd]: http status code OK 23:14:05 DEBUG --- status code --- 23:14:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:14:05 DEBUG --- http response --- 23:14:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:14:05 DEBUG ----- output ----- 23:14:05 DEBUG { 23:14:05 DEBUG "status": "success", 23:14:05 DEBUG "data": { 23:14:05 DEBUG "resultType": "vector", 23:14:05 DEBUG "result": [ 23:14:05 DEBUG { 23:14:05 DEBUG "metric": { 23:14:05 DEBUG "pod": "am-555f4db595-fz8sn" 23:14:05 DEBUG }, 23:14:05 DEBUG "value": [ 23:14:05 DEBUG 23:14:05 DEBUG 23:14:05 DEBUG 23:14:05 DEBUG [...] 23:14:05 DEBUG 23:14:05 DEBUG 23:14:05 DEBUG 23:14:05 DEBUG "pod": "am-555f4db595-kzxsp" 23:14:05 DEBUG }, 23:14:05 DEBUG "value": [ 23:14:05 DEBUG 1713305645, 23:14:05 DEBUG "2174.9666666666667" 23:14:05 DEBUG ] 23:14:05 DEBUG } 23:14:05 DEBUG ] 23:14:05 DEBUG } 23:14:05 DEBUG } 23:14:05 INFO [http_cmd]: http status code OK 23:14:05 DEBUG --- status code --- 23:14:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:14:05 DEBUG --- http response --- 23:14:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:14:05 DEBUG ----- output ----- 23:14:05 DEBUG { 23:14:05 DEBUG "status": "success", 23:14:05 DEBUG "data": { 23:14:05 DEBUG "resultType": "vector", 23:14:05 DEBUG "result": [ 23:14:05 DEBUG { 23:14:05 DEBUG "metric": { 23:14:05 DEBUG "instance": "10.142.0.107:9100" 23:14:05 DEBUG }, 23:14:05 DEBUG "value": [ 23:14:05 DEBUG 23:14:05 DEBUG 23:14:05 DEBUG 23:14:05 DEBUG [...] 23:14:05 DEBUG 23:14:05 DEBUG 23:14:05 DEBUG 23:14:05 DEBUG "instance": "10.142.0.43:9100" 23:14:05 DEBUG }, 23:14:05 DEBUG "value": [ 23:14:05 DEBUG 1713305645, 23:14:05 DEBUG "7356.099999999999" 23:14:05 DEBUG ] 23:14:05 DEBUG } 23:14:05 DEBUG ] 23:14:05 DEBUG } 23:14:05 DEBUG } 23:14:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713305645" 23:14:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305645" 23:14:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713305645" 23:14:05 INFO [http_cmd]: http status code OK 23:14:05 DEBUG --- status code --- 23:14:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:14:05 DEBUG --- http response --- 23:14:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:14:05 DEBUG ----- output ----- 23:14:05 DEBUG { 23:14:05 DEBUG "status": "success", 23:14:05 DEBUG "data": { 23:14:05 DEBUG "resultType": "vector", 23:14:05 DEBUG "result": [ 23:14:05 DEBUG { 23:14:05 DEBUG "metric": { 23:14:05 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 23:14:05 DEBUG }, 23:14:05 DEBUG "value": [ 23:14:05 DEBUG 23:14:05 DEBUG 23:14:05 DEBUG 23:14:05 DEBUG [...] 23:14:05 DEBUG 23:14:05 DEBUG 23:14:05 DEBUG 23:14:05 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 23:14:05 DEBUG }, 23:14:05 DEBUG "value": [ 23:14:05 DEBUG 1713305645, 23:14:05 DEBUG "1.029571656684787" 23:14:05 DEBUG ] 23:14:05 DEBUG } 23:14:05 DEBUG ] 23:14:05 DEBUG } 23:14:05 DEBUG } 23:14:05 INFO [http_cmd]: http status code OK 23:14:05 DEBUG --- status code --- 23:14:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:14:05 DEBUG --- http response --- 23:14:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 23:14:05 DEBUG ----- output ----- 23:14:05 DEBUG { 23:14:05 DEBUG "status": "success", 23:14:05 DEBUG "data": { 23:14:05 DEBUG "resultType": "vector", 23:14:05 DEBUG "result": [ 23:14:05 DEBUG { 23:14:05 DEBUG "metric": { 23:14:05 DEBUG "pod": "ds-cts-1" 23:14:05 DEBUG }, 23:14:05 DEBUG "value": [ 23:14:05 DEBUG 23:14:05 DEBUG 23:14:05 DEBUG 23:14:05 DEBUG [...] 23:14:05 DEBUG 23:14:05 INFO [http_cmd]: http status code OK 23:14:05 DEBUG 23:14:05 DEBUG --- status code --- 23:14:05 DEBUG 23:14:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:14:05 DEBUG "pod": "am-555f4db595-kzxsp" 23:14:05 DEBUG --- http response --- 23:14:05 DEBUG }, 23:14:05 DEBUG "value": [ 23:14:05 DEBUG 1713305645, 23:14:05 DEBUG "0" 23:14:05 DEBUG ] 23:14:05 DEBUG } 23:14:05 DEBUG ] 23:14:05 DEBUG } 23:14:05 DEBUG } 23:14:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:14:05 DEBUG ----- output ----- 23:14:05 DEBUG { 23:14:05 DEBUG "status": "success", 23:14:05 DEBUG "data": { 23:14:05 DEBUG "resultType": "vector", 23:14:05 DEBUG "result": [ 23:14:05 DEBUG { 23:14:05 DEBUG "metric": { 23:14:05 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 23:14:05 DEBUG }, 23:14:05 DEBUG "value": [ 23:14:05 DEBUG 23:14:05 DEBUG 23:14:05 DEBUG 23:14:05 DEBUG [...] 23:14:05 DEBUG 23:14:05 DEBUG 23:14:05 DEBUG 23:14:05 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 23:14:05 DEBUG }, 23:14:05 DEBUG "value": [ 23:14:05 DEBUG 1713305645, 23:14:05 DEBUG "1527996416" 23:14:05 DEBUG ] 23:14:05 DEBUG } 23:14:05 DEBUG ] 23:14:05 DEBUG } 23:14:05 DEBUG } 23:14:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305645" 23:14:05 INFO [http_cmd]: http status code OK 23:14:05 DEBUG --- status code --- 23:14:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:14:05 DEBUG --- http response --- 23:14:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 23:14:05 DEBUG ----- output ----- 23:14:05 DEBUG { 23:14:05 DEBUG "status": "success", 23:14:05 DEBUG "data": { 23:14:05 DEBUG "resultType": "vector", 23:14:05 DEBUG "result": [ 23:14:05 DEBUG { 23:14:05 DEBUG "metric": { 23:14:05 DEBUG "pod": "ds-cts-1" 23:14:05 DEBUG }, 23:14:05 DEBUG "value": [ 23:14:05 DEBUG 23:14:05 DEBUG 23:14:05 DEBUG 23:14:05 DEBUG [...] 23:14:05 DEBUG 23:14:05 DEBUG 23:14:05 DEBUG 23:14:05 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 23:14:05 DEBUG }, 23:14:05 DEBUG "value": [ 23:14:05 DEBUG 1713305645, 23:14:05 DEBUG "0" 23:14:05 DEBUG ] 23:14:05 DEBUG } 23:14:05 DEBUG ] 23:14:05 DEBUG } 23:14:05 DEBUG } 23:14:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305645" 23:14:05 INFO [http_cmd]: http status code OK 23:14:05 DEBUG --- status code --- 23:14:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:14:05 DEBUG --- http response --- 23:14:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:14:05 DEBUG ----- output ----- 23:14:05 DEBUG { 23:14:05 DEBUG "status": "success", 23:14:05 DEBUG "data": { 23:14:05 DEBUG "resultType": "vector", 23:14:05 DEBUG "result": [ 23:14:05 DEBUG { 23:14:05 DEBUG "metric": { 23:14:05 DEBUG "pod": "ds-idrepo-2" 23:14:05 DEBUG }, 23:14:05 DEBUG "value": [ 23:14:05 DEBUG 23:14:05 DEBUG 23:14:05 DEBUG 23:14:05 DEBUG [...] 23:14:05 DEBUG 23:14:05 DEBUG 23:14:05 DEBUG 23:14:05 DEBUG "pod": "ds-idrepo-0" 23:14:05 DEBUG }, 23:14:05 DEBUG "value": [ 23:14:05 DEBUG 1713305645, 23:14:05 DEBUG "0" 23:14:05 DEBUG ] 23:14:05 DEBUG } 23:14:05 DEBUG ] 23:14:05 DEBUG } 23:14:05 DEBUG } 23:14:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305645" 23:14:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713305645" 23:14:05 INFO [http_cmd]: http status code OK 23:14:05 DEBUG --- status code --- 23:14:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:14:05 DEBUG --- http response --- 23:14:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:14:05 DEBUG ----- output ----- 23:14:05 DEBUG { 23:14:05 DEBUG "status": "success", 23:14:05 DEBUG "data": { 23:14:05 DEBUG "resultType": "vector", 23:14:05 DEBUG "result": [ 23:14:05 DEBUG { 23:14:05 DEBUG "metric": { 23:14:05 DEBUG "pod": "am-555f4db595-fz8sn" 23:14:05 DEBUG }, 23:14:05 DEBUG "value": [ 23:14:05 DEBUG 23:14:05 DEBUG 23:14:05 DEBUG 23:14:05 DEBUG [...] 23:14:05 DEBUG 23:14:05 DEBUG 23:14:05 DEBUG 23:14:05 DEBUG "pod": "am-555f4db595-kzxsp" 23:14:05 DEBUG }, 23:14:05 DEBUG "value": [ 23:14:05 DEBUG 1713305645, 23:14:05 DEBUG "0" 23:14:05 DEBUG ] 23:14:05 DEBUG } 23:14:05 DEBUG ] 23:14:05 DEBUG } 23:14:05 DEBUG } 23:14:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305645" 23:14:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305645" 23:14:05 INFO [http_cmd]: http status code OK 23:14:05 DEBUG --- status code --- 23:14:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:14:05 DEBUG --- http response --- 23:14:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:14:05 DEBUG ----- output ----- 23:14:05 DEBUG { 23:14:05 DEBUG "status": "success", 23:14:05 DEBUG "data": { 23:14:05 DEBUG "resultType": "vector", 23:14:05 DEBUG "result": [ 23:14:05 DEBUG { 23:14:05 DEBUG "metric": { 23:14:05 DEBUG "instance": "10.142.0.107:9100" 23:14:05 DEBUG }, 23:14:05 DEBUG "value": [ 23:14:05 DEBUG 23:14:05 DEBUG 23:14:05 DEBUG 23:14:05 DEBUG [...] 23:14:05 DEBUG 23:14:05 DEBUG 23:14:05 DEBUG 23:14:05 DEBUG "instance": "10.142.0.43:9100" 23:14:05 DEBUG }, 23:14:05 DEBUG "value": [ 23:14:05 DEBUG 1713305645, 23:14:05 DEBUG "0" 23:14:05 DEBUG ] 23:14:05 DEBUG } 23:14:05 DEBUG ] 23:14:05 DEBUG } 23:14:05 DEBUG } 23:14:05 INFO [http_cmd]: http status code OK 23:14:05 DEBUG --- status code --- 23:14:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:14:05 DEBUG --- http response --- 23:14:05 INFO [http_cmd]: http status code OK 23:14:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:14:05 DEBUG --- status code --- 23:14:05 DEBUG ----- output ----- 23:14:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:14:05 DEBUG { 23:14:05 DEBUG --- http response --- 23:14:05 DEBUG "status": "success", 23:14:05 DEBUG "data": { 23:14:05 DEBUG "resultType": "vector", 23:14:05 DEBUG "result": [ 23:14:05 DEBUG { 23:14:05 DEBUG "metric": { 23:14:05 DEBUG "pod": "ds-idrepo-2" 23:14:05 DEBUG }, 23:14:05 DEBUG "value": [ 23:14:05 DEBUG 23:14:05 DEBUG 23:14:05 DEBUG 23:14:05 DEBUG [...] 23:14:05 DEBUG 23:14:05 DEBUG 23:14:05 DEBUG 23:14:05 DEBUG "pod": "ds-idrepo-0" 23:14:05 DEBUG }, 23:14:05 DEBUG "value": [ 23:14:05 DEBUG 1713305645, 23:14:05 DEBUG "0" 23:14:05 DEBUG ] 23:14:05 DEBUG } 23:14:05 DEBUG ] 23:14:05 DEBUG } 23:14:05 DEBUG } 23:14:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:14:05 DEBUG ----- output ----- 23:14:05 DEBUG { 23:14:05 DEBUG "status": "success", 23:14:06 DEBUG "data": { 23:14:06 DEBUG "resultType": "vector", 23:14:06 DEBUG "result": [ 23:14:06 DEBUG { 23:14:06 DEBUG "metric": { 23:14:06 DEBUG "pod": "am-555f4db595-fz8sn" 23:14:06 DEBUG }, 23:14:06 DEBUG "value": [ 23:14:06 DEBUG 23:14:06 DEBUG 23:14:06 DEBUG 23:14:06 DEBUG [...] 23:14:06 DEBUG 23:14:06 DEBUG 23:14:06 DEBUG 23:14:06 DEBUG "pod": "am-555f4db595-kzxsp" 23:14:06 DEBUG }, 23:14:06 DEBUG "value": [ 23:14:06 DEBUG 1713305645, 23:14:06 DEBUG "0" 23:14:06 DEBUG ] 23:14:06 DEBUG } 23:14:06 DEBUG ] 23:14:06 DEBUG } 23:14:06 DEBUG } 23:14:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713305646" 23:14:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305646" 23:14:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713305646" 23:14:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305646" 23:14:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713305646" 23:14:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713305646" 23:14:06 INFO [http_cmd]: http status code OK 23:14:06 DEBUG --- status code --- 23:14:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:14:06 DEBUG --- http response --- 23:14:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:14:06 DEBUG ----- output ----- 23:14:06 DEBUG { 23:14:06 DEBUG "status": "success", 23:14:06 DEBUG "data": { 23:14:06 DEBUG "resultType": "vector", 23:14:06 DEBUG "result": [ 23:14:06 DEBUG { 23:14:06 DEBUG "metric": { 23:14:06 DEBUG "instance": "10.142.0.107:9100" 23:14:06 DEBUG }, 23:14:06 DEBUG "value": [ 23:14:06 DEBUG 23:14:06 DEBUG 23:14:06 DEBUG 23:14:06 DEBUG [...] 23:14:06 DEBUG 23:14:06 DEBUG 23:14:06 DEBUG 23:14:06 DEBUG "instance": "10.142.0.43:9100" 23:14:06 DEBUG }, 23:14:06 DEBUG "value": [ 23:14:06 DEBUG 1713305646, 23:14:06 DEBUG "0.0003333333333330302" 23:14:06 DEBUG ] 23:14:06 DEBUG } 23:14:06 DEBUG ] 23:14:06 DEBUG } 23:14:06 DEBUG } 23:14:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305646" 23:14:06 INFO [http_cmd]: http status code OK 23:14:06 DEBUG --- status code --- 23:14:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:14:06 DEBUG --- http response --- 23:14:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 23:14:06 DEBUG ----- output ----- 23:14:06 DEBUG { 23:14:06 DEBUG "status": "success", 23:14:06 DEBUG "data": { 23:14:06 DEBUG "resultType": "vector", 23:14:06 DEBUG "result": [ 23:14:06 DEBUG { 23:14:06 DEBUG "metric": { 23:14:06 DEBUG "pod": "ds-idrepo-2" 23:14:06 DEBUG }, 23:14:06 DEBUG "value": [ 23:14:06 DEBUG 23:14:06 DEBUG 23:14:06 DEBUG 23:14:06 DEBUG [...] 23:14:06 DEBUG 23:14:06 DEBUG 23:14:06 DEBUG 23:14:06 DEBUG "pod": "ds-idrepo-0" 23:14:06 DEBUG }, 23:14:06 DEBUG "value": [ 23:14:06 DEBUG 1713305646, 23:14:06 DEBUG "0" 23:14:06 DEBUG ] 23:14:06 DEBUG } 23:14:06 DEBUG ] 23:14:06 DEBUG } 23:14:06 DEBUG } 23:14:06 INFO [http_cmd]: http status code OK 23:14:06 INFO [http_cmd]: http status code OK 23:14:06 DEBUG --- status code --- 23:14:06 INFO [http_cmd]: http status code OK 23:14:06 INFO [http_cmd]: http status code OK 23:14:06 DEBUG --- status code --- 23:14:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:14:06 DEBUG --- status code --- 23:14:06 DEBUG --- status code --- 23:14:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:14:06 DEBUG --- http response --- 23:14:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:14:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:14:06 DEBUG --- http response --- 23:14:06 DEBUG --- http response --- 23:14:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:14:06 DEBUG --- http response --- 23:14:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:14:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:14:06 DEBUG ----- output ----- 23:14:06 DEBUG ----- output ----- 23:14:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 23:14:06 DEBUG ----- output ----- 23:14:06 DEBUG { 23:14:06 DEBUG { 23:14:06 DEBUG ----- output ----- 23:14:06 DEBUG { 23:14:06 DEBUG "status": "success", 23:14:06 DEBUG "status": "success", 23:14:06 DEBUG { 23:14:06 DEBUG "status": "success", 23:14:06 DEBUG "data": { 23:14:06 DEBUG "data": { 23:14:06 DEBUG "status": "success", 23:14:06 DEBUG "data": { 23:14:06 DEBUG "resultType": "vector", 23:14:06 DEBUG "resultType": "vector", 23:14:06 DEBUG "data": { 23:14:06 DEBUG "resultType": "vector", 23:14:06 DEBUG "result": [ 23:14:06 DEBUG "result": [ 23:14:06 DEBUG "resultType": "vector", 23:14:06 DEBUG "result": [ 23:14:06 DEBUG { 23:14:06 DEBUG { 23:14:06 DEBUG "result": [ 23:14:06 DEBUG { 23:14:06 DEBUG "metric": { 23:14:06 DEBUG "metric": { 23:14:06 DEBUG { 23:14:06 DEBUG "metric": { 23:14:06 DEBUG "pod": "am-555f4db595-fz8sn" 23:14:06 DEBUG "pod": "am-555f4db595-fz8sn" 23:14:06 DEBUG "metric": { 23:14:06 DEBUG "instance": "10.142.0.107:9100" 23:14:06 DEBUG }, 23:14:06 DEBUG }, 23:14:06 DEBUG "pod": "ds-idrepo-2" 23:14:06 DEBUG }, 23:14:06 DEBUG "value": [ 23:14:06 DEBUG "value": [ 23:14:06 DEBUG }, 23:14:06 DEBUG "value": [ 23:14:06 DEBUG 23:14:06 DEBUG 23:14:06 DEBUG "value": [ 23:14:06 DEBUG 23:14:06 DEBUG 23:14:06 DEBUG 23:14:06 DEBUG 23:14:06 DEBUG 23:14:06 DEBUG 23:14:06 DEBUG 23:14:06 DEBUG 23:14:06 DEBUG 23:14:06 DEBUG [...] 23:14:06 DEBUG [...] 23:14:06 DEBUG 23:14:06 DEBUG [...] 23:14:06 DEBUG 23:14:06 DEBUG 23:14:06 DEBUG [...] 23:14:06 DEBUG 23:14:06 DEBUG 23:14:06 DEBUG 23:14:06 DEBUG 23:14:06 DEBUG 23:14:06 DEBUG 23:14:06 DEBUG 23:14:06 DEBUG 23:14:06 DEBUG 23:14:06 DEBUG "pod": "am-555f4db595-kzxsp" 23:14:06 DEBUG "pod": "am-555f4db595-kzxsp" 23:14:06 DEBUG 23:14:06 DEBUG "instance": "10.142.0.43:9100" 23:14:06 DEBUG }, 23:14:06 DEBUG }, 23:14:06 DEBUG "pod": "ds-cts-2" 23:14:06 DEBUG }, 23:14:06 DEBUG "value": [ 23:14:06 DEBUG "value": [ 23:14:06 DEBUG }, 23:14:06 DEBUG "value": [ 23:14:06 DEBUG 1713305646, 23:14:06 DEBUG 1713305646, 23:14:06 DEBUG "value": [ 23:14:06 INFO [http_cmd]: http status code OK 23:14:06 DEBUG 1713305646, 23:14:06 DEBUG "0" 23:14:06 DEBUG "0.029" 23:14:06 DEBUG 1713305646, 23:14:06 DEBUG --- status code --- 23:14:06 DEBUG "16144.633333333331" 23:14:06 DEBUG ] 23:14:06 DEBUG ] 23:14:06 DEBUG "0" 23:14:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:14:06 DEBUG ] 23:14:06 DEBUG } 23:14:06 DEBUG } 23:14:06 DEBUG ] 23:14:06 DEBUG --- http response --- 23:14:06 DEBUG } 23:14:06 DEBUG ] 23:14:06 DEBUG ] 23:14:06 DEBUG } 23:14:06 DEBUG ] 23:14:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:14:06 DEBUG } 23:14:06 DEBUG } 23:14:06 DEBUG ] 23:14:06 DEBUG } 23:14:06 DEBUG ----- output ----- 23:14:06 DEBUG } 23:14:06 DEBUG } 23:14:06 DEBUG } 23:14:06 DEBUG } 23:14:06 DEBUG { 23:14:06 DEBUG } 23:14:06 DEBUG "status": "success", 23:14:06 DEBUG "data": { 23:14:06 DEBUG "resultType": "vector", 23:14:06 DEBUG "result": [ 23:14:06 DEBUG { 23:14:06 DEBUG "metric": { 23:14:06 DEBUG "pod": "am-555f4db595-fz8sn" 23:14:06 DEBUG }, 23:14:06 DEBUG "value": [ 23:14:06 DEBUG 23:14:06 DEBUG 23:14:06 DEBUG 23:14:06 DEBUG [...] 23:14:06 DEBUG 23:14:06 DEBUG 23:14:06 DEBUG 23:14:06 DEBUG "pod": "am-555f4db595-kzxsp" 23:14:06 DEBUG }, 23:14:06 DEBUG "value": [ 23:14:06 DEBUG 1713305646, 23:14:06 DEBUG "3040750.5" 23:14:06 DEBUG ] 23:14:06 DEBUG } 23:14:06 DEBUG ] 23:14:06 DEBUG } 23:14:06 DEBUG } 23:14:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305646" 23:14:06 INFO [http_cmd]: http status code OK 23:14:06 DEBUG --- status code --- 23:14:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:14:06 DEBUG --- http response --- 23:14:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 23:14:06 DEBUG ----- output ----- 23:14:06 DEBUG { 23:14:06 DEBUG "status": "success", 23:14:06 DEBUG "data": { 23:14:06 DEBUG "resultType": "vector", 23:14:06 DEBUG "result": [ 23:14:06 DEBUG { 23:14:06 DEBUG "metric": { 23:14:06 DEBUG "pod": "ds-cts-1" 23:14:06 DEBUG }, 23:14:06 DEBUG "value": [ 23:14:06 DEBUG 23:14:06 DEBUG 23:14:06 DEBUG 23:14:06 DEBUG [...] 23:14:06 DEBUG 23:14:06 DEBUG 23:14:06 DEBUG 23:14:06 DEBUG "pod": "am-555f4db595-kzxsp" 23:14:06 DEBUG }, 23:14:06 DEBUG "value": [ 23:14:06 DEBUG 1713305646, 23:14:06 DEBUG "543.7950147698231" 23:14:06 DEBUG ] 23:14:06 DEBUG } 23:14:06 DEBUG ] 23:14:06 DEBUG } 23:14:06 DEBUG } 23:14:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305646" 23:14:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305646" 23:14:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305646" 23:14:06 INFO 23:14:06 INFO [loop_until]: kubectl --namespace=xlou top node 23:14:06 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 23:14:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713305646" 23:14:06 INFO [http_cmd]: http status code OK 23:14:06 DEBUG --- status code --- 23:14:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:14:06 DEBUG --- http response --- 23:14:06 INFO [http_cmd]: http status code OK 23:14:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:14:06 DEBUG --- status code --- 23:14:06 DEBUG ----- output ----- 23:14:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:14:06 DEBUG { 23:14:06 DEBUG --- http response --- 23:14:06 DEBUG "status": "success", 23:14:06 DEBUG "data": { 23:14:06 DEBUG "resultType": "vector", 23:14:06 DEBUG "result": [ 23:14:06 DEBUG { 23:14:06 DEBUG "metric": { 23:14:06 DEBUG "pod": "ds-idrepo-2" 23:14:06 DEBUG }, 23:14:06 DEBUG "value": [ 23:14:06 DEBUG 23:14:06 DEBUG 23:14:06 DEBUG 23:14:06 DEBUG [...] 23:14:06 DEBUG 23:14:06 DEBUG 23:14:06 DEBUG 23:14:06 DEBUG "pod": "ds-idrepo-0" 23:14:06 DEBUG }, 23:14:06 DEBUG "value": [ 23:14:06 DEBUG 1713305646, 23:14:06 DEBUG "0" 23:14:06 DEBUG ] 23:14:06 DEBUG } 23:14:06 DEBUG ] 23:14:06 DEBUG } 23:14:06 DEBUG } 23:14:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 23:14:06 INFO [http_cmd]: http status code OK 23:14:06 DEBUG ----- output ----- 23:14:06 DEBUG --- status code --- 23:14:06 DEBUG { 23:14:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:14:06 DEBUG "status": "success", 23:14:06 DEBUG --- http response --- 23:14:06 DEBUG "data": { 23:14:06 DEBUG "resultType": "vector", 23:14:06 DEBUG "result": [ 23:14:06 DEBUG { 23:14:06 DEBUG "metric": { 23:14:06 DEBUG "pod": "ds-idrepo-2" 23:14:06 DEBUG }, 23:14:06 DEBUG "value": [ 23:14:06 DEBUG 23:14:06 DEBUG 23:14:06 DEBUG 23:14:06 DEBUG [...] 23:14:06 DEBUG 23:14:06 DEBUG 23:14:06 DEBUG 23:14:06 DEBUG "pod": "ds-idrepo-0" 23:14:06 DEBUG }, 23:14:06 DEBUG "value": [ 23:14:06 DEBUG 1713305646, 23:14:06 DEBUG "0" 23:14:06 DEBUG ] 23:14:06 DEBUG } 23:14:06 DEBUG ] 23:14:06 DEBUG } 23:14:06 DEBUG } 23:14:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 23:14:06 DEBUG ----- output ----- 23:14:06 DEBUG { 23:14:06 DEBUG "status": "success", 23:14:06 DEBUG "data": { 23:14:06 DEBUG "resultType": "vector", 23:14:06 DEBUG "result": [ 23:14:06 DEBUG { 23:14:06 DEBUG "metric": { 23:14:06 DEBUG "pod": "ds-idrepo-2" 23:14:06 DEBUG }, 23:14:06 DEBUG "value": [ 23:14:06 DEBUG 23:14:06 DEBUG 23:14:06 DEBUG 23:14:06 DEBUG [...] 23:14:06 DEBUG 23:14:06 DEBUG 23:14:06 DEBUG 23:14:06 DEBUG "pod": "ds-idrepo-0" 23:14:06 DEBUG }, 23:14:06 DEBUG "value": [ 23:14:06 DEBUG 1713305646, 23:14:06 DEBUG "0" 23:14:06 DEBUG ] 23:14:06 DEBUG } 23:14:06 DEBUG ] 23:14:06 DEBUG } 23:14:06 DEBUG } 23:14:06 INFO [http_cmd]: http status code OK 23:14:06 DEBUG --- status code --- 23:14:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:14:06 DEBUG --- http response --- 23:14:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:14:06 DEBUG ----- output ----- 23:14:06 DEBUG { 23:14:06 DEBUG "status": "success", 23:14:06 DEBUG "data": { 23:14:06 DEBUG "resultType": "vector", 23:14:06 DEBUG "result": [ 23:14:06 DEBUG { 23:14:06 DEBUG "metric": { 23:14:06 DEBUG "instance": "10.142.0.107:9100" 23:14:06 DEBUG }, 23:14:06 DEBUG "value": [ 23:14:06 DEBUG 23:14:06 DEBUG 23:14:06 DEBUG 23:14:06 DEBUG [...] 23:14:06 DEBUG 23:14:06 DEBUG 23:14:06 DEBUG 23:14:06 DEBUG "instance": "10.142.0.43:9100" 23:14:06 DEBUG }, 23:14:06 DEBUG "value": [ 23:14:06 DEBUG 1713305646, 23:14:06 DEBUG "19990770688" 23:14:06 DEBUG ] 23:14:06 DEBUG } 23:14:06 DEBUG ] 23:14:06 DEBUG } 23:14:06 DEBUG } 23:14:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305646" 23:14:06 INFO [http_cmd]: http status code OK 23:14:06 DEBUG --- status code --- 23:14:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:14:06 DEBUG --- http response --- 23:14:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:14:06 DEBUG ----- output ----- 23:14:06 DEBUG { 23:14:06 DEBUG "status": "success", 23:14:06 DEBUG "data": { 23:14:06 DEBUG "resultType": "vector", 23:14:06 DEBUG "result": [ 23:14:06 DEBUG { 23:14:06 DEBUG "metric": { 23:14:06 DEBUG "pod": "am-555f4db595-fz8sn" 23:14:06 DEBUG }, 23:14:06 DEBUG "value": [ 23:14:06 DEBUG 23:14:06 DEBUG 23:14:06 DEBUG 23:14:06 DEBUG [...] 23:14:06 DEBUG 23:14:06 DEBUG 23:14:06 DEBUG 23:14:06 DEBUG "pod": "am-555f4db595-kzxsp" 23:14:06 DEBUG }, 23:14:06 DEBUG "value": [ 23:14:06 DEBUG 1713305646, 23:14:06 DEBUG "0" 23:14:06 DEBUG ] 23:14:06 DEBUG } 23:14:06 DEBUG ] 23:14:06 DEBUG } 23:14:06 DEBUG } 23:14:06 INFO [loop_until]: OK (rc = 0) 23:14:06 DEBUG --- stdout --- 23:14:06 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 60m 0% 5102Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 8735m 29% 15708Mi 14% gke-xlou-cdm-default-pool-1b0769c8-nhlf 2005m 6% 10797Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 85m 0% 2646Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 646m 2% 6607Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 10231m 34% 17644Mi 15% gke-xlou-cdm-default-pool-1b0769c8-xnmp 59m 0% 3161Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 2135m 7% 10477Mi 9% gke-xlou-cdm-ds-0f601751-hmm7 714m 2% 5680Mi 5% gke-xlou-cdm-ds-0f601751-j488 8869m 29% 15150Mi 13% gke-xlou-cdm-ds-0f601751-mc5d 84m 0% 3679Mi 3% gke-xlou-cdm-ds-0f601751-nj6q 2138m 7% 10467Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 634m 2% 7396Mi 6% gke-xlou-cdm-frontend-a8771548-i82n 1079m 6% 6863Mi 11% 23:14:06 DEBUG --- stderr --- 23:14:06 DEBUG 23:14:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713305699" 23:14:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305699" 23:14:59 ERROR [http_cmd]: ERROR 23:14:59 INFO [http_cmd]: http status code OK 23:14:59 ERROR --- status code --- 23:14:59 DEBUG --- status code --- 23:14:59 ERROR http status code is 400 (expected [200, 401, 403, 404]) 23:14:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:14:59 ERROR --- http response --- 23:14:59 DEBUG --- http response --- 23:14:59 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 23:14:59 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 23:14:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 23:15:02 INFO 23:15:02 INFO [loop_until]: kubectl --namespace=xlou top pods 23:15:02 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 23:15:02 INFO [loop_until]: OK (rc = 0) 23:15:02 DEBUG --- stdout --- 23:15:02 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 9Mi am-555f4db595-4lbss 2115m 8661Mi am-555f4db595-fz8sn 2095m 8535Mi am-555f4db595-kzxsp 2056m 8561Mi ds-cts-0 8581m 13076Mi ds-cts-1 8439m 13205Mi ds-cts-2 8481m 13798Mi ds-idrepo-0 596m 5044Mi ds-idrepo-1 603m 2913Mi ds-idrepo-2 582m 3240Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 3m 1208Mi idm-b85f9d978-rl76f 4m 1226Mi lodemon-bc756c647-cr7x6 1m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-7c9b667974-9mr2s 435m 1584Mi 23:15:02 DEBUG --- stderr --- 23:15:02 DEBUG 23:15:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305705" 23:15:05 INFO [http_cmd]: http status code OK 23:15:05 DEBUG --- status code --- 23:15:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:15:05 DEBUG --- http response --- 23:15:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 23:15:05 DEBUG ----- output ----- 23:15:05 DEBUG { 23:15:05 DEBUG "status": "success", 23:15:05 DEBUG "data": { 23:15:05 DEBUG "resultType": "vector", 23:15:05 DEBUG "result": [ 23:15:05 DEBUG { 23:15:05 DEBUG "metric": { 23:15:05 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 23:15:05 DEBUG }, 23:15:05 DEBUG "value": [ 23:15:05 DEBUG 23:15:05 DEBUG 23:15:05 DEBUG 23:15:05 DEBUG [...] 23:15:05 DEBUG 23:15:05 DEBUG 23:15:05 DEBUG 23:15:05 DEBUG "pod": "ds-cts-2" 23:15:05 DEBUG }, 23:15:05 DEBUG "value": [ 23:15:05 DEBUG 1713305705, 23:15:05 DEBUG "19195936.507936507" 23:15:05 DEBUG ] 23:15:05 DEBUG } 23:15:05 DEBUG ] 23:15:05 DEBUG } 23:15:05 DEBUG } 23:15:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713305705" 23:15:05 INFO [http_cmd]: http status code OK 23:15:05 DEBUG --- status code --- 23:15:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:15:05 DEBUG --- http response --- 23:15:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:15:05 DEBUG ----- output ----- 23:15:05 DEBUG { 23:15:05 DEBUG "status": "success", 23:15:05 DEBUG "data": { 23:15:05 DEBUG "resultType": "vector", 23:15:05 DEBUG "result": [ 23:15:05 DEBUG { 23:15:05 DEBUG "metric": { 23:15:05 DEBUG "instance": "10.142.0.107:9100" 23:15:05 DEBUG }, 23:15:05 DEBUG "value": [ 23:15:05 DEBUG 23:15:05 DEBUG 23:15:05 DEBUG 23:15:05 DEBUG [...] 23:15:05 DEBUG 23:15:05 DEBUG 23:15:05 DEBUG 23:15:05 DEBUG "instance": "10.142.0.43:9100" 23:15:05 DEBUG }, 23:15:05 DEBUG "value": [ 23:15:05 DEBUG 1713305705, 23:15:05 DEBUG "6673" 23:15:05 DEBUG ] 23:15:05 DEBUG } 23:15:05 DEBUG ] 23:15:05 DEBUG } 23:15:05 DEBUG } 23:15:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305705" 23:15:05 INFO [http_cmd]: http status code OK 23:15:05 DEBUG --- status code --- 23:15:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:15:05 DEBUG --- http response --- 23:15:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:15:05 DEBUG ----- output ----- 23:15:05 DEBUG { 23:15:05 DEBUG "status": "success", 23:15:05 DEBUG "data": { 23:15:05 DEBUG "resultType": "vector", 23:15:05 DEBUG "result": [ 23:15:05 DEBUG { 23:15:05 DEBUG "metric": { 23:15:05 DEBUG "pod": "am-555f4db595-fz8sn" 23:15:05 DEBUG }, 23:15:05 DEBUG "value": [ 23:15:05 DEBUG 23:15:05 DEBUG 23:15:05 DEBUG 23:15:05 DEBUG [...] 23:15:05 DEBUG 23:15:05 DEBUG 23:15:05 DEBUG 23:15:05 DEBUG "pod": "am-555f4db595-kzxsp" 23:15:05 DEBUG }, 23:15:05 DEBUG "value": [ 23:15:05 DEBUG 1713305705, 23:15:05 DEBUG "2170.9333333333334" 23:15:05 DEBUG ] 23:15:05 DEBUG } 23:15:05 DEBUG ] 23:15:05 DEBUG } 23:15:05 DEBUG } 23:15:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713305705" 23:15:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305705" 23:15:05 INFO [http_cmd]: http status code OK 23:15:05 DEBUG --- status code --- 23:15:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:15:05 DEBUG --- http response --- 23:15:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 23:15:05 DEBUG ----- output ----- 23:15:05 DEBUG { 23:15:05 DEBUG "status": "success", 23:15:05 DEBUG "data": { 23:15:05 DEBUG "resultType": "vector", 23:15:05 DEBUG "result": [ 23:15:05 DEBUG { 23:15:05 DEBUG "metric": { 23:15:05 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 23:15:05 DEBUG }, 23:15:05 DEBUG "value": [ 23:15:05 DEBUG 23:15:05 DEBUG 23:15:05 DEBUG 23:15:05 DEBUG [...] 23:15:05 DEBUG 23:15:05 DEBUG 23:15:05 DEBUG 23:15:05 DEBUG "pod": "ds-cts-2" 23:15:05 DEBUG }, 23:15:05 DEBUG "value": [ 23:15:05 DEBUG 1713305705, 23:15:05 DEBUG "34321434.81210148" 23:15:05 DEBUG ] 23:15:05 DEBUG } 23:15:05 DEBUG ] 23:15:05 DEBUG } 23:15:05 DEBUG } 23:15:05 INFO [http_cmd]: http status code OK 23:15:05 DEBUG --- status code --- 23:15:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:15:05 DEBUG --- http response --- 23:15:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:15:05 DEBUG ----- output ----- 23:15:05 DEBUG { 23:15:05 DEBUG "status": "success", 23:15:05 DEBUG "data": { 23:15:05 DEBUG "resultType": "vector", 23:15:05 DEBUG "result": [ 23:15:05 DEBUG { 23:15:05 DEBUG "metric": { 23:15:05 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 23:15:05 DEBUG }, 23:15:05 DEBUG "value": [ 23:15:05 DEBUG 23:15:05 DEBUG 23:15:05 DEBUG 23:15:05 DEBUG [...] 23:15:05 DEBUG 23:15:05 DEBUG 23:15:05 DEBUG 23:15:05 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 23:15:05 DEBUG }, 23:15:05 DEBUG "value": [ 23:15:05 DEBUG 1713305705, 23:15:05 DEBUG "1500483584" 23:15:05 DEBUG ] 23:15:05 DEBUG } 23:15:05 DEBUG ] 23:15:05 DEBUG } 23:15:05 DEBUG } 23:15:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305705" 23:15:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713305705" 23:15:05 INFO [http_cmd]: http status code OK 23:15:05 DEBUG --- status code --- 23:15:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:15:05 DEBUG --- http response --- 23:15:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:15:05 DEBUG ----- output ----- 23:15:05 DEBUG { 23:15:05 DEBUG "status": "success", 23:15:05 DEBUG "data": { 23:15:05 DEBUG "resultType": "vector", 23:15:05 DEBUG "result": [ 23:15:05 DEBUG { 23:15:05 DEBUG "metric": { 23:15:05 DEBUG "pod": "ds-idrepo-2" 23:15:05 DEBUG }, 23:15:05 DEBUG "value": [ 23:15:05 DEBUG 23:15:05 DEBUG 23:15:05 DEBUG 23:15:05 DEBUG [...] 23:15:05 DEBUG 23:15:05 DEBUG 23:15:05 DEBUG 23:15:05 DEBUG "pod": "ds-idrepo-0" 23:15:05 DEBUG }, 23:15:05 DEBUG "value": [ 23:15:05 DEBUG 1713305705, 23:15:05 DEBUG "0" 23:15:05 DEBUG ] 23:15:05 DEBUG } 23:15:05 DEBUG ] 23:15:05 DEBUG } 23:15:05 DEBUG } 23:15:05 INFO [http_cmd]: http status code OK 23:15:05 DEBUG --- status code --- 23:15:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:15:05 DEBUG --- http response --- 23:15:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:15:05 DEBUG ----- output ----- 23:15:05 DEBUG { 23:15:05 DEBUG "status": "success", 23:15:05 DEBUG "data": { 23:15:05 DEBUG "resultType": "vector", 23:15:05 DEBUG "result": [ 23:15:05 DEBUG { 23:15:05 DEBUG "metric": { 23:15:05 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 23:15:05 DEBUG }, 23:15:05 DEBUG "value": [ 23:15:05 DEBUG 23:15:05 DEBUG 23:15:05 DEBUG 23:15:05 DEBUG [...] 23:15:05 DEBUG 23:15:05 DEBUG 23:15:05 DEBUG 23:15:05 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 23:15:05 DEBUG }, 23:15:05 DEBUG "value": [ 23:15:05 DEBUG 1713305705, 23:15:05 DEBUG "0.8508614407774335" 23:15:05 DEBUG ] 23:15:05 DEBUG } 23:15:05 DEBUG ] 23:15:05 DEBUG } 23:15:05 DEBUG } 23:15:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305706" 23:15:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713305706" 23:15:06 INFO [http_cmd]: http status code OK 23:15:06 DEBUG --- status code --- 23:15:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:15:06 DEBUG --- http response --- 23:15:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:15:06 DEBUG ----- output ----- 23:15:06 DEBUG { 23:15:06 DEBUG "status": "success", 23:15:06 DEBUG "data": { 23:15:06 DEBUG "resultType": "vector", 23:15:06 DEBUG "result": [ 23:15:06 DEBUG { 23:15:06 DEBUG "metric": { 23:15:06 DEBUG "pod": "am-555f4db595-fz8sn" 23:15:06 DEBUG }, 23:15:06 DEBUG "value": [ 23:15:06 DEBUG 23:15:06 DEBUG 23:15:06 DEBUG 23:15:06 DEBUG [...] 23:15:06 DEBUG 23:15:06 DEBUG 23:15:06 DEBUG 23:15:06 DEBUG "pod": "am-555f4db595-kzxsp" 23:15:06 DEBUG }, 23:15:06 DEBUG "value": [ 23:15:06 DEBUG 1713305706, 23:15:06 DEBUG "0" 23:15:06 DEBUG ] 23:15:06 DEBUG } 23:15:06 DEBUG ] 23:15:06 DEBUG } 23:15:06 DEBUG } 23:15:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305706" 23:15:06 INFO [http_cmd]: http status code OK 23:15:06 DEBUG --- status code --- 23:15:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:15:06 DEBUG --- http response --- 23:15:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:15:06 DEBUG ----- output ----- 23:15:06 DEBUG { 23:15:06 DEBUG "status": "success", 23:15:06 DEBUG "data": { 23:15:06 DEBUG "resultType": "vector", 23:15:06 DEBUG "result": [ 23:15:06 DEBUG { 23:15:06 DEBUG "metric": { 23:15:06 DEBUG "instance": "10.142.0.107:9100" 23:15:06 DEBUG }, 23:15:06 DEBUG "value": [ 23:15:06 DEBUG 23:15:06 DEBUG 23:15:06 DEBUG 23:15:06 DEBUG [...] 23:15:06 DEBUG 23:15:06 DEBUG 23:15:06 DEBUG 23:15:06 DEBUG "instance": "10.142.0.43:9100" 23:15:06 DEBUG }, 23:15:06 DEBUG "value": [ 23:15:06 DEBUG 1713305706, 23:15:06 DEBUG "0" 23:15:06 DEBUG ] 23:15:06 DEBUG } 23:15:06 DEBUG ] 23:15:06 DEBUG } 23:15:06 DEBUG } 23:15:06 INFO [http_cmd]: http status code OK 23:15:06 DEBUG --- status code --- 23:15:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:15:06 DEBUG --- http response --- 23:15:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 23:15:06 DEBUG ----- output ----- 23:15:06 DEBUG { 23:15:06 DEBUG "status": "success", 23:15:06 DEBUG "data": { 23:15:06 DEBUG "resultType": "vector", 23:15:06 DEBUG "result": [ 23:15:06 DEBUG { 23:15:06 DEBUG "metric": { 23:15:06 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 23:15:06 DEBUG }, 23:15:06 DEBUG "value": [ 23:15:06 DEBUG 23:15:06 DEBUG 23:15:06 DEBUG 23:15:06 DEBUG [...] 23:15:06 DEBUG 23:15:06 DEBUG 23:15:06 DEBUG 23:15:06 DEBUG "pod": "ds-cts-2" 23:15:06 DEBUG }, 23:15:06 DEBUG "value": [ 23:15:06 DEBUG 1713305706, 23:15:06 DEBUG "8.177896418396502" 23:15:06 DEBUG ] 23:15:06 DEBUG } 23:15:06 DEBUG ] 23:15:06 DEBUG } 23:15:06 DEBUG } 23:15:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305706" 23:15:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713305706" 23:15:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305706" 23:15:06 INFO [http_cmd]: http status code OK 23:15:06 DEBUG --- status code --- 23:15:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:15:06 DEBUG --- http response --- 23:15:06 INFO [http_cmd]: http status code OK 23:15:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:15:06 DEBUG --- status code --- 23:15:06 DEBUG ----- output ----- 23:15:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:15:06 DEBUG { 23:15:06 DEBUG --- http response --- 23:15:06 DEBUG "status": "success", 23:15:06 DEBUG "data": { 23:15:06 DEBUG "resultType": "vector", 23:15:06 DEBUG "result": [ 23:15:06 DEBUG { 23:15:06 DEBUG "metric": { 23:15:06 DEBUG "pod": "am-555f4db595-fz8sn" 23:15:06 DEBUG }, 23:15:06 DEBUG "value": [ 23:15:06 DEBUG 23:15:06 DEBUG 23:15:06 DEBUG 23:15:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:15:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305706" 23:15:06 DEBUG [...] 23:15:06 DEBUG ----- output ----- 23:15:06 DEBUG 23:15:06 DEBUG { 23:15:06 DEBUG 23:15:06 DEBUG "status": "success", 23:15:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305706" 23:15:06 DEBUG 23:15:06 DEBUG "data": { 23:15:06 DEBUG "pod": "am-555f4db595-kzxsp" 23:15:06 DEBUG "resultType": "vector", 23:15:06 DEBUG }, 23:15:06 DEBUG "result": [ 23:15:06 DEBUG "value": [ 23:15:06 DEBUG { 23:15:06 DEBUG 1713305706, 23:15:06 DEBUG "metric": { 23:15:06 DEBUG "0" 23:15:06 DEBUG "instance": "10.142.0.107:9100" 23:15:06 DEBUG ] 23:15:06 DEBUG }, 23:15:06 DEBUG } 23:15:06 DEBUG "value": [ 23:15:06 DEBUG ] 23:15:06 DEBUG 23:15:06 DEBUG } 23:15:06 DEBUG 23:15:06 DEBUG } 23:15:06 DEBUG 23:15:06 DEBUG [...] 23:15:06 DEBUG 23:15:06 DEBUG 23:15:06 DEBUG 23:15:06 DEBUG "instance": "10.142.0.43:9100" 23:15:06 DEBUG }, 23:15:06 DEBUG "value": [ 23:15:06 DEBUG 1713305706, 23:15:06 DEBUG "0.0003333333333330302" 23:15:06 DEBUG ] 23:15:06 DEBUG } 23:15:06 DEBUG ] 23:15:06 DEBUG } 23:15:06 DEBUG } 23:15:06 INFO [http_cmd]: http status code OK 23:15:06 DEBUG --- status code --- 23:15:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:15:06 DEBUG --- http response --- 23:15:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 23:15:06 DEBUG ----- output ----- 23:15:06 DEBUG { 23:15:06 DEBUG "status": "success", 23:15:06 DEBUG "data": { 23:15:06 DEBUG "resultType": "vector", 23:15:06 DEBUG "result": [ 23:15:06 DEBUG { 23:15:06 DEBUG "metric": { 23:15:06 DEBUG "pod": "ds-idrepo-2" 23:15:06 DEBUG }, 23:15:06 DEBUG "value": [ 23:15:06 DEBUG 23:15:06 DEBUG 23:15:06 DEBUG 23:15:06 DEBUG [...] 23:15:06 DEBUG 23:15:06 DEBUG 23:15:06 DEBUG 23:15:06 DEBUG "pod": "ds-idrepo-0" 23:15:06 DEBUG }, 23:15:06 DEBUG "value": [ 23:15:06 DEBUG 1713305706, 23:15:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713305706" 23:15:06 DEBUG "0" 23:15:06 DEBUG ] 23:15:06 DEBUG } 23:15:06 DEBUG ] 23:15:06 DEBUG } 23:15:06 DEBUG } 23:15:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713305706" 23:15:06 INFO [http_cmd]: http status code OK 23:15:06 INFO [http_cmd]: http status code OK 23:15:06 INFO [http_cmd]: http status code OK 23:15:06 DEBUG --- status code --- 23:15:06 DEBUG --- status code --- 23:15:06 DEBUG --- status code --- 23:15:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:15:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:15:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:15:06 DEBUG --- http response --- 23:15:06 DEBUG --- http response --- 23:15:06 DEBUG --- http response --- 23:15:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 23:15:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:15:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:15:06 DEBUG ----- output ----- 23:15:06 DEBUG ----- output ----- 23:15:06 DEBUG ----- output ----- 23:15:06 DEBUG { 23:15:06 DEBUG { 23:15:06 DEBUG { 23:15:06 DEBUG "status": "success", 23:15:06 DEBUG "status": "success", 23:15:06 DEBUG "status": "success", 23:15:06 DEBUG "data": { 23:15:06 DEBUG "data": { 23:15:06 DEBUG "data": { 23:15:06 DEBUG "resultType": "vector", 23:15:06 DEBUG "resultType": "vector", 23:15:06 DEBUG "resultType": "vector", 23:15:06 DEBUG "result": [ 23:15:06 DEBUG "result": [ 23:15:06 DEBUG "result": [ 23:15:06 DEBUG { 23:15:06 DEBUG { 23:15:06 DEBUG { 23:15:06 DEBUG "metric": { 23:15:06 DEBUG "metric": { 23:15:06 DEBUG "metric": { 23:15:06 DEBUG "pod": "ds-idrepo-2" 23:15:06 DEBUG "pod": "ds-idrepo-2" 23:15:06 DEBUG "pod": "am-555f4db595-fz8sn" 23:15:06 DEBUG }, 23:15:06 DEBUG }, 23:15:06 DEBUG }, 23:15:06 DEBUG "value": [ 23:15:06 DEBUG "value": [ 23:15:06 DEBUG "value": [ 23:15:06 DEBUG 23:15:06 DEBUG 23:15:06 DEBUG 23:15:06 DEBUG 23:15:06 DEBUG 23:15:06 DEBUG 23:15:06 DEBUG 23:15:06 DEBUG 23:15:06 DEBUG 23:15:06 DEBUG [...] 23:15:06 DEBUG [...] 23:15:06 DEBUG [...] 23:15:06 DEBUG 23:15:06 DEBUG 23:15:06 DEBUG 23:15:06 DEBUG 23:15:06 DEBUG 23:15:06 DEBUG 23:15:06 DEBUG 23:15:06 DEBUG 23:15:06 DEBUG 23:15:06 DEBUG "pod": "ds-cts-2" 23:15:06 DEBUG "pod": "ds-idrepo-0" 23:15:06 DEBUG "pod": "am-555f4db595-kzxsp" 23:15:06 DEBUG }, 23:15:06 DEBUG }, 23:15:06 DEBUG }, 23:15:06 DEBUG "value": [ 23:15:06 DEBUG "value": [ 23:15:06 DEBUG "value": [ 23:15:06 DEBUG 1713305706, 23:15:06 DEBUG 1713305706, 23:15:06 DEBUG 1713305706, 23:15:06 DEBUG "0" 23:15:06 DEBUG "0" 23:15:06 DEBUG "0" 23:15:06 DEBUG ] 23:15:06 DEBUG ] 23:15:06 DEBUG ] 23:15:06 DEBUG } 23:15:06 DEBUG } 23:15:06 DEBUG } 23:15:06 DEBUG ] 23:15:06 DEBUG ] 23:15:06 DEBUG ] 23:15:06 DEBUG } 23:15:06 DEBUG } 23:15:06 DEBUG } 23:15:06 DEBUG } 23:15:06 DEBUG } 23:15:06 DEBUG } 23:15:06 INFO [http_cmd]: http status code OK 23:15:06 DEBUG --- status code --- 23:15:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:15:06 DEBUG --- http response --- 23:15:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:15:06 DEBUG ----- output ----- 23:15:06 DEBUG { 23:15:06 DEBUG "status": "success", 23:15:06 DEBUG "data": { 23:15:06 DEBUG "resultType": "vector", 23:15:06 DEBUG "result": [ 23:15:06 DEBUG { 23:15:06 DEBUG "metric": { 23:15:06 DEBUG "pod": "am-555f4db595-fz8sn" 23:15:06 DEBUG }, 23:15:06 DEBUG "value": [ 23:15:06 DEBUG 23:15:06 DEBUG 23:15:06 DEBUG 23:15:06 DEBUG [...] 23:15:06 DEBUG 23:15:06 DEBUG 23:15:06 DEBUG 23:15:06 DEBUG "pod": "am-555f4db595-kzxsp" 23:15:06 DEBUG }, 23:15:06 DEBUG "value": [ 23:15:06 DEBUG 1713305706, 23:15:06 DEBUG "0.029" 23:15:06 DEBUG ] 23:15:06 DEBUG } 23:15:06 DEBUG ] 23:15:06 DEBUG } 23:15:06 DEBUG } 23:15:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305706" 23:15:06 INFO [http_cmd]: http status code OK 23:15:06 DEBUG --- status code --- 23:15:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:15:06 DEBUG --- http response --- 23:15:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:15:06 DEBUG ----- output ----- 23:15:06 DEBUG { 23:15:06 DEBUG "status": "success", 23:15:06 DEBUG "data": { 23:15:06 DEBUG "resultType": "vector", 23:15:06 DEBUG "result": [ 23:15:06 DEBUG { 23:15:06 DEBUG "metric": { 23:15:06 DEBUG "pod": "ds-idrepo-2" 23:15:06 DEBUG }, 23:15:06 DEBUG "value": [ 23:15:06 DEBUG 23:15:06 DEBUG 23:15:06 DEBUG 23:15:06 DEBUG [...] 23:15:06 DEBUG 23:15:06 DEBUG 23:15:06 DEBUG 23:15:06 DEBUG "pod": "ds-idrepo-0" 23:15:06 DEBUG }, 23:15:06 DEBUG "value": [ 23:15:06 DEBUG 1713305706, 23:15:06 DEBUG "0" 23:15:06 DEBUG ] 23:15:06 DEBUG } 23:15:06 DEBUG ] 23:15:06 DEBUG } 23:15:06 DEBUG } 23:15:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713305706" 23:15:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305706" 23:15:06 INFO [http_cmd]: http status code OK 23:15:06 DEBUG --- status code --- 23:15:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:15:06 DEBUG --- http response --- 23:15:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:15:06 DEBUG ----- output ----- 23:15:06 DEBUG { 23:15:06 DEBUG "status": "success", 23:15:06 DEBUG "data": { 23:15:06 DEBUG "resultType": "vector", 23:15:06 DEBUG "result": [ 23:15:06 DEBUG { 23:15:06 DEBUG "metric": { 23:15:06 DEBUG "instance": "10.142.0.107:9100" 23:15:06 DEBUG }, 23:15:06 DEBUG "value": [ 23:15:06 DEBUG 23:15:06 DEBUG 23:15:06 DEBUG 23:15:06 DEBUG [...] 23:15:06 DEBUG 23:15:06 DEBUG 23:15:06 DEBUG 23:15:06 DEBUG "instance": "10.142.0.43:9100" 23:15:06 DEBUG }, 23:15:06 DEBUG "value": [ 23:15:06 DEBUG 1713305706, 23:15:06 DEBUG "15992.266666666665" 23:15:06 DEBUG ] 23:15:06 DEBUG } 23:15:06 DEBUG ] 23:15:06 DEBUG } 23:15:06 DEBUG } 23:15:06 INFO [http_cmd]: http status code OK 23:15:06 DEBUG --- status code --- 23:15:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:15:06 DEBUG --- http response --- 23:15:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:15:06 DEBUG ----- output ----- 23:15:06 DEBUG { 23:15:06 DEBUG "status": "success", 23:15:06 DEBUG "data": { 23:15:06 DEBUG "resultType": "vector", 23:15:06 DEBUG "result": [ 23:15:06 DEBUG { 23:15:06 DEBUG "metric": { 23:15:06 DEBUG "pod": "am-555f4db595-fz8sn" 23:15:06 DEBUG }, 23:15:06 DEBUG "value": [ 23:15:06 DEBUG 23:15:06 DEBUG 23:15:06 DEBUG 23:15:06 DEBUG [...] 23:15:06 DEBUG 23:15:06 DEBUG 23:15:06 DEBUG 23:15:06 DEBUG "pod": "am-555f4db595-kzxsp" 23:15:06 DEBUG }, 23:15:06 DEBUG "value": [ 23:15:06 DEBUG 1713305706, 23:15:06 DEBUG "2910864" 23:15:06 DEBUG ] 23:15:06 DEBUG } 23:15:06 DEBUG ] 23:15:06 DEBUG } 23:15:06 DEBUG } 23:15:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305706" 23:15:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713305706" 23:15:06 INFO [http_cmd]: http status code OK 23:15:06 DEBUG --- status code --- 23:15:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:15:06 DEBUG --- http response --- 23:15:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:15:06 DEBUG ----- output ----- 23:15:06 DEBUG { 23:15:06 DEBUG "status": "success", 23:15:06 DEBUG "data": { 23:15:06 DEBUG "resultType": "vector", 23:15:06 DEBUG "result": [ 23:15:06 DEBUG { 23:15:06 DEBUG "metric": { 23:15:06 DEBUG "instance": "10.142.0.107:9100" 23:15:06 DEBUG }, 23:15:06 DEBUG "value": [ 23:15:06 DEBUG 23:15:06 DEBUG 23:15:06 DEBUG 23:15:06 DEBUG [...] 23:15:06 DEBUG 23:15:06 DEBUG 23:15:06 DEBUG 23:15:06 DEBUG "instance": "10.142.0.43:9100" 23:15:06 DEBUG }, 23:15:06 DEBUG "value": [ 23:15:06 DEBUG 1713305706, 23:15:06 DEBUG "19991303168" 23:15:06 DEBUG ] 23:15:06 DEBUG } 23:15:06 DEBUG ] 23:15:06 DEBUG } 23:15:06 DEBUG } 23:15:06 INFO [http_cmd]: http status code OK 23:15:06 DEBUG --- status code --- 23:15:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:15:06 DEBUG --- http response --- 23:15:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 23:15:06 DEBUG ----- output ----- 23:15:06 DEBUG { 23:15:06 DEBUG "status": "success", 23:15:06 DEBUG "data": { 23:15:06 DEBUG "resultType": "vector", 23:15:06 DEBUG "result": [ 23:15:06 DEBUG { 23:15:06 DEBUG "metric": { 23:15:06 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 23:15:06 DEBUG }, 23:15:06 DEBUG "value": [ 23:15:06 DEBUG 23:15:06 DEBUG 23:15:06 DEBUG 23:15:06 DEBUG [...] 23:15:06 DEBUG 23:15:06 DEBUG 23:15:06 DEBUG 23:15:06 DEBUG "pod": "ds-cts-2" 23:15:06 DEBUG }, 23:15:06 DEBUG "value": [ 23:15:06 DEBUG 1713305706, 23:15:06 DEBUG "7492801.519468186" 23:15:06 DEBUG ] 23:15:06 DEBUG } 23:15:06 DEBUG ] 23:15:06 DEBUG } 23:15:06 DEBUG } 23:15:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305706" 23:15:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305706" 23:15:06 INFO 23:15:06 INFO [loop_until]: kubectl --namespace=xlou top node 23:15:06 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 23:15:06 INFO [http_cmd]: http status code OK 23:15:06 DEBUG --- status code --- 23:15:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:15:06 DEBUG --- http response --- 23:15:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 23:15:06 DEBUG ----- output ----- 23:15:06 DEBUG { 23:15:06 DEBUG "status": "success", 23:15:06 DEBUG "data": { 23:15:06 DEBUG "resultType": "vector", 23:15:06 DEBUG "result": [ 23:15:06 DEBUG { 23:15:06 DEBUG "metric": { 23:15:06 DEBUG "pod": "ds-idrepo-2" 23:15:06 DEBUG }, 23:15:06 DEBUG "value": [ 23:15:06 DEBUG 23:15:06 DEBUG 23:15:06 DEBUG 23:15:06 DEBUG [...] 23:15:06 DEBUG 23:15:06 DEBUG 23:15:06 DEBUG 23:15:06 DEBUG "pod": "ds-idrepo-0" 23:15:06 DEBUG }, 23:15:06 DEBUG "value": [ 23:15:06 DEBUG 1713305706, 23:15:06 DEBUG "0" 23:15:06 DEBUG ] 23:15:06 DEBUG } 23:15:06 DEBUG ] 23:15:06 DEBUG } 23:15:06 DEBUG } 23:15:06 INFO [http_cmd]: http status code OK 23:15:06 DEBUG --- status code --- 23:15:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:15:06 DEBUG --- http response --- 23:15:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 23:15:06 DEBUG ----- output ----- 23:15:06 DEBUG { 23:15:06 DEBUG "status": "success", 23:15:06 DEBUG "data": { 23:15:06 DEBUG "resultType": "vector", 23:15:06 DEBUG "result": [ 23:15:06 DEBUG { 23:15:06 DEBUG "metric": { 23:15:06 DEBUG "pod": "ds-idrepo-2" 23:15:06 DEBUG }, 23:15:06 DEBUG "value": [ 23:15:06 DEBUG 23:15:06 DEBUG 23:15:06 DEBUG 23:15:06 DEBUG [...] 23:15:06 DEBUG 23:15:06 DEBUG 23:15:06 DEBUG 23:15:06 DEBUG "pod": "ds-idrepo-0" 23:15:06 DEBUG }, 23:15:06 DEBUG "value": [ 23:15:06 DEBUG 1713305706, 23:15:06 DEBUG "0" 23:15:06 DEBUG ] 23:15:06 DEBUG } 23:15:06 DEBUG ] 23:15:06 DEBUG } 23:15:06 DEBUG } 23:15:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305706" 23:15:06 INFO [http_cmd]: http status code OK 23:15:06 DEBUG --- status code --- 23:15:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:15:06 DEBUG --- http response --- 23:15:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:15:06 DEBUG ----- output ----- 23:15:06 DEBUG { 23:15:06 DEBUG "status": "success", 23:15:06 DEBUG "data": { 23:15:06 DEBUG "resultType": "vector", 23:15:06 DEBUG "result": [ 23:15:06 DEBUG { 23:15:06 DEBUG "metric": { 23:15:06 DEBUG "pod": "am-555f4db595-fz8sn" 23:15:06 DEBUG }, 23:15:06 DEBUG "value": [ 23:15:06 DEBUG 23:15:06 DEBUG 23:15:06 DEBUG 23:15:06 DEBUG [...] 23:15:06 DEBUG 23:15:06 DEBUG 23:15:06 DEBUG 23:15:06 DEBUG "pod": "am-555f4db595-kzxsp" 23:15:06 DEBUG }, 23:15:06 DEBUG "value": [ 23:15:06 DEBUG 1713305706, 23:15:06 DEBUG "0" 23:15:06 DEBUG ] 23:15:06 DEBUG } 23:15:06 DEBUG ] 23:15:06 DEBUG } 23:15:06 DEBUG } 23:15:06 INFO [loop_until]: OK (rc = 0) 23:15:06 DEBUG --- stdout --- 23:15:06 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 58m 0% 5088Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 8106m 27% 16299Mi 14% gke-xlou-cdm-default-pool-1b0769c8-nhlf 2091m 7% 10802Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 89m 0% 2643Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 637m 2% 6600Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 9292m 31% 17567Mi 15% gke-xlou-cdm-default-pool-1b0769c8-xnmp 60m 0% 3161Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 2151m 7% 10487Mi 9% gke-xlou-cdm-ds-0f601751-hmm7 706m 2% 5668Mi 5% gke-xlou-cdm-ds-0f601751-j488 8596m 28% 15777Mi 14% gke-xlou-cdm-ds-0f601751-mc5d 86m 0% 3655Mi 3% gke-xlou-cdm-ds-0f601751-nj6q 2160m 7% 10458Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 674m 2% 7390Mi 6% gke-xlou-cdm-frontend-a8771548-i82n 788m 4% 7006Mi 11% 23:15:06 DEBUG --- stderr --- 23:15:06 DEBUG 23:15:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713305759" 23:15:59 ERROR [http_cmd]: ERROR 23:15:59 ERROR --- status code --- 23:15:59 ERROR http status code is 400 (expected [200, 401, 403, 404]) 23:15:59 ERROR --- http response --- 23:15:59 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 23:15:59 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 23:15:59 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305759" 23:15:59 INFO [http_cmd]: http status code OK 23:15:59 DEBUG --- status code --- 23:15:59 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:15:59 DEBUG --- http response --- 23:15:59 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 23:16:02 INFO 23:16:02 INFO [loop_until]: kubectl --namespace=xlou top pods 23:16:02 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 23:16:02 INFO [loop_until]: OK (rc = 0) 23:16:02 DEBUG --- stdout --- 23:16:02 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 9Mi am-555f4db595-4lbss 2000m 8662Mi am-555f4db595-fz8sn 2153m 8534Mi am-555f4db595-kzxsp 1993m 8563Mi ds-cts-0 8995m 12974Mi ds-cts-1 8383m 13594Mi ds-cts-2 8662m 13666Mi ds-idrepo-0 609m 5044Mi ds-idrepo-1 603m 2914Mi ds-idrepo-2 585m 3239Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 3m 1208Mi idm-b85f9d978-rl76f 4m 1226Mi lodemon-bc756c647-cr7x6 1m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-7c9b667974-9mr2s 1010m 2861Mi 23:16:02 DEBUG --- stderr --- 23:16:02 DEBUG 23:16:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305765" 23:16:05 INFO [http_cmd]: http status code OK 23:16:05 DEBUG --- status code --- 23:16:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:16:05 DEBUG --- http response --- 23:16:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 23:16:05 DEBUG ----- output ----- 23:16:05 DEBUG { 23:16:05 DEBUG "status": "success", 23:16:05 DEBUG "data": { 23:16:05 DEBUG "resultType": "vector", 23:16:05 DEBUG "result": [ 23:16:05 DEBUG { 23:16:05 DEBUG "metric": { 23:16:05 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 23:16:05 DEBUG }, 23:16:05 DEBUG "value": [ 23:16:05 DEBUG 23:16:05 DEBUG 23:16:05 DEBUG 23:16:05 DEBUG [...] 23:16:05 DEBUG 23:16:05 DEBUG 23:16:05 DEBUG 23:16:05 DEBUG "pod": "ds-cts-2" 23:16:05 DEBUG }, 23:16:05 DEBUG "value": [ 23:16:05 DEBUG 1713305765, 23:16:05 DEBUG "20633666.598206338" 23:16:05 DEBUG ] 23:16:05 DEBUG } 23:16:05 DEBUG ] 23:16:05 DEBUG } 23:16:05 DEBUG } 23:16:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305765" 23:16:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713305765" 23:16:05 INFO [http_cmd]: http status code OK 23:16:05 DEBUG --- status code --- 23:16:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:16:05 DEBUG --- http response --- 23:16:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:16:05 DEBUG ----- output ----- 23:16:05 DEBUG { 23:16:05 DEBUG "status": "success", 23:16:05 DEBUG "data": { 23:16:05 DEBUG "resultType": "vector", 23:16:05 DEBUG "result": [ 23:16:05 DEBUG { 23:16:05 DEBUG "metric": { 23:16:05 DEBUG "pod": "am-555f4db595-fz8sn" 23:16:05 DEBUG }, 23:16:05 DEBUG "value": [ 23:16:05 DEBUG 23:16:05 DEBUG 23:16:05 DEBUG 23:16:05 DEBUG [...] 23:16:05 DEBUG 23:16:05 DEBUG 23:16:05 DEBUG 23:16:05 DEBUG "pod": "am-555f4db595-kzxsp" 23:16:05 DEBUG }, 23:16:05 DEBUG "value": [ 23:16:05 DEBUG 1713305765, 23:16:05 DEBUG "2165.1" 23:16:05 DEBUG ] 23:16:05 DEBUG } 23:16:05 DEBUG ] 23:16:05 DEBUG } 23:16:05 DEBUG } 23:16:05 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305765" 23:16:05 INFO [http_cmd]: http status code OK 23:16:05 DEBUG --- status code --- 23:16:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:16:05 DEBUG --- http response --- 23:16:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:16:05 DEBUG ----- output ----- 23:16:05 DEBUG { 23:16:05 DEBUG "status": "success", 23:16:05 DEBUG "data": { 23:16:05 DEBUG "resultType": "vector", 23:16:05 DEBUG "result": [ 23:16:05 DEBUG { 23:16:05 DEBUG "metric": { 23:16:05 DEBUG "instance": "10.142.0.107:9100" 23:16:05 DEBUG }, 23:16:05 DEBUG "value": [ 23:16:05 DEBUG 23:16:05 DEBUG 23:16:05 DEBUG 23:16:05 DEBUG [...] 23:16:05 DEBUG 23:16:05 DEBUG 23:16:05 DEBUG 23:16:05 DEBUG "instance": "10.142.0.43:9100" 23:16:05 DEBUG }, 23:16:05 DEBUG "value": [ 23:16:05 DEBUG 1713305765, 23:16:05 DEBUG "7166.133333333333" 23:16:05 DEBUG ] 23:16:05 DEBUG } 23:16:05 DEBUG ] 23:16:05 DEBUG } 23:16:05 DEBUG } 23:16:05 INFO [http_cmd]: http status code OK 23:16:05 DEBUG --- status code --- 23:16:05 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:16:05 DEBUG --- http response --- 23:16:05 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 23:16:05 DEBUG ----- output ----- 23:16:05 DEBUG { 23:16:05 DEBUG "status": "success", 23:16:05 DEBUG "data": { 23:16:05 DEBUG "resultType": "vector", 23:16:05 DEBUG "result": [ 23:16:05 DEBUG { 23:16:05 DEBUG "metric": { 23:16:05 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 23:16:05 DEBUG }, 23:16:05 DEBUG "value": [ 23:16:05 DEBUG 23:16:05 DEBUG 23:16:05 DEBUG 23:16:05 DEBUG [...] 23:16:05 DEBUG 23:16:05 DEBUG 23:16:05 DEBUG 23:16:05 DEBUG "pod": "ds-cts-2" 23:16:05 DEBUG }, 23:16:05 DEBUG "value": [ 23:16:05 DEBUG 1713305765, 23:16:05 DEBUG "34098842.472787015" 23:16:05 DEBUG ] 23:16:05 DEBUG } 23:16:05 DEBUG ] 23:16:05 DEBUG } 23:16:05 DEBUG } 23:16:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713305766" 23:16:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305766" 23:16:06 INFO [http_cmd]: http status code OK 23:16:06 DEBUG --- status code --- 23:16:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:16:06 DEBUG --- http response --- 23:16:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:16:06 DEBUG ----- output ----- 23:16:06 DEBUG { 23:16:06 DEBUG "status": "success", 23:16:06 DEBUG "data": { 23:16:06 DEBUG "resultType": "vector", 23:16:06 DEBUG "result": [ 23:16:06 DEBUG { 23:16:06 DEBUG "metric": { 23:16:06 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 23:16:06 DEBUG }, 23:16:06 DEBUG "value": [ 23:16:06 DEBUG 23:16:06 DEBUG 23:16:06 DEBUG 23:16:06 DEBUG [...] 23:16:06 DEBUG 23:16:06 DEBUG 23:16:06 DEBUG 23:16:06 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 23:16:06 DEBUG }, 23:16:06 DEBUG "value": [ 23:16:06 DEBUG 1713305766, 23:16:06 DEBUG "1.076623580999586" 23:16:06 DEBUG ] 23:16:06 DEBUG } 23:16:06 DEBUG ] 23:16:06 DEBUG } 23:16:06 DEBUG } 23:16:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713305766" 23:16:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713305766" 23:16:06 INFO [http_cmd]: http status code OK 23:16:06 DEBUG --- status code --- 23:16:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:16:06 DEBUG --- http response --- 23:16:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:16:06 DEBUG ----- output ----- 23:16:06 DEBUG { 23:16:06 DEBUG "status": "success", 23:16:06 DEBUG "data": { 23:16:06 DEBUG "resultType": "vector", 23:16:06 DEBUG "result": [ 23:16:06 DEBUG { 23:16:06 DEBUG "metric": { 23:16:06 DEBUG "pod": "am-555f4db595-fz8sn" 23:16:06 DEBUG }, 23:16:06 DEBUG "value": [ 23:16:06 DEBUG 23:16:06 DEBUG 23:16:06 DEBUG 23:16:06 DEBUG [...] 23:16:06 DEBUG 23:16:06 DEBUG 23:16:06 DEBUG 23:16:06 DEBUG "pod": "am-555f4db595-kzxsp" 23:16:06 DEBUG }, 23:16:06 DEBUG "value": [ 23:16:06 DEBUG 1713305766, 23:16:06 DEBUG "0" 23:16:06 DEBUG ] 23:16:06 DEBUG } 23:16:06 DEBUG ] 23:16:06 DEBUG } 23:16:06 DEBUG } 23:16:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305766" 23:16:06 INFO [http_cmd]: http status code OK 23:16:06 DEBUG --- status code --- 23:16:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:16:06 DEBUG --- http response --- 23:16:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:16:06 DEBUG ----- output ----- 23:16:06 DEBUG { 23:16:06 DEBUG "status": "success", 23:16:06 DEBUG "data": { 23:16:06 DEBUG "resultType": "vector", 23:16:06 DEBUG "result": [ 23:16:06 DEBUG { 23:16:06 DEBUG "metric": { 23:16:06 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 23:16:06 DEBUG }, 23:16:06 DEBUG "value": [ 23:16:06 DEBUG 23:16:06 DEBUG 23:16:06 DEBUG 23:16:06 DEBUG [...] 23:16:06 DEBUG 23:16:06 DEBUG 23:16:06 DEBUG 23:16:06 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 23:16:06 DEBUG }, 23:16:06 DEBUG "value": [ 23:16:06 DEBUG 1713305766, 23:16:06 DEBUG "2394849280" 23:16:06 DEBUG ] 23:16:06 DEBUG } 23:16:06 DEBUG ] 23:16:06 DEBUG } 23:16:06 DEBUG } 23:16:06 INFO [http_cmd]: http status code OK 23:16:06 DEBUG --- status code --- 23:16:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:16:06 DEBUG --- http response --- 23:16:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:16:06 INFO [http_cmd]: http status code OK 23:16:06 DEBUG ----- output ----- 23:16:06 DEBUG --- status code --- 23:16:06 DEBUG { 23:16:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:16:06 DEBUG "status": "success", 23:16:06 DEBUG --- http response --- 23:16:06 DEBUG "data": { 23:16:06 DEBUG "resultType": "vector", 23:16:06 DEBUG "result": [ 23:16:06 DEBUG { 23:16:06 DEBUG "metric": { 23:16:06 DEBUG "instance": "10.142.0.107:9100" 23:16:06 DEBUG }, 23:16:06 DEBUG "value": [ 23:16:06 DEBUG 23:16:06 DEBUG 23:16:06 DEBUG 23:16:06 DEBUG [...] 23:16:06 DEBUG 23:16:06 DEBUG 23:16:06 DEBUG 23:16:06 DEBUG "instance": "10.142.0.43:9100" 23:16:06 DEBUG }, 23:16:06 DEBUG "value": [ 23:16:06 DEBUG 1713305766, 23:16:06 DEBUG "0.00033333333333333365" 23:16:06 DEBUG ] 23:16:06 DEBUG } 23:16:06 DEBUG ] 23:16:06 DEBUG } 23:16:06 DEBUG } 23:16:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:16:06 DEBUG ----- output ----- 23:16:06 DEBUG { 23:16:06 DEBUG "status": "success", 23:16:06 DEBUG "data": { 23:16:06 DEBUG "resultType": "vector", 23:16:06 DEBUG "result": [ 23:16:06 DEBUG { 23:16:06 DEBUG "metric": { 23:16:06 DEBUG "pod": "ds-idrepo-2" 23:16:06 DEBUG }, 23:16:06 DEBUG "value": [ 23:16:06 DEBUG 23:16:06 DEBUG 23:16:06 DEBUG 23:16:06 DEBUG [...] 23:16:06 DEBUG 23:16:06 DEBUG 23:16:06 DEBUG 23:16:06 DEBUG "pod": "ds-idrepo-0" 23:16:06 DEBUG }, 23:16:06 DEBUG "value": [ 23:16:06 DEBUG 1713305766, 23:16:06 DEBUG "0" 23:16:06 DEBUG ] 23:16:06 DEBUG } 23:16:06 DEBUG ] 23:16:06 DEBUG } 23:16:06 DEBUG } 23:16:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713305766" 23:16:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305766" 23:16:06 INFO [http_cmd]: http status code OK 23:16:06 DEBUG --- status code --- 23:16:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:16:06 DEBUG --- http response --- 23:16:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:16:06 DEBUG ----- output ----- 23:16:06 DEBUG { 23:16:06 DEBUG "status": "success", 23:16:06 DEBUG "data": { 23:16:06 DEBUG "resultType": "vector", 23:16:06 DEBUG "result": [ 23:16:06 DEBUG { 23:16:06 DEBUG "metric": { 23:16:06 DEBUG "pod": "am-555f4db595-fz8sn" 23:16:06 DEBUG }, 23:16:06 DEBUG "value": [ 23:16:06 DEBUG 23:16:06 DEBUG 23:16:06 DEBUG 23:16:06 DEBUG [...] 23:16:06 DEBUG 23:16:06 DEBUG 23:16:06 DEBUG 23:16:06 DEBUG "pod": "am-555f4db595-kzxsp" 23:16:06 DEBUG }, 23:16:06 DEBUG "value": [ 23:16:06 DEBUG 1713305766, 23:16:06 DEBUG "0" 23:16:06 DEBUG ] 23:16:06 DEBUG } 23:16:06 DEBUG ] 23:16:06 DEBUG } 23:16:06 DEBUG } 23:16:06 INFO [http_cmd]: http status code OK 23:16:06 DEBUG --- status code --- 23:16:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:16:06 DEBUG --- http response --- 23:16:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:16:06 DEBUG ----- output ----- 23:16:06 DEBUG { 23:16:06 DEBUG "status": "success", 23:16:06 DEBUG "data": { 23:16:06 DEBUG "resultType": "vector", 23:16:06 DEBUG "result": [ 23:16:06 DEBUG { 23:16:06 DEBUG "metric": { 23:16:06 DEBUG "instance": "10.142.0.107:9100" 23:16:06 DEBUG }, 23:16:06 DEBUG "value": [ 23:16:06 DEBUG 23:16:06 DEBUG 23:16:06 DEBUG 23:16:06 DEBUG [...] 23:16:06 DEBUG 23:16:06 DEBUG 23:16:06 DEBUG 23:16:06 DEBUG "instance": "10.142.0.43:9100" 23:16:06 DEBUG }, 23:16:06 DEBUG "value": [ 23:16:06 DEBUG 1713305766, 23:16:06 DEBUG "0.0005333333333330377" 23:16:06 DEBUG ] 23:16:06 DEBUG } 23:16:06 DEBUG ] 23:16:06 DEBUG } 23:16:06 DEBUG } 23:16:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305766" 23:16:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713305766" 23:16:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305766" 23:16:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305766" 23:16:06 INFO [http_cmd]: http status code OK 23:16:06 INFO [http_cmd]: http status code OK 23:16:06 INFO [http_cmd]: http status code OK 23:16:06 DEBUG --- status code --- 23:16:06 DEBUG --- status code --- 23:16:06 DEBUG --- status code --- 23:16:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:16:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:16:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:16:06 DEBUG --- http response --- 23:16:06 DEBUG --- http response --- 23:16:06 DEBUG --- http response --- 23:16:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:16:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 23:16:06 DEBUG ----- output ----- 23:16:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 23:16:06 DEBUG ----- output ----- 23:16:06 DEBUG { 23:16:06 DEBUG ----- output ----- 23:16:06 DEBUG { 23:16:06 DEBUG "status": "success", 23:16:06 DEBUG { 23:16:06 DEBUG "status": "success", 23:16:06 DEBUG "data": { 23:16:06 DEBUG "status": "success", 23:16:06 DEBUG "data": { 23:16:06 DEBUG "resultType": "vector", 23:16:06 DEBUG "data": { 23:16:06 DEBUG "resultType": "vector", 23:16:06 DEBUG "result": [ 23:16:06 DEBUG "resultType": "vector", 23:16:06 DEBUG "result": [ 23:16:06 DEBUG { 23:16:06 DEBUG "result": [ 23:16:06 DEBUG { 23:16:06 DEBUG "metric": { 23:16:06 DEBUG { 23:16:06 DEBUG "metric": { 23:16:06 DEBUG "pod": "am-555f4db595-fz8sn" 23:16:06 DEBUG "metric": { 23:16:06 DEBUG "pod": "ds-idrepo-2" 23:16:06 DEBUG }, 23:16:06 DEBUG "pod": "ds-idrepo-2" 23:16:06 DEBUG }, 23:16:06 DEBUG "value": [ 23:16:06 DEBUG }, 23:16:06 DEBUG "value": [ 23:16:06 DEBUG 23:16:06 DEBUG "value": [ 23:16:06 DEBUG 23:16:06 DEBUG 23:16:06 DEBUG 23:16:06 DEBUG 23:16:06 DEBUG 23:16:06 DEBUG 23:16:06 DEBUG 23:16:06 DEBUG [...] 23:16:06 DEBUG 23:16:06 DEBUG [...] 23:16:06 DEBUG 23:16:06 DEBUG [...] 23:16:06 DEBUG 23:16:06 DEBUG 23:16:06 DEBUG 23:16:06 DEBUG 23:16:06 DEBUG 23:16:06 DEBUG 23:16:06 DEBUG 23:16:06 DEBUG "pod": "am-555f4db595-kzxsp" 23:16:06 DEBUG 23:16:06 DEBUG "pod": "ds-cts-2" 23:16:06 DEBUG }, 23:16:06 DEBUG "pod": "ds-idrepo-0" 23:16:06 DEBUG }, 23:16:06 DEBUG "value": [ 23:16:06 DEBUG }, 23:16:06 DEBUG "value": [ 23:16:06 DEBUG 1713305766, 23:16:06 DEBUG "value": [ 23:16:06 DEBUG 1713305766, 23:16:06 INFO [http_cmd]: http status code OK 23:16:06 DEBUG "0" 23:16:06 DEBUG 1713305766, 23:16:06 DEBUG "0" 23:16:06 DEBUG --- status code --- 23:16:06 DEBUG ] 23:16:06 DEBUG "0" 23:16:06 DEBUG ] 23:16:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:16:06 DEBUG } 23:16:06 DEBUG ] 23:16:06 DEBUG } 23:16:06 DEBUG --- http response --- 23:16:06 DEBUG ] 23:16:06 DEBUG } 23:16:06 DEBUG ] 23:16:06 DEBUG } 23:16:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 23:16:06 DEBUG ] 23:16:06 DEBUG } 23:16:06 DEBUG } 23:16:06 DEBUG ----- output ----- 23:16:06 DEBUG } 23:16:06 DEBUG } 23:16:06 DEBUG { 23:16:06 DEBUG } 23:16:06 DEBUG "status": "success", 23:16:06 DEBUG "data": { 23:16:06 DEBUG "resultType": "vector", 23:16:06 DEBUG "result": [ 23:16:06 DEBUG { 23:16:06 DEBUG "metric": { 23:16:06 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 23:16:06 DEBUG }, 23:16:06 DEBUG "value": [ 23:16:06 DEBUG 23:16:06 DEBUG 23:16:06 DEBUG 23:16:06 DEBUG [...] 23:16:06 DEBUG 23:16:06 DEBUG 23:16:06 DEBUG 23:16:06 DEBUG "pod": "ds-cts-2" 23:16:06 DEBUG }, 23:16:06 DEBUG "value": [ 23:16:06 DEBUG 1713305766, 23:16:06 DEBUG "0" 23:16:06 DEBUG ] 23:16:06 DEBUG } 23:16:06 DEBUG ] 23:16:06 DEBUG } 23:16:06 DEBUG } 23:16:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305766" 23:16:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713305766" 23:16:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713305766" 23:16:06 INFO [http_cmd]: http status code OK 23:16:06 DEBUG --- status code --- 23:16:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:16:06 DEBUG --- http response --- 23:16:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:16:06 DEBUG ----- output ----- 23:16:06 DEBUG { 23:16:06 DEBUG "status": "success", 23:16:06 DEBUG "data": { 23:16:06 DEBUG "resultType": "vector", 23:16:06 DEBUG "result": [ 23:16:06 DEBUG { 23:16:06 DEBUG "metric": { 23:16:06 DEBUG "pod": "ds-idrepo-2" 23:16:06 DEBUG }, 23:16:06 DEBUG "value": [ 23:16:06 DEBUG 23:16:06 DEBUG 23:16:06 DEBUG 23:16:06 DEBUG [...] 23:16:06 DEBUG 23:16:06 DEBUG 23:16:06 DEBUG 23:16:06 DEBUG "pod": "ds-idrepo-0" 23:16:06 DEBUG }, 23:16:06 DEBUG "value": [ 23:16:06 DEBUG 1713305766, 23:16:06 DEBUG "0" 23:16:06 DEBUG ] 23:16:06 DEBUG } 23:16:06 DEBUG ] 23:16:06 DEBUG } 23:16:06 DEBUG } 23:16:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713305766" 23:16:06 INFO [http_cmd]: http status code OK 23:16:06 INFO [http_cmd]: http status code OK 23:16:06 DEBUG --- status code --- 23:16:06 DEBUG --- status code --- 23:16:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:16:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:16:06 DEBUG --- http response --- 23:16:06 DEBUG --- http response --- 23:16:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:16:06 DEBUG ----- output ----- 23:16:06 DEBUG { 23:16:06 DEBUG "status": "success", 23:16:06 DEBUG "data": { 23:16:06 DEBUG "resultType": "vector", 23:16:06 DEBUG "result": [ 23:16:06 DEBUG { 23:16:06 DEBUG "metric": { 23:16:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:16:06 DEBUG "pod": "am-555f4db595-fz8sn" 23:16:06 DEBUG ----- output ----- 23:16:06 DEBUG }, 23:16:06 DEBUG { 23:16:06 DEBUG "value": [ 23:16:06 DEBUG "status": "success", 23:16:06 DEBUG 23:16:06 DEBUG "data": { 23:16:06 DEBUG 23:16:06 DEBUG "resultType": "vector", 23:16:06 DEBUG 23:16:06 DEBUG "result": [ 23:16:06 DEBUG [...] 23:16:06 DEBUG { 23:16:06 DEBUG 23:16:06 DEBUG "metric": { 23:16:06 DEBUG 23:16:06 DEBUG "instance": "10.142.0.107:9100" 23:16:06 DEBUG 23:16:06 DEBUG }, 23:16:06 DEBUG "pod": "am-555f4db595-kzxsp" 23:16:06 DEBUG "value": [ 23:16:06 DEBUG }, 23:16:06 DEBUG 23:16:06 DEBUG "value": [ 23:16:06 DEBUG 23:16:06 DEBUG 1713305766, 23:16:06 DEBUG 23:16:06 DEBUG "0.029" 23:16:06 DEBUG [...] 23:16:06 DEBUG ] 23:16:06 DEBUG 23:16:06 DEBUG } 23:16:06 DEBUG 23:16:06 DEBUG ] 23:16:06 DEBUG 23:16:06 DEBUG } 23:16:06 DEBUG "instance": "10.142.0.43:9100" 23:16:06 DEBUG } 23:16:06 DEBUG }, 23:16:06 DEBUG "value": [ 23:16:06 DEBUG 1713305766, 23:16:06 DEBUG "16334.233333333332" 23:16:06 DEBUG ] 23:16:06 DEBUG } 23:16:06 DEBUG ] 23:16:06 DEBUG } 23:16:06 DEBUG } 23:16:06 INFO [http_cmd]: http status code OK 23:16:06 DEBUG --- status code --- 23:16:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:16:06 DEBUG --- http response --- 23:16:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 23:16:06 DEBUG ----- output ----- 23:16:06 DEBUG { 23:16:06 DEBUG "status": "success", 23:16:06 DEBUG "data": { 23:16:06 DEBUG "resultType": "vector", 23:16:06 DEBUG "result": [ 23:16:06 DEBUG { 23:16:06 DEBUG "metric": { 23:16:06 DEBUG "instance": "10.142.0.107:9100" 23:16:06 DEBUG }, 23:16:06 DEBUG "value": [ 23:16:06 DEBUG 23:16:06 DEBUG 23:16:06 DEBUG 23:16:06 DEBUG [...] 23:16:06 DEBUG 23:16:06 DEBUG 23:16:06 DEBUG 23:16:06 DEBUG "instance": "10.142.0.43:9100" 23:16:06 DEBUG }, 23:16:06 DEBUG "value": [ 23:16:06 DEBUG 1713305766, 23:16:06 DEBUG "19992093696" 23:16:06 DEBUG ] 23:16:06 DEBUG } 23:16:06 DEBUG ] 23:16:06 DEBUG } 23:16:06 DEBUG } 23:16:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305766" 23:16:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305766" 23:16:06 INFO [http_cmd]: http status code OK 23:16:06 DEBUG --- status code --- 23:16:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:16:06 DEBUG --- http response --- 23:16:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:16:06 DEBUG ----- output ----- 23:16:06 DEBUG { 23:16:06 DEBUG "status": "success", 23:16:06 DEBUG "data": { 23:16:06 DEBUG "resultType": "vector", 23:16:06 DEBUG "result": [ 23:16:06 DEBUG { 23:16:06 DEBUG "metric": { 23:16:06 DEBUG "pod": "ds-idrepo-2" 23:16:06 DEBUG }, 23:16:06 DEBUG "value": [ 23:16:06 DEBUG 23:16:06 DEBUG 23:16:06 DEBUG 23:16:06 DEBUG [...] 23:16:06 DEBUG 23:16:06 DEBUG 23:16:06 DEBUG 23:16:06 DEBUG "pod": "ds-idrepo-0" 23:16:06 DEBUG }, 23:16:06 DEBUG "value": [ 23:16:06 DEBUG 1713305766, 23:16:06 DEBUG "0" 23:16:06 DEBUG ] 23:16:06 DEBUG } 23:16:06 DEBUG ] 23:16:06 DEBUG } 23:16:06 DEBUG } 23:16:06 INFO [http_cmd]: http status code OK 23:16:06 DEBUG --- status code --- 23:16:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:16:06 DEBUG --- http response --- 23:16:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 23:16:06 DEBUG ----- output ----- 23:16:06 DEBUG { 23:16:06 DEBUG "status": "success", 23:16:06 DEBUG "data": { 23:16:06 DEBUG "resultType": "vector", 23:16:06 DEBUG "result": [ 23:16:06 DEBUG { 23:16:06 DEBUG "metric": { 23:16:06 DEBUG "pod": "ds-idrepo-2" 23:16:06 DEBUG }, 23:16:06 DEBUG "value": [ 23:16:06 DEBUG 23:16:06 DEBUG 23:16:06 DEBUG 23:16:06 DEBUG [...] 23:16:06 DEBUG 23:16:06 DEBUG 23:16:06 DEBUG 23:16:06 DEBUG "pod": "ds-idrepo-0" 23:16:06 DEBUG }, 23:16:06 DEBUG "value": [ 23:16:06 DEBUG 1713305766, 23:16:06 DEBUG "0" 23:16:06 DEBUG ] 23:16:06 DEBUG } 23:16:06 DEBUG ] 23:16:06 DEBUG } 23:16:06 DEBUG } 23:16:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305766" 23:16:06 INFO [http_cmd]: http status code OK 23:16:06 DEBUG --- status code --- 23:16:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:16:06 DEBUG --- http response --- 23:16:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:16:06 DEBUG ----- output ----- 23:16:06 DEBUG { 23:16:06 DEBUG "status": "success", 23:16:06 DEBUG "data": { 23:16:06 DEBUG "resultType": "vector", 23:16:06 DEBUG "result": [ 23:16:06 DEBUG { 23:16:06 DEBUG "metric": { 23:16:06 DEBUG "pod": "am-555f4db595-fz8sn" 23:16:06 DEBUG }, 23:16:06 DEBUG "value": [ 23:16:06 DEBUG 23:16:06 DEBUG 23:16:06 DEBUG 23:16:06 DEBUG [...] 23:16:06 DEBUG 23:16:06 DEBUG 23:16:06 DEBUG 23:16:06 DEBUG "pod": "am-555f4db595-kzxsp" 23:16:06 DEBUG }, 23:16:06 DEBUG "value": [ 23:16:06 DEBUG 1713305766, 23:16:06 DEBUG "2780778" 23:16:06 DEBUG ] 23:16:06 DEBUG } 23:16:06 DEBUG ] 23:16:06 DEBUG } 23:16:06 DEBUG } 23:16:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305766" 23:16:06 INFO [http_cmd]: http status code OK 23:16:06 DEBUG --- status code --- 23:16:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:16:06 DEBUG --- http response --- 23:16:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 23:16:06 DEBUG ----- output ----- 23:16:06 DEBUG { 23:16:06 DEBUG "status": "success", 23:16:06 DEBUG "data": { 23:16:06 DEBUG "resultType": "vector", 23:16:06 DEBUG "result": [ 23:16:06 DEBUG { 23:16:06 DEBUG "metric": { 23:16:06 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 23:16:06 DEBUG }, 23:16:06 DEBUG "value": [ 23:16:06 DEBUG 23:16:06 DEBUG 23:16:06 DEBUG 23:16:06 DEBUG [...] 23:16:06 DEBUG 23:16:06 DEBUG 23:16:06 DEBUG 23:16:06 DEBUG "pod": "idm-b85f9d978-j5pbm" 23:16:06 DEBUG }, 23:16:06 DEBUG "value": [ 23:16:06 DEBUG 1713305766, 23:16:06 DEBUG "483.63877304008025" 23:16:06 DEBUG ] 23:16:06 DEBUG } 23:16:06 DEBUG ] 23:16:06 DEBUG } 23:16:06 DEBUG } 23:16:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305766" 23:16:06 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713305766" 23:16:06 INFO [http_cmd]: http status code OK 23:16:06 DEBUG --- status code --- 23:16:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:16:06 DEBUG --- http response --- 23:16:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 23:16:06 DEBUG ----- output ----- 23:16:06 DEBUG { 23:16:06 DEBUG "status": "success", 23:16:06 DEBUG "data": { 23:16:06 DEBUG "resultType": "vector", 23:16:06 DEBUG "result": [ 23:16:06 DEBUG { 23:16:06 DEBUG "metric": { 23:16:06 DEBUG "pod": "ds-idrepo-2" 23:16:06 DEBUG }, 23:16:06 DEBUG "value": [ 23:16:06 DEBUG 23:16:06 DEBUG 23:16:06 DEBUG 23:16:06 DEBUG [...] 23:16:06 DEBUG 23:16:06 DEBUG 23:16:06 DEBUG 23:16:06 DEBUG "pod": "ds-idrepo-0" 23:16:06 DEBUG }, 23:16:06 DEBUG "value": [ 23:16:06 DEBUG 1713305766, 23:16:06 DEBUG "0" 23:16:06 DEBUG ] 23:16:06 DEBUG } 23:16:06 DEBUG ] 23:16:06 DEBUG } 23:16:06 DEBUG } 23:16:06 INFO [http_cmd]: http status code OK 23:16:06 DEBUG --- status code --- 23:16:06 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 23:16:06 DEBUG --- http response --- 23:16:06 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 23:16:06 DEBUG ----- output ----- 23:16:06 DEBUG { 23:16:06 DEBUG "status": "success", 23:16:06 DEBUG "data": { 23:16:06 DEBUG "resultType": "vector", 23:16:06 DEBUG "result": [ 23:16:06 DEBUG { 23:16:06 DEBUG "metric": { 23:16:06 DEBUG "pod": "am-555f4db595-fz8sn" 23:16:06 DEBUG }, 23:16:06 DEBUG "value": [ 23:16:06 DEBUG 23:16:06 DEBUG 23:16:06 DEBUG 23:16:06 DEBUG [...] 23:16:06 DEBUG 23:16:06 DEBUG 23:16:06 DEBUG 23:16:06 DEBUG "pod": "am-555f4db595-kzxsp" 23:16:06 DEBUG }, 23:16:06 DEBUG "value": [ 23:16:06 DEBUG 1713305766, 23:16:06 DEBUG "0" 23:16:06 DEBUG ] 23:16:06 DEBUG } 23:16:06 DEBUG ] 23:16:06 DEBUG } 23:16:06 DEBUG } 23:16:06 INFO 23:16:06 INFO [loop_until]: kubectl --namespace=xlou top node 23:16:06 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 23:16:06 INFO [loop_until]: OK (rc = 0) 23:16:06 DEBUG --- stdout --- 23:16:06 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 5088Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 8583m 28% 16249Mi 14% gke-xlou-cdm-default-pool-1b0769c8-nhlf 2055m 6% 10799Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 81m 0% 2658Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 646m 2% 6610Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 9738m 32% 17494Mi 15% gke-xlou-cdm-default-pool-1b0769c8-xnmp 57m 0% 3156Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 2139m 7% 10494Mi 9% gke-xlou-cdm-ds-0f601751-hmm7 685m 2% 5670Mi 5% gke-xlou-cdm-ds-0f601751-j488 8625m 28% 15712Mi 14% gke-xlou-cdm-ds-0f601751-mc5d 80m 0% 3606Mi 3% gke-xlou-cdm-ds-0f601751-nj6q 2150m 7% 10481Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 668m 2% 7386Mi 6% gke-xlou-cdm-frontend-a8771548-i82n 1114m 7% 8293Mi 14% 23:16:06 DEBUG --- stderr --- 23:16:06 DEBUG 23:16:17 INFO Finished: True 23:16:17 INFO Waiting for threads to register finish flag 23:17:06 INFO Done. Have a nice day! :) 127.0.0.1 - - [16/Apr/2024 23:17:06] "GET /monitoring/stop HTTP/1.1" 200 - 23:17:08 WARNING File /tmp/lodemon_data-20240416_220944/node_disk_read_bytes_total.json does not exist. Skipping graph generation... WARNING:bokeh.core.validation.check:W-1000 (MISSING_RENDERERS): Plot has no renderers: figure(id='p6521', ...) 127.0.0.1 - - [16/Apr/2024 23:17:12] "GET /monitoring/process HTTP/1.1" 200 -